From 82f9d18a03cc82b97d81ba427fdb43b150e2e99a Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:06:45 +0300 Subject: [PATCH 1/4] add blob size metrics --- Cargo.lock | 1 + core/node/proof_data_handler/Cargo.toml | 1 + core/node/proof_data_handler/src/lib.rs | 1 + core/node/proof_data_handler/src/metrics.rs | 53 +++++++++++++++++++ .../src/request_processor.rs | 4 +- 5 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 core/node/proof_data_handler/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index a5093d36a7cf..acbddaf554b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9165,6 +9165,7 @@ dependencies = [ "tokio", "tower", "tracing", + "vise", "zksync_basic_types", "zksync_config", "zksync_contracts", diff --git a/core/node/proof_data_handler/Cargo.toml b/core/node/proof_data_handler/Cargo.toml index 0ab5d4bb1913..92e6b45f6fa5 100644 --- a/core/node/proof_data_handler/Cargo.toml +++ b/core/node/proof_data_handler/Cargo.toml @@ -10,6 +10,7 @@ keywords.workspace = true categories.workspace = true [dependencies] +vise.workspace = true zksync_config.workspace = true zksync_dal.workspace = true zksync_object_store.workspace = true diff --git a/core/node/proof_data_handler/src/lib.rs b/core/node/proof_data_handler/src/lib.rs index 06b88b395139..618a786ea658 100644 --- a/core/node/proof_data_handler/src/lib.rs +++ b/core/node/proof_data_handler/src/lib.rs @@ -18,6 +18,7 @@ use zksync_types::commitment::L1BatchCommitmentMode; mod tests; mod errors; +mod metrics; mod request_processor; mod tee_request_processor; diff --git a/core/node/proof_data_handler/src/metrics.rs b/core/node/proof_data_handler/src/metrics.rs new file mode 100644 index 000000000000..c3df429232ac --- /dev/null +++ b/core/node/proof_data_handler/src/metrics.rs @@ -0,0 +1,53 @@ +use vise::{Histogram, Metrics}; +use zksync_basic_types::basic_fri_types::Eip4844Blobs; +use zksync_object_store::StoredObject; +use zksync_prover_interface::inputs::{ + VMRunWitnessInputData, WitnessInputData, WitnessInputMerklePaths, +}; + +#[derive(Debug, Metrics)] +pub(super) struct ProofDataHandlerMetrics { + #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))] + pub vm_run_data_blob_size_in_mb: Histogram, + #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))] + pub merkle_paths_blob_size_in_mb: Histogram, + #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))] + pub eip_4844_blob_size_in_mb: Histogram, + #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))] + pub total_blob_size_in_mb: Histogram, +} + +impl ProofDataHandlerMetrics { + pub fn observe_blob_sizes(&self, blob: &WitnessInputData) { + let vm_run_data_blob_size_in_mb = + ::serialize(&blob.vm_run_data) + .unwrap() + .len() as u64 + / (1024 * 1024); + let merkle_paths_blob_size_in_mb = + ::serialize(&blob.merkle_paths) + .unwrap() + .len() as u64 + / (1024 * 1024); + let eip_4844_blob_size_in_mb = + ::serialize(&blob.eip_4844_blobs) + .unwrap() + .len() as u64 + / (1024 * 1024); + let total_blob_size_in_mb = ::serialize(blob) + .unwrap() + .len() as u64 + / (1024 * 1024); + + self.vm_run_data_blob_size_in_mb + .observe(vm_run_data_blob_size_in_mb); + self.merkle_paths_blob_size_in_mb + .observe(merkle_paths_blob_size_in_mb); + self.eip_4844_blob_size_in_mb + .observe(eip_4844_blob_size_in_mb); + self.total_blob_size_in_mb.observe(total_blob_size_in_mb); + } +} + +#[vise::register] +pub(super) static METRICS: vise::Global = vise::Global::new(); diff --git a/core/node/proof_data_handler/src/request_processor.rs b/core/node/proof_data_handler/src/request_processor.rs index bdb55237c4b6..a89f9b63a848 100644 --- a/core/node/proof_data_handler/src/request_processor.rs +++ b/core/node/proof_data_handler/src/request_processor.rs @@ -20,7 +20,7 @@ use zksync_types::{ L1BatchNumber, H256, }; -use crate::errors::RequestProcessorError; +use crate::{errors::RequestProcessorError, metrics::METRICS}; #[derive(Clone)] pub(crate) struct RequestProcessor { @@ -147,6 +147,8 @@ impl RequestProcessor { }, }; + METRICS.observe_blob_sizes(&blob); + let proof_gen_data = ProofGenerationData { l1_batch_number, witness_input_data: blob, From 4d4a0de65c0cf38644bc5a58c8c64514d38c236f Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:26:26 +0300 Subject: [PATCH 2/4] remove eip4844 blob metrics --- core/node/proof_data_handler/src/metrics.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/core/node/proof_data_handler/src/metrics.rs b/core/node/proof_data_handler/src/metrics.rs index c3df429232ac..c3525ec2fcf1 100644 --- a/core/node/proof_data_handler/src/metrics.rs +++ b/core/node/proof_data_handler/src/metrics.rs @@ -1,5 +1,4 @@ use vise::{Histogram, Metrics}; -use zksync_basic_types::basic_fri_types::Eip4844Blobs; use zksync_object_store::StoredObject; use zksync_prover_interface::inputs::{ VMRunWitnessInputData, WitnessInputData, WitnessInputMerklePaths, @@ -12,8 +11,6 @@ pub(super) struct ProofDataHandlerMetrics { #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))] pub merkle_paths_blob_size_in_mb: Histogram, #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))] - pub eip_4844_blob_size_in_mb: Histogram, - #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))] pub total_blob_size_in_mb: Histogram, } @@ -29,11 +26,6 @@ impl ProofDataHandlerMetrics { .unwrap() .len() as u64 / (1024 * 1024); - let eip_4844_blob_size_in_mb = - ::serialize(&blob.eip_4844_blobs) - .unwrap() - .len() as u64 - / (1024 * 1024); let total_blob_size_in_mb = ::serialize(blob) .unwrap() .len() as u64 @@ -43,8 +35,6 @@ impl ProofDataHandlerMetrics { .observe(vm_run_data_blob_size_in_mb); self.merkle_paths_blob_size_in_mb .observe(merkle_paths_blob_size_in_mb); - self.eip_4844_blob_size_in_mb - .observe(eip_4844_blob_size_in_mb); self.total_blob_size_in_mb.observe(total_blob_size_in_mb); } } From 18515a15ab64f1beaa0adcc52370eb39489ba683 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Wed, 10 Jul 2024 12:34:57 +0300 Subject: [PATCH 3/4] Revert "remove eip4844 blob metrics" This reverts commit 4d4a0de65c0cf38644bc5a58c8c64514d38c236f. --- core/node/proof_data_handler/src/metrics.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/node/proof_data_handler/src/metrics.rs b/core/node/proof_data_handler/src/metrics.rs index c3525ec2fcf1..c3df429232ac 100644 --- a/core/node/proof_data_handler/src/metrics.rs +++ b/core/node/proof_data_handler/src/metrics.rs @@ -1,4 +1,5 @@ use vise::{Histogram, Metrics}; +use zksync_basic_types::basic_fri_types::Eip4844Blobs; use zksync_object_store::StoredObject; use zksync_prover_interface::inputs::{ VMRunWitnessInputData, WitnessInputData, WitnessInputMerklePaths, @@ -11,6 +12,8 @@ pub(super) struct ProofDataHandlerMetrics { #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))] pub merkle_paths_blob_size_in_mb: Histogram, #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))] + pub eip_4844_blob_size_in_mb: Histogram, + #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))] pub total_blob_size_in_mb: Histogram, } @@ -26,6 +29,11 @@ impl ProofDataHandlerMetrics { .unwrap() .len() as u64 / (1024 * 1024); + let eip_4844_blob_size_in_mb = + ::serialize(&blob.eip_4844_blobs) + .unwrap() + .len() as u64 + / (1024 * 1024); let total_blob_size_in_mb = ::serialize(blob) .unwrap() .len() as u64 @@ -35,6 +43,8 @@ impl ProofDataHandlerMetrics { .observe(vm_run_data_blob_size_in_mb); self.merkle_paths_blob_size_in_mb .observe(merkle_paths_blob_size_in_mb); + self.eip_4844_blob_size_in_mb + .observe(eip_4844_blob_size_in_mb); self.total_blob_size_in_mb.observe(total_blob_size_in_mb); } } From e56e683f4d8ecfc718fc077cd819a6ca3360336f Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Wed, 10 Jul 2024 12:42:28 +0300 Subject: [PATCH 4/4] fix 4844 metrics --- core/node/proof_data_handler/src/metrics.rs | 30 +++++++-------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/core/node/proof_data_handler/src/metrics.rs b/core/node/proof_data_handler/src/metrics.rs index c3df429232ac..edccda90dc24 100644 --- a/core/node/proof_data_handler/src/metrics.rs +++ b/core/node/proof_data_handler/src/metrics.rs @@ -1,9 +1,8 @@ use vise::{Histogram, Metrics}; -use zksync_basic_types::basic_fri_types::Eip4844Blobs; -use zksync_object_store::StoredObject; -use zksync_prover_interface::inputs::{ - VMRunWitnessInputData, WitnessInputData, WitnessInputMerklePaths, -}; +use zksync_object_store::bincode; +use zksync_prover_interface::inputs::WitnessInputData; + +const BYTES_IN_MEGABYTE: u64 = 1024 * 1024; #[derive(Debug, Metrics)] pub(super) struct ProofDataHandlerMetrics { @@ -20,24 +19,13 @@ pub(super) struct ProofDataHandlerMetrics { impl ProofDataHandlerMetrics { pub fn observe_blob_sizes(&self, blob: &WitnessInputData) { let vm_run_data_blob_size_in_mb = - ::serialize(&blob.vm_run_data) - .unwrap() - .len() as u64 - / (1024 * 1024); + bincode::serialize(&blob.vm_run_data).unwrap().len() as u64 / BYTES_IN_MEGABYTE; let merkle_paths_blob_size_in_mb = - ::serialize(&blob.merkle_paths) - .unwrap() - .len() as u64 - / (1024 * 1024); + bincode::serialize(&blob.merkle_paths).unwrap().len() as u64 / BYTES_IN_MEGABYTE; let eip_4844_blob_size_in_mb = - ::serialize(&blob.eip_4844_blobs) - .unwrap() - .len() as u64 - / (1024 * 1024); - let total_blob_size_in_mb = ::serialize(blob) - .unwrap() - .len() as u64 - / (1024 * 1024); + bincode::serialize(&blob.eip_4844_blobs).unwrap().len() as u64 / BYTES_IN_MEGABYTE; + let total_blob_size_in_mb = + bincode::serialize(blob).unwrap().len() as u64 / BYTES_IN_MEGABYTE; self.vm_run_data_blob_size_in_mb .observe(vm_run_data_blob_size_in_mb);