Merge remote-tracking branch 'origin/main' into daniyar/vm_runner/tim…

…e_taken

.github/workflows/ci-core-reusable.yml

@@ -135,7 +135,7 @@ jobs:
         base_token: ["Eth", "Custom"]
         deployment_mode: ["Rollup", "Validium"]
     env:
-      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,da_dispatcher,base_token_ratio_persister${{ matrix.consensus && ',consensus' || '' }}"
+      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,da_dispatcher,base_token_ratio_persister${{ matrix.consensus && ',consensus' || '' }}"
 
     runs-on: [matterlabs-ci-runner]
     steps:
@@ -309,7 +309,7 @@ jobs:
     runs-on: [matterlabs-ci-runner]
 
     env:
-      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,da_dispatcher,base_token_ratio_persister${{ matrix.consensus && ',consensus' || '' }}"
+      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,da_dispatcher,base_token_ratio_persister${{ matrix.consensus && ',consensus' || '' }}"
       EXT_NODE_FLAGS: "${{ matrix.consensus && '-- --enable-consensus' || '' }}"
 
     steps:

core/bin/zksync_server/src/main.rs

@@ -11,10 +11,10 @@ use zksync_config::{
         },
         fri_prover_group::FriProverGroupConfig,
         house_keeper::HouseKeeperConfig,
-        ContractsConfig, DatabaseSecrets, FriProofCompressorConfig, FriProverConfig,
-        FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig,
-        L1Secrets, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig,
-        ProtectiveReadsWriterConfig, Secrets,
+        BasicWitnessInputProducerConfig, ContractsConfig, DatabaseSecrets,
+        FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig,
+        FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, L1Secrets, ObservabilityConfig,
+        PrometheusConfig, ProofDataHandlerConfig, ProtectiveReadsWriterConfig, Secrets,
     },
     ApiConfig, BaseTokenAdjusterConfig, ContractVerifierConfig, DADispatcherConfig, DBConfig,
     EthConfig, EthWatchConfig, GasAdjusterConfig, GenesisConfig, ObjectStoreConfig, PostgresConfig,
@@ -271,6 +271,7 @@ fn load_env_config() -> anyhow::Result<TempConfigStore> {
         snapshot_creator: SnapshotsCreatorConfig::from_env().ok(),
         da_dispatcher_config: DADispatcherConfig::from_env().ok(),
         protective_reads_writer_config: ProtectiveReadsWriterConfig::from_env().ok(),
+        basic_witness_input_producer_config: BasicWitnessInputProducerConfig::from_env().ok(),
         core_object_store: ObjectStoreConfig::from_env().ok(),
         base_token_adjuster_config: BaseTokenAdjusterConfig::from_env().ok(),
         commitment_generator: None,

core/bin/zksync_server/src/node_builder.rs

@@ -48,7 +48,9 @@ use zksync_node_framework::{
             output_handler::OutputHandlerLayer, RocksdbStorageOptions, StateKeeperLayer,
         },
         tee_verifier_input_producer::TeeVerifierInputProducerLayer,
-        vm_runner::protective_reads::ProtectiveReadsWriterLayer,
+        vm_runner::{
+            bwip::BasicWitnessInputProducerLayer, protective_reads::ProtectiveReadsWriterLayer,
+        },
         web3_api::{
             caches::MempoolCacheLayer,
             server::{Web3ServerLayer, Web3ServerOptionalConfig},
@@ -503,6 +505,17 @@ impl MainNodeBuilder {
         Ok(self)
     }
 
+    fn add_vm_runner_bwip_layer(mut self) -> anyhow::Result<Self> {
+        let basic_witness_input_producer_config =
+            try_load_config!(self.configs.basic_witness_input_producer_config);
+        self.node.add_layer(BasicWitnessInputProducerLayer::new(
+            basic_witness_input_producer_config,
+            self.genesis_config.l2_chain_id,
+        ));
+
+        Ok(self)
+    }
+
     fn add_base_token_ratio_persister_layer(mut self) -> anyhow::Result<Self> {
         let config = try_load_config!(self.configs.base_token_adjuster);
         self.node
@@ -604,6 +617,9 @@ impl MainNodeBuilder {
                 Component::BaseTokenRatioPersister => {
                     self = self.add_base_token_ratio_persister_layer()?;
                 }
+                Component::VmRunnerBwip => {
+                    self = self.add_vm_runner_bwip_layer()?;
+                }
             }
         }
         Ok(self.node.build()?)

core/bin/zksync_tee_prover/Cargo.toml

@@ -19,6 +19,7 @@ thiserror.workspace = true
 tokio = { workspace = true, features = ["full"] }
 tracing.workspace = true
 url.workspace = true
+vise.workspace = true
 zksync_basic_types.workspace = true
 zksync_config.workspace = true
 zksync_env_config.workspace = true

core/bin/zksync_tee_prover/src/api_client.rs

@@ -13,7 +13,7 @@ use zksync_prover_interface::{
 };
 use zksync_types::{tee_types::TeeType, L1BatchNumber};
 
-use crate::error::TeeProverError;
+use crate::{error::TeeProverError, metrics::METRICS};
 
 /// Implementation of the API client for the proof data handler, run by
 /// [`zksync_proof_data_handler::run_server`].
@@ -97,11 +97,13 @@ impl TeeApiClient {
             proof: root_hash.as_bytes().into(),
             tee_type,
         }));
+        let observer = METRICS.proof_submitting_time.start();
         self.post::<_, SubmitTeeProofResponse, _>(
             format!("/tee/submit_proofs/{batch_number}").as_str(),
             request,
         )
         .await?;
+        observer.observe();
         tracing::info!(
             "Proof submitted successfully for batch number {}",
             batch_number

core/bin/zksync_tee_prover/src/main.rs

@@ -1,15 +1,20 @@
 use anyhow::Context as _;
 use config::TeeProverConfig;
 use tee_prover::TeeProverLayer;
-use zksync_config::configs::ObservabilityConfig;
+use zksync_config::configs::{ObservabilityConfig, PrometheusConfig};
 use zksync_env_config::FromEnv;
 use zksync_node_framework::{
-    implementations::layers::sigint::SigintHandlerLayer, service::ZkStackServiceBuilder,
+    implementations::layers::{
+        prometheus_exporter::PrometheusExporterLayer, sigint::SigintHandlerLayer,
+    },
+    service::ZkStackServiceBuilder,
 };
+use zksync_vlog::prometheus::PrometheusExporterConfig;
 
 mod api_client;
 mod config;
 mod error;
+mod metrics;
 mod tee_prover;
 
 /// This application serves as a TEE verifier, a.k.a. a TEE prover.
@@ -41,8 +46,15 @@ fn main() -> anyhow::Result<()> {
     let tee_prover_config = TeeProverConfig::from_env()?;
     let attestation_quote_bytes = std::fs::read(tee_prover_config.attestation_quote_file_path)?;
 
+    let prometheus_config = PrometheusConfig::from_env()?;
+    let exporter_config = PrometheusExporterConfig::push(
+        prometheus_config.gateway_endpoint(),
+        prometheus_config.push_interval(),
+    );
+
     ZkStackServiceBuilder::new()
         .add_layer(SigintHandlerLayer)
+        .add_layer(PrometheusExporterLayer(exporter_config))
         .add_layer(TeeProverLayer::new(
             tee_prover_config.api_url,
             tee_prover_config.signing_key,

core/bin/zksync_tee_prover/src/metrics.rs

@@ -0,0 +1,21 @@
+//! Metrics for the TEE Prover.
+
+use std::time::Duration;
+
+use vise::{Buckets, Gauge, Histogram, Metrics, Unit};
+
+#[derive(Debug, Metrics)]
+#[metrics(prefix = "tee_prover")]
+pub(crate) struct TeeProverMetrics {
+    #[metrics(buckets = Buckets::LATENCIES, unit = Unit::Seconds)]
+    pub job_waiting_time: Histogram<Duration>,
+    #[metrics(buckets = Buckets::LATENCIES, unit = Unit::Seconds)]
+    pub proof_generation_time: Histogram<Duration>,
+    #[metrics(buckets = Buckets::LATENCIES, unit = Unit::Seconds)]
+    pub proof_submitting_time: Histogram<Duration>,
+    pub network_errors_counter: Gauge<u64>,
+    pub last_batch_number_processed: Gauge<u64>,
+}
+
+#[vise::register]
+pub(super) static METRICS: vise::Global<TeeProverMetrics> = vise::Global::new();

core/bin/zksync_tee_prover/src/tee_prover.rs

@@ -12,7 +12,7 @@ use zksync_prover_interface::inputs::TeeVerifierInput;
 use zksync_tee_verifier::Verify;
 use zksync_types::{tee_types::TeeType, L1BatchNumber};
 
-use crate::{api_client::TeeApiClient, error::TeeProverError};
+use crate::{api_client::TeeApiClient, error::TeeProverError, metrics::METRICS};
 
 /// Wiring layer for `TeeProver`
 ///
@@ -83,12 +83,14 @@ impl TeeProver {
     ) -> Result<(Signature, L1BatchNumber, H256), TeeProverError> {
         match tvi {
             TeeVerifierInput::V1(tvi) => {
+                let observer = METRICS.proof_generation_time.start();
                 let verification_result = tvi.verify().map_err(TeeProverError::Verification)?;
                 let root_hash_bytes = verification_result.value_hash.as_bytes();
                 let batch_number = verification_result.batch_number;
                 let msg_to_sign = Message::from_slice(root_hash_bytes)
                     .map_err(|e| TeeProverError::Verification(e.into()))?;
                 let signature = self.signing_key.sign_ecdsa(msg_to_sign);
+                observer.observe();
                 Ok((signature, batch_number, verification_result.value_hash))
             }
             _ => Err(TeeProverError::Verification(anyhow::anyhow!(
@@ -97,7 +99,7 @@ impl TeeProver {
         }
     }
 
-    async fn step(&self) -> Result<(), TeeProverError> {
+    async fn step(&self) -> Result<Option<L1BatchNumber>, TeeProverError> {
         match self.api_client.get_job().await? {
             Some(job) => {
                 let (signature, batch_number, root_hash) = self.verify(*job)?;
@@ -110,10 +112,13 @@ impl TeeProver {
                         self.tee_type,
                     )
                     .await?;
+                Ok(Some(batch_number))
+            }
+            None => {
+                tracing::trace!("There are currently no pending batches to be proven");
+                Ok(None)
             }
-            None => tracing::trace!("There are currently no pending batches to be proven"),
         }
-        Ok(())
     }
 }
 
@@ -156,6 +161,7 @@ impl Task for TeeProver {
 
         let mut retries = 1;
         let mut backoff = self.config.initial_retry_backoff;
+        let mut observer = METRICS.job_waiting_time.start();
 
         loop {
             if *stop_receiver.0.borrow() {
@@ -164,11 +170,19 @@ impl Task for TeeProver {
             }
             let result = self.step().await;
             match result {
-                Ok(()) => {
+                Ok(batch_number) => {
                     retries = 1;
                     backoff = self.config.initial_retry_backoff;
+                    if let Some(batch_number) = batch_number {
+                        observer.observe();
+                        observer = METRICS.job_waiting_time.start();
+                        METRICS
+                            .last_batch_number_processed
+                            .set(batch_number.0 as u64);
+                    }
                 }
                 Err(err) => {
+                    METRICS.network_errors_counter.inc_by(1);
                     if !err.is_transient() || retries > self.config.max_retries {
                         return Err(err.into());
                     }

core/lib/basic_types/src/prover_dal.rs

@@ -267,6 +267,7 @@ pub struct ProverJobFriInfo {
 pub struct BasicWitnessGeneratorJobInfo {
     pub l1_batch_number: L1BatchNumber,
     pub merkle_tree_paths_blob_url: Option<String>,
+    pub witness_inputs_blob_url: Option<String>,
     pub attempts: u32,
     pub status: WitnessJobStatus,
     pub error: Option<String>,

core/lib/config/src/configs/general.rs

@@ -7,7 +7,7 @@ use crate::{
         house_keeper::HouseKeeperConfig,
         pruning::PruningConfig,
         snapshot_recovery::SnapshotRecoveryConfig,
-        vm_runner::ProtectiveReadsWriterConfig,
+        vm_runner::{BasicWitnessInputProducerConfig, ProtectiveReadsWriterConfig},
         CommitmentGeneratorConfig, FriProofCompressorConfig, FriProverConfig,
         FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig,
         ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig,
@@ -40,6 +40,7 @@ pub struct GeneralConfig {
     pub observability: Option<ObservabilityConfig>,
     pub da_dispatcher_config: Option<DADispatcherConfig>,
     pub protective_reads_writer_config: Option<ProtectiveReadsWriterConfig>,
+    pub basic_witness_input_producer_config: Option<BasicWitnessInputProducerConfig>,
     pub commitment_generator: Option<CommitmentGeneratorConfig>,
     pub snapshot_recovery: Option<SnapshotRecoveryConfig>,
     pub pruning: Option<PruningConfig>,

core/lib/config/src/configs/mod.rs

@@ -25,7 +25,7 @@ pub use self::{
     snapshot_recovery::SnapshotRecoveryConfig,
     snapshots_creator::SnapshotsCreatorConfig,
     utils::PrometheusConfig,
-    vm_runner::ProtectiveReadsWriterConfig,
+    vm_runner::{BasicWitnessInputProducerConfig, ProtectiveReadsWriterConfig},
 };
 
 pub mod api;

core/lib/config/src/configs/vm_runner.rs

@@ -17,3 +17,20 @@ impl ProtectiveReadsWriterConfig {
         "./db/protective_reads_writer".to_owned()
     }
 }
+
+#[derive(Debug, Deserialize, Clone, PartialEq, Default)]
+pub struct BasicWitnessInputProducerConfig {
+    /// Path to the RocksDB data directory that serves state cache.
+    #[serde(default = "BasicWitnessInputProducerConfig::default_db_path")]
+    pub db_path: String,
+    /// How many max batches should be processed at the same time.
+    pub window_size: u32,
+    /// All batches before this one (inclusive) are always considered to be processed.
+    pub first_processed_batch: L1BatchNumber,
+}
+
+impl BasicWitnessInputProducerConfig {
+    fn default_db_path() -> String {
+        "./db/basic_witness_input_producer".to_owned()
+    }
+}