feat(vm): Fast VM integration (#1949)

## What ❔ Integrates the new VM into server codebase: - Allows to run the new VM in the state keeper / batch executor (but not the API server sandbox). The new VM is disabled by default everywhere and requires an explicit opt-in. - Introduces the new component, VM playground, to test the new VM. - Runs the load test with the new VM in CI. ## Why ❔ New VM is several times more efficient as per preliminary benchmarks and is easier to maintain. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. --------- Co-authored-by: Fedor Sakharov <[email protected]> Co-authored-by: Alex Ostrovski <[email protected]> Co-authored-by: Alex Ostrovski <[email protected]>
matter-labs · Aug 9, 2024 · b752a54 · b752a54
1 parent b8609eb
commit b752a54
Show file tree

Hide file tree

Showing 150 changed files with 9,112 additions and 1,016 deletions.
diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml
@@ -67,6 +67,10 @@ jobs:
 
   loadtest:
     runs-on: [matterlabs-ci-runner]
+    strategy:
+      fail-fast: false
+      matrix:
+        vm_mode: ["old", "new"]
 
     steps:
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
@@ -82,7 +86,8 @@ jobs:
 
       - name: Loadtest configuration
         run: |
-          echo EXPECTED_TX_COUNT="16000" >> .env
+          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'new' && 24000 || 18000 }} >> .env
+          echo ACCOUNTS_AMOUNT="150" >> .env
           echo FAIL_FAST=true >> .env
           echo IN_DOCKER=1 >> .env
           echo DATABASE_MERKLE_TREE_MODE=lightweight >> .env
@@ -105,7 +110,9 @@ jobs:
       # `sleep 60` because we need to wait until server added all the tokens
       - name: Run server
         run: |
-          ci_run zk server --uring --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
+          EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE=${{ matrix.vm_mode }} \
+          PASSED_ENV_VARS="EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE" \
+            ci_run zk server --uring --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
           ci_run sleep 60
 
       - name: Deploy legacy era contracts
@@ -135,7 +142,7 @@ jobs:
         base_token: ["Eth", "Custom"]
         deployment_mode: ["Rollup", "Validium"]
     env:
-      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,da_dispatcher${{ matrix.consensus && ',consensus' || '' }}${{ matrix.base_token == 'Custom' && ',base_token_ratio_persister' || '' }}"
+      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher${{ matrix.consensus && ',consensus' || '' }}${{ matrix.base_token == 'Custom' && ',base_token_ratio_persister' || '' }}"
 
     runs-on: [matterlabs-ci-runner]
     steps:

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -215,6 +215,9 @@ zk_evm_1_4_0 = { package = "zk_evm", version = "0.140.0" }
 zk_evm_1_4_1 = { package = "zk_evm", version = "0.141.0" }
 zk_evm_1_5_0 = { package = "zk_evm", version = "=0.150.0" }
 
+# New VM; pinned to a specific commit because of instability
+vm2 = { git = "https://github.com/matter-labs/vm2.git", rev = "9a38900d7af9b1d72b47ce3be980e77c1239a61d" }
+
 # Consensus dependencies.
 zksync_concurrency = "=0.1.0-rc.9"
 zksync_consensus_bft = "=0.1.0-rc.9"

diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs
@@ -94,8 +94,8 @@ async fn build_state_keeper(
         stop_receiver_clone.changed().await?;
         result
     }));
-    let batch_executor_base: Box<dyn BatchExecutor> =
-        Box::new(MainBatchExecutor::new(save_call_traces, true));
+    let batch_executor = MainBatchExecutor::new(save_call_traces, true);
+    let batch_executor: Box<dyn BatchExecutor> = Box::new(batch_executor);
 
     let io = ExternalIO::new(
         connection_pool,
@@ -108,7 +108,7 @@ async fn build_state_keeper(
     Ok(ZkSyncStateKeeper::new(
         stop_receiver,
         Box::new(io),
-        batch_executor_base,
+        batch_executor,
         output_handler,
         Arc::new(NoopSealer),
         Arc::new(storage_factory),

diff --git a/core/bin/system-constants-generator/src/utils.rs b/core/bin/system-constants-generator/src/utils.rs
@@ -8,7 +8,7 @@ use zksync_contracts::{
 use zksync_multivm::{
     interface::{
         dyn_tracers::vm_1_5_0::DynTracer, tracer::VmExecutionStopReason, L1BatchEnv, L2BlockEnv,
-        SystemEnv, TxExecutionMode, VmExecutionMode, VmInterface,
+        SystemEnv, TxExecutionMode, VmExecutionMode, VmFactory, VmInterface,
     },
     vm_latest::{
         constants::{BATCH_COMPUTATIONAL_GAS_LIMIT, BOOTLOADER_HEAP_PAGE},
@@ -260,8 +260,7 @@ pub(super) fn execute_internal_transfer_test() -> u32 {
         output: tracer_result.clone(),
     }
     .into_tracer_pointer();
-    let mut vm: Vm<_, HistoryEnabled> =
-        Vm::new(l1_batch, system_env, Rc::new(RefCell::new(storage_view)));
+    let mut vm: Vm<_, HistoryEnabled> = Vm::new(l1_batch, system_env, storage_view.to_rc_ptr());
     let result = vm.inspect(tracer.into(), VmExecutionMode::Bootloader);
 
     assert!(!result.result.is_failed(), "The internal call has reverted");

diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs
@@ -11,7 +11,7 @@ use zksync_config::{
         },
         fri_prover_group::FriProverGroupConfig,
         house_keeper::HouseKeeperConfig,
-        BasicWitnessInputProducerConfig, ContractsConfig, DatabaseSecrets,
+        BasicWitnessInputProducerConfig, ContractsConfig, DatabaseSecrets, ExperimentalVmConfig,
         ExternalPriceApiClientConfig, FriProofCompressorConfig, FriProverConfig,
         FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig,
         L1Secrets, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig,
@@ -209,5 +209,6 @@ fn load_env_config() -> anyhow::Result<TempConfigStore> {
         snapshot_recovery: None,
         external_price_api_client_config: ExternalPriceApiClientConfig::from_env().ok(),
         external_proof_integration_api_config: ExternalProofIntegrationApiConfig::from_env().ok(),
+        experimental_vm_config: ExperimentalVmConfig::from_env().ok(),
     })
 }
diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs
@@ -55,7 +55,8 @@ use zksync_node_framework::{
         },
         tee_verifier_input_producer::TeeVerifierInputProducerLayer,
         vm_runner::{
-            bwip::BasicWitnessInputProducerLayer, protective_reads::ProtectiveReadsWriterLayer,
+            bwip::BasicWitnessInputProducerLayer, playground::VmPlaygroundLayer,
+            protective_reads::ProtectiveReadsWriterLayer,
         },
         web3_api::{
             caches::MempoolCacheLayer,
@@ -248,8 +249,10 @@ impl MainNodeBuilder {
             try_load_config!(wallets.state_keeper),
         );
         let db_config = try_load_config!(self.configs.db_config);
+        let experimental_vm_config = try_load_config!(self.configs.experimental_vm_config);
         let main_node_batch_executor_builder_layer =
-            MainBatchExecutorLayer::new(sk_config.save_call_traces, OPTIONAL_BYTECODE_COMPRESSION);
+            MainBatchExecutorLayer::new(sk_config.save_call_traces, OPTIONAL_BYTECODE_COMPRESSION)
+                .with_fast_vm_mode(experimental_vm_config.state_keeper_fast_vm_mode);
 
         let rocksdb_options = RocksdbStorageOptions {
             block_cache_capacity: db_config
@@ -573,6 +576,16 @@ impl MainNodeBuilder {
         Ok(self)
     }
 
+    fn add_vm_playground_layer(mut self) -> anyhow::Result<Self> {
+        let vm_config = try_load_config!(self.configs.experimental_vm_config);
+        self.node.add_layer(VmPlaygroundLayer::new(
+            vm_config.playground,
+            self.genesis_config.l2_chain_id,
+        ));
+
+        Ok(self)
+    }
+
     fn add_base_token_ratio_persister_layer(mut self) -> anyhow::Result<Self> {
         let config = try_load_config!(self.configs.base_token_adjuster);
         let contracts_config = self.contracts_config.clone();
@@ -736,6 +749,9 @@ impl MainNodeBuilder {
                 Component::VmRunnerBwip => {
                     self = self.add_vm_runner_bwip_layer()?;
                 }
+                Component::VmPlayground => {
+                    self = self.add_vm_playground_layer()?;
+                }
                 Component::ExternalProofIntegrationApi => {
                     self = self.add_external_proof_integration_api_layer()?;
                 }

diff --git a/core/lib/basic_types/src/lib.rs b/core/lib/basic_types/src/lib.rs
@@ -29,7 +29,7 @@ pub mod prover_dal;
 pub mod settlement;
 pub mod tee_types;
 pub mod url;
-pub mod vm_version;
+pub mod vm;
 pub mod web3;
 
 /// Account place in the global state tree is uniquely identified by its address.

diff --git a/core/lib/basic_types/src/protocol_version.rs b/core/lib/basic_types/src/protocol_version.rs
@@ -12,7 +12,7 @@ use serde_with::{DeserializeFromStr, SerializeDisplay};
 
 use crate::{
     ethabi::Token,
-    vm_version::VmVersion,
+    vm::VmVersion,
     web3::contract::{Detokenize, Error},
     H256, U256,
 };

diff --git a/core/lib/basic_types/src/vm.rs b/core/lib/basic_types/src/vm.rs
@@ -0,0 +1,39 @@
+//! Basic VM types that shared widely enough to not put them in the `multivm` crate.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy)]
+pub enum VmVersion {
+    M5WithoutRefunds,
+    M5WithRefunds,
+    M6Initial,
+    M6BugWithCompressionFixed,
+    Vm1_3_2,
+    VmVirtualBlocks,
+    VmVirtualBlocksRefundsEnhancement,
+    VmBoojumIntegration,
+    Vm1_4_1,
+    Vm1_4_2,
+    Vm1_5_0SmallBootloaderMemory,
+    Vm1_5_0IncreasedBootloaderMemory,
+}
+
+impl VmVersion {
+    /// Returns the latest supported VM version.
+    pub const fn latest() -> VmVersion {
+        Self::Vm1_5_0IncreasedBootloaderMemory
+    }
+}
+
+/// Mode in which to run the new fast VM implementation.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum FastVmMode {
+    /// Run only the old VM.
+    #[default]
+    Old,
+    /// Run only the new Vm.
+    New,
+    /// Run both the new and old VM and compare their outputs for each transaction execution.
+    Shadow,
+}
diff --git a/core/lib/basic_types/src/vm_version.rs b/core/lib/basic_types/src/vm_version.rs
diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs
@@ -3,6 +3,7 @@
 use std::num::NonZeroU32;
 
 use serde::Deserialize;
+use zksync_basic_types::{vm::FastVmMode, L1BatchNumber};
 
 #[derive(Debug, Clone, PartialEq, Deserialize)]
 pub struct ExperimentalDBConfig {
@@ -60,3 +61,50 @@ impl ExperimentalDBConfig {
         100
     }
 }
+
+/// Configuration for the VM playground (an experimental component that's unlikely to ever be stabilized).
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct ExperimentalVmPlaygroundConfig {
+    /// Mode in which to run the fast VM implementation. Note that for it to actually be used, L1 batches should have a recent version.
+    #[serde(default)]
+    pub fast_vm_mode: FastVmMode,
+    /// Path to the RocksDB cache directory.
+    #[serde(default = "ExperimentalVmPlaygroundConfig::default_db_path")]
+    pub db_path: String,
+    /// First L1 batch to consider processed. Will not be used if the processing cursor is persisted, unless the `reset` flag is set.
+    #[serde(default)]
+    pub first_processed_batch: L1BatchNumber,
+    /// If set to true, processing cursor will reset `first_processed_batch` regardless of the current progress. Beware that this will likely
+    /// require to drop the RocksDB cache.
+    #[serde(default)]
+    pub reset: bool,
+}
+
+impl Default for ExperimentalVmPlaygroundConfig {
+    fn default() -> Self {
+        Self {
+            fast_vm_mode: FastVmMode::default(),
+            db_path: Self::default_db_path(),
+            first_processed_batch: L1BatchNumber(0),
+            reset: false,
+        }
+    }
+}
+
+impl ExperimentalVmPlaygroundConfig {
+    pub fn default_db_path() -> String {
+        "./db/vm_playground".to_owned()
+    }
+}
+
+/// Experimental VM configuration options.
+#[derive(Debug, Clone, Default, PartialEq, Deserialize)]
+pub struct ExperimentalVmConfig {
+    #[serde(skip)] // Isn't properly deserialized by `envy`
+    pub playground: ExperimentalVmPlaygroundConfig,
+
+    /// Mode in which to run the fast VM implementation in the state keeper. Should not be set in production;
+    /// the new VM doesn't produce call traces and can diverge from the old VM!
+    #[serde(default)]
+    pub state_keeper_fast_vm_mode: FastVmMode,
+}
diff --git a/core/lib/config/src/configs/general.rs b/core/lib/config/src/configs/general.rs
@@ -9,10 +9,10 @@ use crate::{
         pruning::PruningConfig,
         snapshot_recovery::SnapshotRecoveryConfig,
         vm_runner::{BasicWitnessInputProducerConfig, ProtectiveReadsWriterConfig},
-        CommitmentGeneratorConfig, ExternalPriceApiClientConfig, FriProofCompressorConfig,
-        FriProverConfig, FriProverGatewayConfig, FriWitnessGeneratorConfig,
-        FriWitnessVectorGeneratorConfig, ObservabilityConfig, PrometheusConfig,
-        ProofDataHandlerConfig,
+        CommitmentGeneratorConfig, ExperimentalVmConfig, ExternalPriceApiClientConfig,
+        FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig,
+        FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, ObservabilityConfig,
+        PrometheusConfig, ProofDataHandlerConfig,
     },
     ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, ExternalProofIntegrationApiConfig,
     ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
@@ -51,4 +51,5 @@ pub struct GeneralConfig {
     pub external_price_api_client_config: Option<ExternalPriceApiClientConfig>,
     pub consensus_config: Option<ConsensusConfig>,
     pub external_proof_integration_api_config: Option<ExternalProofIntegrationApiConfig>,
+    pub experimental_vm_config: Option<ExperimentalVmConfig>,
 }
diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs
@@ -9,7 +9,7 @@ pub use self::{
     database::{DBConfig, PostgresConfig},
     eth_sender::{EthConfig, GasAdjusterConfig},
     eth_watch::EthWatchConfig,
-    experimental::ExperimentalDBConfig,
+    experimental::{ExperimentalDBConfig, ExperimentalVmConfig, ExperimentalVmPlaygroundConfig},
     external_price_api_client::ExternalPriceApiClientConfig,
     external_proof_integration_api::ExternalProofIntegrationApiConfig,
     fri_proof_compressor::FriProofCompressorConfig,