chore(vm): use internal types for log query (matter-labs#860)

## What ❔

Reducing the number of `glue_into` in the multivm crate. It should
improve performance and general refactoring.

**Notes**

- The number of commits is large, since this PR used to be based on the
1.4.1 integration branch & it was squashed when merged to main.
- There are some optimizations that can be done (for instance, ensure
that zkevm_test_harness @1.4.0 and @1.4.1 accept iterator instead of
slice: matter-labs/era-zkevm_test_harness#68

## Why ❔

<!-- Why are these changes done? What goal do they contribute to? What
are the principles behind them? -->
<!-- Example: PR templates ensure PR reviewers, observers, and future
iterators are in context about the evolution of repos. -->

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.
- [ ] Spellcheck has been run via `zk spellcheck`.

---------

Co-authored-by: AntonD3 <[email protected]>
Co-authored-by: AntonD3 <[email protected]>

core/bin/system-constants-generator/src/utils.rs

@@ -10,7 +10,7 @@ use multivm::{
         BootloaderState, HistoryEnabled, HistoryMode, SimpleMemory, ToTracerPointer, Vm, VmTracer,
         ZkSyncVmState,
     },
-    zk_evm_1_4_1::aux_structures::Timestamp,
+    zk_evm_latest::aux_structures::Timestamp,
 };
 use once_cell::sync::Lazy;
 use zksync_contracts::{

core/lib/commitment_utils/Cargo.toml

@@ -12,5 +12,6 @@ categories = ["cryptography"]
 [dependencies]
 zksync_types = { path = "../../lib/types" }
 zksync_utils = { path = "../../lib/utils" }
-zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.0" }
+zkevm_test_harness_1_4_0 = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.0", package = "zkevm_test_harness" }
+zkevm_test_harness_1_4_1 = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.1", package = "zkevm_test_harness" }
 multivm = { path = "../../lib/multivm" }

core/lib/commitment_utils/src/lib.rs

@@ -1,16 +1,25 @@
 //! Utils for commitment calculation.
 use multivm::utils::get_used_bootloader_memory_bytes;
-use zkevm_test_harness::witness::utils::{
-    events_queue_commitment_fixed, initial_heap_content_commitment_fixed,
-};
-use zksync_types::{LogQuery, ProtocolVersionId, H256, U256};
+use zksync_types::{zk_evm_types::LogQuery, ProtocolVersionId, VmVersion, H256, U256};
 use zksync_utils::expand_memory_contents;
 
 pub fn events_queue_commitment(
-    events_queue: &Vec<LogQuery>,
+    events_queue: &[LogQuery],
     protocol_version: ProtocolVersionId,
 ) -> Option<H256> {
-    (!protocol_version.is_pre_boojum()).then(|| H256(events_queue_commitment_fixed(events_queue)))
+    match VmVersion::from(protocol_version) {
+        VmVersion::VmBoojumIntegration => Some(H256(
+            zkevm_test_harness_1_4_0::witness::utils::events_queue_commitment_fixed(
+                &events_queue.iter().map(|x| (*x).into()).collect(),
+            ),
+        )),
+        VmVersion::Vm1_4_1 => Some(H256(
+            zkevm_test_harness_1_4_1::witness::utils::events_queue_commitment_fixed(
+                &events_queue.iter().map(|x| (*x).into()).collect(),
+            ),
+        )),
+        _ => None,
+    }
 }
 
 pub fn bootloader_initial_content_commitment(
@@ -25,9 +34,18 @@ pub fn bootloader_initial_content_commitment(
 
     let full_bootloader_memory =
         expand_memory_contents(initial_bootloader_contents, expanded_memory_size);
-    let commitment = H256(initial_heap_content_commitment_fixed(
-        &full_bootloader_memory,
-    ));
 
-    Some(commitment)
+    match VmVersion::from(protocol_version) {
+        VmVersion::VmBoojumIntegration => Some(H256(
+            zkevm_test_harness_1_4_0::witness::utils::initial_heap_content_commitment_fixed(
+                &full_bootloader_memory,
+            ),
+        )),
+        VmVersion::Vm1_4_1 => Some(H256(
+            zkevm_test_harness_1_4_1::witness::utils::initial_heap_content_commitment_fixed(
+                &full_bootloader_memory,
+            ),
+        )),
+        _ => unreachable!(),
+    }
 }

core/lib/dal/src/blocks_dal.rs

@@ -11,7 +11,8 @@ use zksync_types::{
     aggregated_operations::AggregatedActionType,
     block::{BlockGasCount, L1BatchHeader, MiniblockHeader},
     commitment::{L1BatchMetadata, L1BatchWithMetadata},
-    Address, L1BatchNumber, LogQuery, MiniblockNumber, ProtocolVersionId, H256, U256,
+    zk_evm_types::LogQuery,
+    Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, H256, U256,
 };
 
 use crate::{

core/lib/dal/src/storage_logs_dedup_dal.rs

@@ -1,7 +1,9 @@
 use std::collections::HashSet;
 
 use sqlx::types::chrono::Utc;
-use zksync_types::{AccountTreeId, Address, L1BatchNumber, LogQuery, StorageKey, H256};
+use zksync_types::{
+    zk_evm_types::LogQuery, AccountTreeId, Address, L1BatchNumber, StorageKey, H256,
+};
 use zksync_utils::u256_to_h256;
 
 use crate::StorageProcessor;

core/lib/multivm/src/glue/types/mod.rs

@@ -7,4 +7,6 @@
 
 mod vm;
 mod zk_evm_1_3_1;
+mod zk_evm_1_3_3;
+mod zk_evm_1_4_0;
 mod zk_evm_1_4_1;

core/lib/multivm/src/glue/types/vm/mod.rs

@@ -1,4 +1,5 @@
 mod block_context_mode;
+mod storage_query;
 mod tx_execution_mode;
 mod tx_revert_reason;
 mod vm_block_result;

core/lib/multivm/src/glue/types/vm/storage_query.rs

@@ -0,0 +1,66 @@
+use zksync_types::StorageLogQuery;
+
+use crate::glue::{GlueFrom, GlueInto};
+
+impl GlueFrom<crate::vm_m5::utils::StorageLogQuery> for StorageLogQuery {
+    fn glue_from(value: crate::vm_m5::utils::StorageLogQuery) -> Self {
+        Self {
+            log_query: value.log_query.glue_into(),
+            log_type: value.log_type,
+        }
+    }
+}
+
+impl GlueFrom<crate::vm_m6::utils::StorageLogQuery> for StorageLogQuery {
+    fn glue_from(value: crate::vm_m6::utils::StorageLogQuery) -> Self {
+        Self {
+            log_query: value.log_query.glue_into(),
+            log_type: value.log_type,
+        }
+    }
+}
+
+impl GlueFrom<crate::vm_1_3_2::utils::StorageLogQuery> for StorageLogQuery {
+    fn glue_from(value: crate::vm_1_3_2::utils::StorageLogQuery) -> Self {
+        Self {
+            log_query: value.log_query.glue_into(),
+            log_type: value.log_type,
+        }
+    }
+}
+
+impl GlueFrom<crate::vm_virtual_blocks::utils::logs::StorageLogQuery> for StorageLogQuery {
+    fn glue_from(value: crate::vm_virtual_blocks::utils::logs::StorageLogQuery) -> Self {
+        Self {
+            log_query: value.log_query.glue_into(),
+            log_type: value.log_type,
+        }
+    }
+}
+
+impl GlueFrom<crate::vm_refunds_enhancement::utils::logs::StorageLogQuery> for StorageLogQuery {
+    fn glue_from(value: crate::vm_refunds_enhancement::utils::logs::StorageLogQuery) -> Self {
+        Self {
+            log_query: value.log_query.glue_into(),
+            log_type: value.log_type,
+        }
+    }
+}
+
+impl GlueFrom<crate::vm_boojum_integration::utils::logs::StorageLogQuery> for StorageLogQuery {
+    fn glue_from(value: crate::vm_boojum_integration::utils::logs::StorageLogQuery) -> Self {
+        Self {
+            log_query: value.log_query.glue_into(),
+            log_type: value.log_type,
+        }
+    }
+}
+
+impl GlueFrom<crate::vm_latest::utils::logs::StorageLogQuery> for StorageLogQuery {
+    fn glue_from(value: crate::vm_latest::utils::logs::StorageLogQuery) -> Self {
+        Self {
+            log_query: value.log_query.glue_into(),
+            log_type: value.log_type,
+        }
+    }
+}

core/lib/multivm/src/glue/types/vm/vm_block_result.rs

@@ -1,3 +1,6 @@
+use itertools::Itertools;
+use zk_evm_1_3_1::aux_structures::LogQuery as LogQuery_1_3_1;
+use zkevm_test_harness_1_3_3::witness::sort_storage_access::sort_storage_access_queries as sort_storage_access_queries_1_3_3;
 use zksync_types::l2_to_l1_log::UserL2ToL1Log;
 
 use crate::{
@@ -15,6 +18,21 @@ use crate::{
 
 impl GlueFrom<crate::vm_m5::vm_instance::VmBlockResult> for crate::interface::FinishedL1Batch {
     fn glue_from(value: crate::vm_m5::vm_instance::VmBlockResult) -> Self {
+        let storage_log_queries = value.full_result.storage_log_queries.clone();
+        let deduplicated_storage_log_queries: Vec<LogQuery_1_3_1> =
+            sort_storage_access_queries_1_3_3(
+                &storage_log_queries
+                    .iter()
+                    .map(|log| {
+                        GlueInto::<zk_evm_1_3_3::aux_structures::LogQuery>::glue_into(log.log_query)
+                    })
+                    .collect_vec(),
+            )
+            .1
+            .into_iter()
+            .map(GlueInto::<LogQuery_1_3_1>::glue_into)
+            .collect();
+
         crate::interface::FinishedL1Batch {
             block_tip_execution_result: VmExecutionResultAndLogs {
                 result: value.block_tip_result.revert_reason.glue_into(),
@@ -32,7 +50,14 @@ impl GlueFrom<crate::vm_m5::vm_instance::VmBlockResult> for crate::interface::Fi
             },
             final_execution_state: CurrentExecutionState {
                 events: value.full_result.events,
-                storage_log_queries: value.full_result.storage_log_queries,
+                storage_log_queries: storage_log_queries
+                    .into_iter()
+                    .map(GlueInto::glue_into)
+                    .collect(),
+                deduplicated_storage_log_queries: deduplicated_storage_log_queries
+                    .into_iter()
+                    .map(GlueInto::glue_into)
+                    .collect(),
                 used_contract_hashes: value.full_result.used_contract_hashes,
                 user_l2_to_l1_logs: value
                     .full_result
@@ -54,6 +79,21 @@ impl GlueFrom<crate::vm_m5::vm_instance::VmBlockResult> for crate::interface::Fi
 
 impl GlueFrom<crate::vm_m6::vm_instance::VmBlockResult> for crate::interface::FinishedL1Batch {
     fn glue_from(value: crate::vm_m6::vm_instance::VmBlockResult) -> Self {
+        let storage_log_queries = value.full_result.storage_log_queries.clone();
+        let deduplicated_storage_log_queries: Vec<LogQuery_1_3_1> =
+            sort_storage_access_queries_1_3_3(
+                &storage_log_queries
+                    .iter()
+                    .map(|log| {
+                        GlueInto::<zk_evm_1_3_3::aux_structures::LogQuery>::glue_into(log.log_query)
+                    })
+                    .collect_vec(),
+            )
+            .1
+            .into_iter()
+            .map(GlueInto::<LogQuery_1_3_1>::glue_into)
+            .collect();
+
         crate::interface::FinishedL1Batch {
             block_tip_execution_result: VmExecutionResultAndLogs {
                 result: value.block_tip_result.revert_reason.glue_into(),
@@ -71,7 +111,14 @@ impl GlueFrom<crate::vm_m6::vm_instance::VmBlockResult> for crate::interface::Fi
             },
             final_execution_state: CurrentExecutionState {
                 events: value.full_result.events,
-                storage_log_queries: value.full_result.storage_log_queries,
+                storage_log_queries: storage_log_queries
+                    .into_iter()
+                    .map(GlueInto::glue_into)
+                    .collect(),
+                deduplicated_storage_log_queries: deduplicated_storage_log_queries
+                    .into_iter()
+                    .map(GlueInto::glue_into)
+                    .collect(),
                 used_contract_hashes: value.full_result.used_contract_hashes,
                 user_l2_to_l1_logs: value
                     .full_result
@@ -93,6 +140,13 @@ impl GlueFrom<crate::vm_m6::vm_instance::VmBlockResult> for crate::interface::Fi
 
 impl GlueFrom<crate::vm_1_3_2::vm_instance::VmBlockResult> for crate::interface::FinishedL1Batch {
     fn glue_from(value: crate::vm_1_3_2::vm_instance::VmBlockResult) -> Self {
+        let storage_log_queries = value.full_result.storage_log_queries.clone();
+        let deduplicated_storage_log_queries =
+            zkevm_test_harness_1_3_3::witness::sort_storage_access::sort_storage_access_queries(
+                storage_log_queries.iter().map(|log| &log.log_query),
+            )
+            .1;
+
         crate::interface::FinishedL1Batch {
             block_tip_execution_result: VmExecutionResultAndLogs {
                 result: value.block_tip_result.revert_reason.glue_into(),
@@ -116,7 +170,14 @@ impl GlueFrom<crate::vm_1_3_2::vm_instance::VmBlockResult> for crate::interface:
             },
             final_execution_state: CurrentExecutionState {
                 events: value.full_result.events,
-                storage_log_queries: value.full_result.storage_log_queries,
+                storage_log_queries: storage_log_queries
+                    .into_iter()
+                    .map(GlueInto::glue_into)
+                    .collect(),
+                deduplicated_storage_log_queries: deduplicated_storage_log_queries
+                    .into_iter()
+                    .map(GlueInto::glue_into)
+                    .collect(),
                 used_contract_hashes: value.full_result.used_contract_hashes,
                 user_l2_to_l1_logs: value
                     .full_result
@@ -161,7 +222,12 @@ impl GlueFrom<crate::vm_1_3_2::vm_instance::VmBlockResult>
                     .map(UserL2ToL1Log)
                     .collect(),
                 system_l2_to_l1_logs: vec![],
-                storage_logs: value.full_result.storage_log_queries,
+                storage_logs: value
+                    .full_result
+                    .storage_log_queries
+                    .into_iter()
+                    .map(GlueInto::glue_into)
+                    .collect(),
                 total_log_queries_count: value.full_result.total_log_queries,
             },
             statistics: VmExecutionStatistics {
@@ -234,7 +300,12 @@ impl GlueFrom<crate::vm_m6::vm_instance::VmBlockResult>
                     .map(UserL2ToL1Log)
                     .collect(),
                 system_l2_to_l1_logs: vec![],
-                storage_logs: value.full_result.storage_log_queries,
+                storage_logs: value
+                    .full_result
+                    .storage_log_queries
+                    .into_iter()
+                    .map(GlueInto::glue_into)
+                    .collect(),
                 total_log_queries_count: value.full_result.total_log_queries,
             },
             statistics: VmExecutionStatistics {