feat(en): Unify snapshot recovery and recovery from L1

.github/workflows/ci-core-reusable.yml

@@ -230,9 +230,10 @@ jobs:
           fi
           ENABLE_CONSENSUS=${{ matrix.consensus }} \
           DEPLOYMENT_MODE=${{ matrix.deployment_mode }} \
+          SNAPSHOTS_CREATOR_VERSION=${{ matrix.deployment_mode == 'Validium' && '0' || '1' }} \
           DISABLE_TREE_DURING_PRUNING=${{ matrix.base_token == 'Eth' }} \
           ETH_CLIENT_WEB3_URL="http://reth:8545" \
-          PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE,DISABLE_TREE_DURING_PRUNING,ETH_CLIENT_WEB3_URL" \
+          PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE,DISABLE_TREE_DURING_PRUNING,SNAPSHOTS_CREATOR_VERSION,ETH_CLIENT_WEB3_URL" \
             ci_run yarn recovery-test snapshot-recovery-test
 
       - name: Genesis recovery test

core/bin/external_node/src/config/mod.rs

@@ -748,6 +748,10 @@ pub(crate) struct ExperimentalENConfig {
     // Snapshot recovery
     /// L1 batch number of the snapshot to use during recovery. Specifying this parameter is mostly useful for testing.
     pub snapshots_recovery_l1_batch: Option<L1BatchNumber>,
+    /// Enables dropping storage key preimages when recovering storage logs from a snapshot with version 0.
+    /// This is a temporary flag that will eventually be removed together with version 0 snapshot support.
+    #[serde(default)]
+    pub snapshots_recovery_drop_storage_key_preimages: bool,
     /// Approximate chunk size (measured in the number of entries) to recover in a single iteration.
     /// Reasonable values are order of 100,000 (meaning an iteration takes several seconds).
     ///
@@ -784,6 +788,7 @@ impl ExperimentalENConfig {
                 Self::default_state_keeper_db_block_cache_capacity_mb(),
             state_keeper_db_max_open_files: None,
             snapshots_recovery_l1_batch: None,
+            snapshots_recovery_drop_storage_key_preimages: false,
             snapshots_recovery_tree_chunk_size: Self::default_snapshots_recovery_tree_chunk_size(),
             snapshots_recovery_tree_parallel_persistence_buffer: None,
             commitment_generator_max_parallelism: None,

core/bin/external_node/src/init.rs

@@ -18,6 +18,7 @@ use crate::config::snapshot_recovery_object_store_config;
 pub(crate) struct SnapshotRecoveryConfig {
     /// If not specified, the latest snapshot will be used.
     pub snapshot_l1_batch_override: Option<L1BatchNumber>,
+    pub drop_storage_key_preimages: bool,
 }
 
 #[derive(Debug)]
@@ -109,6 +110,10 @@ pub(crate) async fn ensure_storage_initialized(
                 );
                 snapshots_applier_task.set_snapshot_l1_batch(snapshot_l1_batch);
             }
+            if recovery_config.drop_storage_key_preimages {
+                tracing::info!("Dropping storage key preimages for snapshot storage logs");
+                snapshots_applier_task.drop_storage_key_preimages();
+            }
             app_health.insert_component(snapshots_applier_task.health_check())?;
 
             let recovery_started_at = Instant::now();

core/bin/external_node/src/main.rs

@@ -938,6 +938,9 @@ async fn run_node(
             .snapshots_recovery_enabled
             .then_some(SnapshotRecoveryConfig {
                 snapshot_l1_batch_override: config.experimental.snapshots_recovery_l1_batch,
+                drop_storage_key_preimages: config
+                    .experimental
+                    .snapshots_recovery_drop_storage_key_preimages,
             });
     ensure_storage_initialized(
         connection_pool.clone(),

core/bin/snapshots_creator/README.md

@@ -51,9 +51,9 @@ Creating a snapshot is a part of the [snapshot recovery integration test]. You c
 
 Each snapshot consists of three types of data (see [`snapshots.rs`] for exact definitions):
 
-- **Header:** Includes basic information, such as the miniblock / L1 batch of the snapshot, miniblock / L1 batch
-  timestamps, miniblock hash and L1 batch root hash. Returned by the methods in the `snapshots` namespace of the
-  JSON-RPC API of the main node.
+- **Header:** Includes basic information, such as the L2 block / L1 batch of the snapshot, L2 block / L1 batch
+  timestamps, L2 block hash and L1 batch root hash. Returned by the methods in the `snapshots` namespace of the JSON-RPC
+  API of the main node.
 - **Storage log chunks:** Latest values for all VM storage slots ever written to at the time the snapshot is made.
   Besides key–value pairs, each storage log record also contains the L1 batch number of its initial write and its
   enumeration index; both are used to restore the contents of the `initial_writes` table. Chunking storage logs is
@@ -64,6 +64,16 @@ Each snapshot consists of three types of data (see [`snapshots.rs`] for exact de
 - **Factory dependencies:** All bytecodes deployed on L2 at the time the snapshot is made. Stored as a single gzipped
   Protobuf message in an object store.
 
+### Versioning
+
+There are currently 2 versions of the snapshot format which differ in how keys are mentioned in storage logs.
+
+- Version 0 includes key preimages (EVM-compatible keys), i.e. address / contract slot tuples.
+- Version 1 includes only hashed keys as used in Era ZKP circuits and in the Merkle tree. Besides reducing the snapshot
+  size (with the change, keys occupy 32 bytes instead of 52), this allows to unify snapshot recovery with recovery from
+  L1 data. Having only hashed keys for snapshot storage logs is safe; key preimages are only required for a couple of
+  components to sort keys in a batch, but these cases only require preimages for L1 batches locally executed on a node.
+
 [`snapshots.rs`]: ../../lib/types/src/snapshots.rs
 [object store]: ../../lib/object_store
 [snapshot recovery integration test]: ../../tests/recovery-test/tests/snapshot-recovery.test.ts