merge main

.dockerignore

@@ -46,3 +46,4 @@ contracts/.git
 !etc/env/dev.toml
 !etc/env/consensus_secrets.yaml
 !etc/env/consensus_config.yaml
+!rust-toolchain

.github/workflows/build-contract-verifier-template.yml

@@ -138,7 +138,7 @@ jobs:
           COMPONENT: ${{ matrix.components }}
           PLATFORM: ${{ matrix.platforms }}
         run: |
-          ci_run rustup default nightly-2023-08-21
+          ci_run rustup default nightly-2024-05-07
           platform=$(echo $PLATFORM | tr '/' '-')
           ci_run zk docker $DOCKER_ACTION --custom-tag=${IMAGE_TAG_SUFFIX} --platform=${PLATFORM} $COMPONENT
       - name: Show sccache stats

.github/workflows/build-core-template.yml

@@ -147,7 +147,7 @@ jobs:
           COMPONENT: ${{ matrix.components }}
           PLATFORM: ${{ matrix.platforms }}
         run: |
-          ci_run rustup default nightly-2023-08-21
+          ci_run rustup default nightly-2024-05-07
           platform=$(echo $PLATFORM | tr '/' '-')
           ci_run zk docker $DOCKER_ACTION --custom-tag=${IMAGE_TAG_SUFFIX} --platform=${PLATFORM} $COMPONENT
       - name: Show sccache stats

.github/workflows/zk-environment-publish.yml

@@ -1,6 +1,10 @@
 name: Publish zk-environment Docker images
 
 on:
+  # Workflow dispatch, to allow building and pushing new environments.
+  # It will NOT mark them as latest.
+  workflow_dispatch:
+
   push:
     branches:
       - main
@@ -46,7 +50,7 @@ jobs:
               - .github/workflows/zk-environment-publish.yml
 
   get_short_sha:
-    if: needs.changed_files.outputs.zk_environment == 'true'
+    if: ${{ (needs.changed_files.outputs.zk_environment == 'true') || (github.event_name == 'workflow_dispatch') }}
     needs: [changed_files]
     runs-on: ubuntu-latest
     outputs:
@@ -60,7 +64,8 @@ jobs:
         run: echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
 
   zk_environment:
-    if: needs.changed_files.outputs.zk_environment == 'true'
+    # Build and push new environment, if workflow dispatch is requested.
+    if: ${{ (needs.changed_files.outputs.zk_environment == 'true') || (github.event_name == 'workflow_dispatch') }}
     needs: [changed_files, get_short_sha]
     name: Build and optionally push zk-environment Docker images to Docker Hub
     strategy:
@@ -79,7 +84,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2
       - name: Log in to Docker Hub
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch') }}
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USER }}
@@ -91,17 +96,18 @@ jobs:
           target: rust-lightweight
           tags: "matterlabs/zk-environment:${{ needs.get_short_sha.outputs.short_sha }}-lightweight-${{ matrix.arch }}"
           build-args: ARCH=${{ matrix.arch }}
-          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+          push: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch') }}
       - name: Build and optionally push zk-environment lightweight Rust nightly
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           file: docker/zk-environment/Dockerfile
           target: rust-lightweight-nightly
           tags: "matterlabs/zk-environment:${{ needs.get_short_sha.outputs.short_sha }}-lightweight-nightly-${{ matrix.arch }}"
           build-args: ARCH=${{ matrix.arch }}
-          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+          push: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch') }}
 
   zk_environment_multiarch_manifest:
+    # We'll update the 'latest' tag, only on environments generated from 'main'.
     if: needs.changed_files.outputs.zk_environment == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
     needs: [changed_files, get_short_sha, zk_environment]
     runs-on: ubuntu-latest

core/bin/block_reverter/src/main.rs

@@ -229,7 +229,7 @@ async fn main() -> anyhow::Result<()> {
                     block_reverter.enable_rolling_back_snapshot_objects(
                         ObjectStoreFactory::new(object_store_config.0)
                             .create_store()
-                            .await,
+                            .await?,
                     );
                 }
             }

core/bin/external_node/src/config/mod.rs

@@ -25,8 +25,8 @@ use zksync_node_api_server::{
 use zksync_protobuf_config::proto;
 use zksync_snapshots_applier::SnapshotsApplierConfig;
 use zksync_types::{
-    api::BridgeAddresses, commitment::L1BatchCommitmentMode, url::SensitiveUrl, Address, L1ChainId,
-    L2ChainId, ETHEREUM_ADDRESS,
+    api::BridgeAddresses, commitment::L1BatchCommitmentMode, url::SensitiveUrl, Address,
+    L1BatchNumber, L1ChainId, L2ChainId, ETHEREUM_ADDRESS,
 };
 use zksync_web3_decl::{
     client::{DynClient, L2},
@@ -746,6 +746,8 @@ pub(crate) struct ExperimentalENConfig {
     pub state_keeper_db_max_open_files: Option<NonZeroU32>,
 
     // Snapshot recovery
+    /// L1 batch number of the snapshot to use during recovery. Specifying this parameter is mostly useful for testing.
+    pub snapshots_recovery_l1_batch: Option<L1BatchNumber>,
     /// Approximate chunk size (measured in the number of entries) to recover in a single iteration.
     /// Reasonable values are order of 100,000 (meaning an iteration takes several seconds).
     ///
@@ -775,6 +777,7 @@ impl ExperimentalENConfig {
             state_keeper_db_block_cache_capacity_mb:
                 Self::default_state_keeper_db_block_cache_capacity_mb(),
             state_keeper_db_max_open_files: None,
+            snapshots_recovery_l1_batch: None,
             snapshots_recovery_tree_chunk_size: Self::default_snapshots_recovery_tree_chunk_size(),
             commitment_generator_max_parallelism: None,
         }
@@ -807,21 +810,11 @@ pub(crate) fn read_consensus_config() -> anyhow::Result<Option<ConsensusConfig>>
     ))
 }
 
-/// Configuration for snapshot recovery. Loaded optionally, only if snapshot recovery is enabled.
-#[derive(Debug)]
-pub(crate) struct SnapshotsRecoveryConfig {
-    pub snapshots_object_store: ObjectStoreConfig,
-}
-
-impl SnapshotsRecoveryConfig {
-    pub fn new() -> anyhow::Result<Self> {
-        let snapshots_object_store = envy::prefixed("EN_SNAPSHOTS_OBJECT_STORE_")
-            .from_env::<ObjectStoreConfig>()
-            .context("failed loading snapshot object store config from env variables")?;
-        Ok(Self {
-            snapshots_object_store,
-        })
-    }
+/// Configuration for snapshot recovery. Should be loaded optionally, only if snapshot recovery is enabled.
+pub(crate) fn snapshot_recovery_object_store_config() -> anyhow::Result<ObjectStoreConfig> {
+    envy::prefixed("EN_SNAPSHOTS_OBJECT_STORE_")
+        .from_env::<ObjectStoreConfig>()
+        .context("failed loading snapshot object store config from env variables")
 }
 
 #[derive(Debug, Deserialize)]

core/bin/external_node/src/helpers.rs

@@ -176,13 +176,11 @@ impl ValidateChainIdsTask {
                 .fuse();
         let main_node_l2_check =
             Self::check_l2_chain_using_main_node(self.main_node_client, self.l2_chain_id).fuse();
-        loop {
-            tokio::select! {
-                Err(err) = eth_client_check => return Err(err),
-                Err(err) = main_node_l1_check => return Err(err),
-                Err(err) = main_node_l2_check => return Err(err),
-                _ = stop_receiver.changed() => return Ok(()),
-            }
+        tokio::select! {
+            Err(err) = eth_client_check =>  Err(err),
+            Err(err) = main_node_l1_check =>  Err(err),
+            Err(err) = main_node_l2_check =>  Err(err),
+            _ = stop_receiver.changed() =>  Ok(()),
         }
     }
 }

core/bin/external_node/src/init.rs

@@ -12,7 +12,13 @@ use zksync_snapshots_applier::{SnapshotsApplierConfig, SnapshotsApplierTask};
 use zksync_types::{L1BatchNumber, L2ChainId};
 use zksync_web3_decl::client::{DynClient, L2};
 
-use crate::config::SnapshotsRecoveryConfig;
+use crate::config::snapshot_recovery_object_store_config;
+
+#[derive(Debug)]
+pub(crate) struct SnapshotRecoveryConfig {
+    /// If not specified, the latest snapshot will be used.
+    pub snapshot_l1_batch_override: Option<L1BatchNumber>,
+}
 
 #[derive(Debug)]
 enum InitDecision {
@@ -27,7 +33,7 @@ pub(crate) async fn ensure_storage_initialized(
     main_node_client: Box<DynClient<L2>>,
     app_health: &AppHealthCheck,
     l2_chain_id: L2ChainId,
-    consider_snapshot_recovery: bool,
+    recovery_config: Option<SnapshotRecoveryConfig>,
 ) -> anyhow::Result<()> {
     let mut storage = pool.connection_tagged("en").await?;
     let genesis_l1_batch = storage
@@ -57,7 +63,7 @@ pub(crate) async fn ensure_storage_initialized(
         }
         (None, None) => {
             tracing::info!("Node has neither genesis L1 batch, nor snapshot recovery info");
-            if consider_snapshot_recovery {
+            if recovery_config.is_some() {
                 InitDecision::SnapshotRecovery
             } else {
                 InitDecision::Genesis
@@ -78,25 +84,31 @@ pub(crate) async fn ensure_storage_initialized(
             .context("performing genesis failed")?;
         }
         InitDecision::SnapshotRecovery => {
-            anyhow::ensure!(
-                consider_snapshot_recovery,
+            let recovery_config = recovery_config.context(
                 "Snapshot recovery is required to proceed, but it is not enabled. Enable by setting \
                  `EN_SNAPSHOTS_RECOVERY_ENABLED=true` env variable to the node binary, or use a Postgres dump for recovery"
-            );
+            )?;
 
             tracing::warn!("Proceeding with snapshot recovery. This is an experimental feature; use at your own risk");
-            let recovery_config = SnapshotsRecoveryConfig::new()?;
-            let blob_store = ObjectStoreFactory::new(recovery_config.snapshots_object_store)
+            let object_store_config = snapshot_recovery_object_store_config()?;
+            let object_store = ObjectStoreFactory::new(object_store_config)
                 .create_store()
-                .await;
+                .await?;
 
             let config = SnapshotsApplierConfig::default();
-            let snapshots_applier_task = SnapshotsApplierTask::new(
+            let mut snapshots_applier_task = SnapshotsApplierTask::new(
                 config,
                 pool,
                 Box::new(main_node_client.for_component("snapshot_recovery")),
-                blob_store,
+                object_store,
             );
+            if let Some(snapshot_l1_batch) = recovery_config.snapshot_l1_batch_override {
+                tracing::info!(
+                    "Using a specific snapshot with L1 batch #{snapshot_l1_batch}; this may not work \
+                     if the snapshot is too old (order of several weeks old) or non-existent"
+                );
+                snapshots_applier_task.set_snapshot_l1_batch(snapshot_l1_batch);
+            }
             app_health.insert_component(snapshots_applier_task.health_check())?;
 
             let recovery_started_at = Instant::now();

core/bin/external_node/src/main.rs

@@ -55,7 +55,7 @@ use zksync_web3_decl::{
 use crate::{
     config::ExternalNodeConfig,
     helpers::{MainNodeHealthCheck, ValidateChainIdsTask},
-    init::ensure_storage_initialized,
+    init::{ensure_storage_initialized, SnapshotRecoveryConfig},
     metrics::RUST_METRICS,
 };
 
@@ -908,12 +908,19 @@ async fn run_node(
     task_handles.extend(prometheus_task);
 
     // Make sure that the node storage is initialized either via genesis or snapshot recovery.
+    let recovery_config =
+        config
+            .optional
+            .snapshots_recovery_enabled
+            .then_some(SnapshotRecoveryConfig {
+                snapshot_l1_batch_override: config.experimental.snapshots_recovery_l1_batch,
+            });
     ensure_storage_initialized(
         connection_pool.clone(),
         main_node_client.clone(),
         &app_health,
         config.required.l2_chain_id,
-        config.optional.snapshots_recovery_enabled,
+        recovery_config,
     )
     .await?;
     let sigint_receiver = env.setup_sigint_handler();

core/bin/snapshots_creator/src/main.rs

@@ -75,7 +75,7 @@ async fn main() -> anyhow::Result<()> {
         SnapshotsObjectStoreConfig::from_env().context("SnapshotsObjectStoreConfig::from_env()")?;
     let blob_store = ObjectStoreFactory::new(object_store_config.0)
         .create_store()
-        .await;
+        .await?;
 
     let database_secrets = DatabaseSecrets::from_env().context("DatabaseSecrets")?;
     let creator_config =