Merge branch 'main' into matias-check-zksync-repo

.dockerignore

@@ -39,7 +39,7 @@ contracts/.git
 !etc/multivm_bootloaders
 !cargo
 !bellman-cuda
-!prover/vk_setup_data_generator_server_fri/data/
+!prover/crates/bin/vk_setup_data_generator_server_fri/data/
 !.github/release-please/manifest.json
 
 !etc/env/file_based

.github/release-please/manifest.json

@@ -1,4 +1,4 @@
 {
-  "core": "24.9.0",
+  "core": "24.10.0",
   "prover": "16.0.0"
 }

.github/workflows/build-docker-from-tag.yml

@@ -61,7 +61,7 @@ jobs:
 
   build-push-tee-prover-images:
     name: Build and push images
-    needs: [setup, changed_files]
+    needs: [setup]
     uses: ./.github/workflows/build-tee-prover-template.yml
     if: contains(github.ref_name, 'core')
     secrets:

.github/workflows/build-prover-fri-gpu-gar.yml

@@ -47,7 +47,6 @@ jobs:
             PROVER_IMAGE=${{ inputs.image_tag_suffix }}
           push: true
           tags: |
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }}
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
       - name: Login to Asia GAR
@@ -56,9 +55,6 @@ jobs:
 
       - name: Build and push to Asia GAR
         run: |
-          docker buildx imagetools create \
-            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }}
           docker buildx imagetools create \
             --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
@@ -69,9 +65,6 @@ jobs:
 
       - name: Build and push to Europe GAR
         run: |
-          docker buildx imagetools create \
-            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }}
           docker buildx imagetools create \
             --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}

.github/workflows/build-prover-template.yml

@@ -199,19 +199,13 @@ jobs:
       - name: Login and push to Asia GAR
         run: |
           gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
-          docker buildx imagetools create \
-            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
           docker buildx imagetools create \
             --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ needs.build-images.outputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ needs.build-images.outputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
       - name: Login and push to Europe GAR
         run: |
           gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
-          docker buildx imagetools create \
-            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
           docker buildx imagetools create \
             --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ needs.build-images.outputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ needs.build-images.outputs.protocol_version }}-${{ inputs.image_tag_suffix }}

.gitignore

@@ -108,7 +108,7 @@ hyperchain-*.yml
 /etc/hyperchains/artifacts
 
 # Prover keys that should not be commited
-prover/vk_setup_data_generator_server_fri/data/setup_*
+prover/crates/bin/vk_setup_data_generator_server_fri/data/setup_*
 
 # Zk Toolbox
 chains/era/configs/*

Cargo.toml

@@ -79,7 +79,7 @@ members = [
     "core/tests/vm-benchmark/harness",
 
     # Parts of prover workspace that are needed for Core workspace
-    "prover/prover_dal"
+    "prover/crates/lib/prover_dal"
 ]
 resolver = "2"
 
@@ -222,7 +222,7 @@ zksync_protobuf_build = "=0.1.0-rc.2"
 
 # "Local" dependencies
 zksync_multivm = { version = "0.1.0", path = "core/lib/multivm" }
-zksync_prover_dal = { version = "0.1.0", path = "prover/prover_dal" }
+zksync_prover_dal = { version = "0.1.0", path = "prover/crates/lib/prover_dal" }
 zksync_vlog = { version = "0.1.0", path = "core/lib/vlog" }
 zksync_vm_utils = { version = "0.1.0", path = "core/lib/vm_utils" }
 zksync_vm_benchmark_harness = { version = "0.1.0", path = "core/tests/vm-benchmark/harness" }

core/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## [24.10.0](https://github.com/matter-labs/zksync-era/compare/core-v24.9.0...core-v24.10.0) (2024-07-22)
+
+
+### Features
+
+* Add blob size metrics ([#2411](https://github.com/matter-labs/zksync-era/issues/2411)) ([41c535a](https://github.com/matter-labs/zksync-era/commit/41c535af2bcc72000116277d5dd9e04b5c0b2372))
+* **en:** Switch EN to use node framework ([#2427](https://github.com/matter-labs/zksync-era/issues/2427)) ([0cee530](https://github.com/matter-labs/zksync-era/commit/0cee530b2f2e8304b7e20a093a32abe116463b57))
+* **eth-sender:** add early return in sending new transactions to not spam logs with errors ([#2425](https://github.com/matter-labs/zksync-era/issues/2425)) ([192f2a3](https://github.com/matter-labs/zksync-era/commit/192f2a374d83eaecb52f198fdcfa615262378530))
+* **eth-watch:** Integrate decentralized upgrades ([#2401](https://github.com/matter-labs/zksync-era/issues/2401)) ([5a48e10](https://github.com/matter-labs/zksync-era/commit/5a48e1026260024c6ae2b4d1100ee9b798a83e8d))
+* L1 batch signing (BFT-474) ([#2414](https://github.com/matter-labs/zksync-era/issues/2414)) ([ab699db](https://github.com/matter-labs/zksync-era/commit/ab699dbe8cffa8bd291d6054579061b47fd4aa0e))
+* **prover:** Make it possible to run prover out of GCP ([#2448](https://github.com/matter-labs/zksync-era/issues/2448)) ([c9da549](https://github.com/matter-labs/zksync-era/commit/c9da5497e2aa9d85f204ab7b74fefcfe941793ff))
+* **zk_toolbox:** Small adjustment for zk toolbox ([#2424](https://github.com/matter-labs/zksync-era/issues/2424)) ([ce43c42](https://github.com/matter-labs/zksync-era/commit/ce43c422fddccfe88c07ee22a2b8726dd0bd5f61))
+
+
+### Bug Fixes
+
+* **eth-sender:** add bump of min 10% when resending txs to avoid "replacement transaction underpriced" ([#2422](https://github.com/matter-labs/zksync-era/issues/2422)) ([a7bcf5d](https://github.com/matter-labs/zksync-era/commit/a7bcf5d7f75eb45384312d7c97f25a50a91e7a31))
+* Set attesters in Connection::adjust_genesis (BFT-489) ([#2429](https://github.com/matter-labs/zksync-era/issues/2429)) ([ca4cb3c](https://github.com/matter-labs/zksync-era/commit/ca4cb3cba04757dc1760397c667a838931cd2d11))
+
 ## [24.9.0](https://github.com/matter-labs/zksync-era/compare/core-v24.8.0...core-v24.9.0) (2024-07-10)
 
 

core/bin/external_node/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "zksync_external_node"
 description = "Non-validator ZKsync node"
-version = "24.9.0" # x-release-please-version
+version = "24.10.0" # x-release-please-version
 edition.workspace = true
 authors.workspace = true
 homepage.workspace = true

core/bin/zksync_server/src/main.rs

@@ -91,12 +91,25 @@ fn main() -> anyhow::Result<()> {
     let tmp_config = load_env_config()?;
 
     let configs = match opt.config_path {
-        None => tmp_config.general(),
+        None => {
+            let mut configs = tmp_config.general();
+            configs.consensus_config =
+                config::read_consensus_config().context("read_consensus_config()")?;
+            configs
+        }
         Some(path) => {
             let yaml =
                 std::fs::read_to_string(&path).with_context(|| path.display().to_string())?;
-            decode_yaml_repr::<zksync_protobuf_config::proto::general::GeneralConfig>(&yaml)
-                .context("failed decoding general YAML config")?
+            let mut configs =
+                decode_yaml_repr::<zksync_protobuf_config::proto::general::GeneralConfig>(&yaml)
+                    .context("failed decoding general YAML config")?;
+            // Fallback to the consensus_config.yaml file.
+            // TODO: remove once we move the consensus config to general config on stage
+            if configs.consensus_config.is_none() {
+                configs.consensus_config =
+                    config::read_consensus_config().context("read_consensus_config()")?;
+            }
+            configs
         }
     };
 
@@ -154,8 +167,6 @@ fn main() -> anyhow::Result<()> {
         },
     };
 
-    let consensus = config::read_consensus_config().context("read_consensus_config()")?;
-
     let contracts_config = match opt.contracts_config_path {
         None => ContractsConfig::from_env().context("contracts_config")?,
         Some(path) => {
@@ -176,14 +187,7 @@ fn main() -> anyhow::Result<()> {
         }
     };
 
-    let node = MainNodeBuilder::new(
-        configs,
-        wallets,
-        genesis,
-        contracts_config,
-        secrets,
-        consensus,
-    );
+    let node = MainNodeBuilder::new(configs, wallets, genesis, contracts_config, secrets);
 
     if opt.genesis {
         // If genesis is requested, we don't need to run the node.

core/bin/zksync_server/src/node_builder.rs

@@ -3,10 +3,7 @@
 
 use anyhow::Context;
 use zksync_config::{
-    configs::{
-        consensus::ConsensusConfig, eth_sender::PubdataSendingMode, wallets::Wallets,
-        GeneralConfig, Secrets,
-    },
+    configs::{eth_sender::PubdataSendingMode, wallets::Wallets, GeneralConfig, Secrets},
     ContractsConfig, GenesisConfig,
 };
 use zksync_core_leftovers::Component;
@@ -86,7 +83,6 @@ pub struct MainNodeBuilder {
     genesis_config: GenesisConfig,
     contracts_config: ContractsConfig,
     secrets: Secrets,
-    consensus_config: Option<ConsensusConfig>,
 }
 
 impl MainNodeBuilder {
@@ -96,7 +92,6 @@ impl MainNodeBuilder {
         genesis_config: GenesisConfig,
         contracts_config: ContractsConfig,
         secrets: Secrets,
-        consensus_config: Option<ConsensusConfig>,
     ) -> Self {
         Self {
             node: ZkStackServiceBuilder::new(),
@@ -105,7 +100,6 @@ impl MainNodeBuilder {
             genesis_config,
             contracts_config,
             secrets,
-            consensus_config,
         }
     }
 
@@ -456,6 +450,7 @@ impl MainNodeBuilder {
     fn add_consensus_layer(mut self) -> anyhow::Result<Self> {
         self.node.add_layer(MainNodeConsensusLayer {
             config: self
+                .configs
                 .consensus_config
                 .clone()
                 .context("Consensus config has to be provided")?,

core/lib/basic_types/src/prover_dal.rs

@@ -5,9 +5,7 @@ use chrono::{DateTime, Duration, NaiveDateTime, NaiveTime, Utc};
 use strum::{Display, EnumString};
 
 use crate::{
-    basic_fri_types::{AggregationRound, Eip4844Blobs},
-    protocol_version::ProtocolVersionId,
-    L1BatchNumber,
+    basic_fri_types::AggregationRound, protocol_version::ProtocolVersionId, L1BatchNumber,
 };
 
 #[derive(Debug, Clone)]
@@ -255,7 +253,6 @@ pub struct ProverJobFriInfo {
     pub created_at: NaiveDateTime,
     pub updated_at: NaiveDateTime,
     pub time_taken: Option<NaiveTime>,
-    pub is_blob_cleaned: Option<bool>,
     pub depth: u32,
     pub is_node_final_proof: bool,
     pub proof_blob_url: Option<String>,
@@ -266,7 +263,6 @@ pub struct ProverJobFriInfo {
 #[derive(Debug, Clone)]
 pub struct BasicWitnessGeneratorJobInfo {
     pub l1_batch_number: L1BatchNumber,
-    pub merkle_tree_paths_blob_url: Option<String>,
     pub witness_inputs_blob_url: Option<String>,
     pub attempts: u32,
     pub status: WitnessJobStatus,
@@ -275,10 +271,8 @@ pub struct BasicWitnessGeneratorJobInfo {
     pub updated_at: NaiveDateTime,
     pub processing_started_at: Option<NaiveDateTime>,
     pub time_taken: Option<NaiveTime>,
-    pub is_blob_cleaned: Option<bool>,
     pub protocol_version: Option<i32>,
     pub picked_by: Option<String>,
-    pub eip_4844_blobs: Option<Eip4844Blobs>,
 }
 
 #[derive(Debug, Clone)]
@@ -294,7 +288,6 @@ pub struct LeafWitnessGeneratorJobInfo {
     pub updated_at: NaiveDateTime,
     pub processing_started_at: Option<NaiveDateTime>,
     pub time_taken: Option<NaiveTime>,
-    pub is_blob_cleaned: Option<bool>,
     pub number_of_basic_circuits: Option<i32>,
     pub protocol_version: Option<i32>,
     pub picked_by: Option<String>,

core/lib/config/src/configs/fri_prover.rs

@@ -10,6 +10,18 @@ pub enum SetupLoadMode {
     FromMemory,
 }
 
+/// Kind of cloud environment prover subsystem runs in.
+///
+/// Currently will only affect how the prover zone is chosen.
+#[derive(Debug, Default, Deserialize, Clone, Copy, PartialEq, Eq)]
+pub enum CloudType {
+    /// Assumes that the prover runs in GCP.
+    #[default]
+    GCP,
+    /// Assumes that the prover runs locally.
+    Local,
+}
+
 /// Configuration for the fri prover application
 #[derive(Debug, Deserialize, Clone, PartialEq)]
 pub struct FriProverConfig {
@@ -28,6 +40,8 @@ pub struct FriProverConfig {
     pub shall_save_to_public_bucket: bool,
     pub prover_object_store: Option<ObjectStoreConfig>,
     pub public_object_store: Option<ObjectStoreConfig>,
+    #[serde(default)]
+    pub cloud_type: CloudType,
 }
 
 impl FriProverConfig {

core/lib/config/src/configs/general.rs

@@ -2,6 +2,7 @@ use crate::{
     configs::{
         base_token_adjuster::BaseTokenAdjusterConfig,
         chain::{CircuitBreakerConfig, MempoolConfig, OperationsManagerConfig, StateKeeperConfig},
+        consensus::ConsensusConfig,
         da_dispatcher::DADispatcherConfig,
         fri_prover_group::FriProverGroupConfig,
         house_keeper::HouseKeeperConfig,
@@ -17,7 +18,7 @@ use crate::{
     SnapshotsCreatorConfig,
 };
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub struct GeneralConfig {
     pub postgres_config: Option<PostgresConfig>,
     pub api_config: Option<ApiConfig>,
@@ -48,4 +49,5 @@ pub struct GeneralConfig {
     pub core_object_store: Option<ObjectStoreConfig>,
     pub base_token_adjuster: Option<BaseTokenAdjusterConfig>,
     pub external_price_api_client_config: Option<ExternalPriceApiClientConfig>,
+    pub consensus_config: Option<ConsensusConfig>,
 }