Merge remote-tracking branch 'origin/main' into gprusak-autogen

matter-labs · Jul 23, 2024 · 4edbbf6 · 4edbbf6
2 parents 77596e8 + 990676c
commit 4edbbf6
Show file tree

Hide file tree

Showing 382 changed files with 1,131 additions and 312 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -39,7 +39,7 @@ contracts/.git
 !etc/multivm_bootloaders
 !cargo
 !bellman-cuda
-!prover/vk_setup_data_generator_server_fri/data/
+!prover/crates/bin/vk_setup_data_generator_server_fri/data/
 !.github/release-please/manifest.json
 
 !etc/env/file_based

diff --git a/.github/workflows/build-prover-fri-gpu-gar.yml b/.github/workflows/build-prover-fri-gpu-gar.yml
@@ -47,7 +47,6 @@ jobs:
             PROVER_IMAGE=${{ inputs.image_tag_suffix }}
           push: true
           tags: |
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }}
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
       - name: Login to Asia GAR
@@ -56,9 +55,6 @@ jobs:
 
       - name: Build and push to Asia GAR
         run: |
-          docker buildx imagetools create \
-            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }}
           docker buildx imagetools create \
             --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
@@ -69,9 +65,6 @@ jobs:
 
       - name: Build and push to Europe GAR
         run: |
-          docker buildx imagetools create \
-            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }}
           docker buildx imagetools create \
             --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml
@@ -199,19 +199,13 @@ jobs:
       - name: Login and push to Asia GAR
         run: |
           gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
-          docker buildx imagetools create \
-            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
           docker buildx imagetools create \
             --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ needs.build-images.outputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ needs.build-images.outputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
       - name: Login and push to Europe GAR
         run: |
           gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
-          docker buildx imagetools create \
-            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
           docker buildx imagetools create \
             --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ needs.build-images.outputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ needs.build-images.outputs.protocol_version }}-${{ inputs.image_tag_suffix }}

diff --git a/.gitignore b/.gitignore
@@ -108,7 +108,7 @@ hyperchain-*.yml
 /etc/hyperchains/artifacts
 
 # Prover keys that should not be commited
-prover/vk_setup_data_generator_server_fri/data/setup_*
+prover/crates/bin/vk_setup_data_generator_server_fri/data/setup_*
 
 # Zk Toolbox
 chains/era/configs/*

diff --git a/Cargo.toml b/Cargo.toml
@@ -79,7 +79,7 @@ members = [
     "core/tests/vm-benchmark/harness",
 
     # Parts of prover workspace that are needed for Core workspace
-    "prover/prover_dal"
+    "prover/crates/lib/prover_dal"
 ]
 resolver = "2"
 
@@ -222,7 +222,7 @@ zksync_protobuf_build = "=0.1.0-rc.2"
 
 # "Local" dependencies
 zksync_multivm = { version = "0.1.0", path = "core/lib/multivm" }
-zksync_prover_dal = { version = "0.1.0", path = "prover/prover_dal" }
+zksync_prover_dal = { version = "0.1.0", path = "prover/crates/lib/prover_dal" }
 zksync_vlog = { version = "0.1.0", path = "core/lib/vlog" }
 zksync_vm_utils = { version = "0.1.0", path = "core/lib/vm_utils" }
 zksync_vm_benchmark_harness = { version = "0.1.0", path = "core/tests/vm-benchmark/harness" }

diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs
@@ -91,12 +91,25 @@ fn main() -> anyhow::Result<()> {
     let tmp_config = load_env_config()?;
 
     let configs = match opt.config_path {
-        None => tmp_config.general(),
+        None => {
+            let mut configs = tmp_config.general();
+            configs.consensus_config =
+                config::read_consensus_config().context("read_consensus_config()")?;
+            configs
+        }
         Some(path) => {
             let yaml =
                 std::fs::read_to_string(&path).with_context(|| path.display().to_string())?;
-            decode_yaml_repr::<zksync_protobuf_config::proto::general::GeneralConfig>(&yaml)
-                .context("failed decoding general YAML config")?
+            let mut configs =
+                decode_yaml_repr::<zksync_protobuf_config::proto::general::GeneralConfig>(&yaml)
+                    .context("failed decoding general YAML config")?;
+            // Fallback to the consensus_config.yaml file.
+            // TODO: remove once we move the consensus config to general config on stage
+            if configs.consensus_config.is_none() {
+                configs.consensus_config =
+                    config::read_consensus_config().context("read_consensus_config()")?;
+            }
+            configs
         }
     };
 
@@ -154,8 +167,6 @@ fn main() -> anyhow::Result<()> {
         },
     };
 
-    let consensus = config::read_consensus_config().context("read_consensus_config()")?;
-
     let contracts_config = match opt.contracts_config_path {
         None => ContractsConfig::from_env().context("contracts_config")?,
         Some(path) => {
@@ -176,14 +187,7 @@ fn main() -> anyhow::Result<()> {
         }
     };
 
-    let node = MainNodeBuilder::new(
-        configs,
-        wallets,
-        genesis,
-        contracts_config,
-        secrets,
-        consensus,
-    );
+    let node = MainNodeBuilder::new(configs, wallets, genesis, contracts_config, secrets);
 
     if opt.genesis {
         // If genesis is requested, we don't need to run the node.

diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs
@@ -3,10 +3,7 @@
 
 use anyhow::Context;
 use zksync_config::{
-    configs::{
-        consensus::ConsensusConfig, eth_sender::PubdataSendingMode, wallets::Wallets,
-        GeneralConfig, Secrets,
-    },
+    configs::{eth_sender::PubdataSendingMode, wallets::Wallets, GeneralConfig, Secrets},
     ContractsConfig, GenesisConfig,
 };
 use zksync_core_leftovers::Component;
@@ -86,7 +83,6 @@ pub struct MainNodeBuilder {
     genesis_config: GenesisConfig,
     contracts_config: ContractsConfig,
     secrets: Secrets,
-    consensus_config: Option<ConsensusConfig>,
 }
 
 impl MainNodeBuilder {
@@ -96,7 +92,6 @@ impl MainNodeBuilder {
         genesis_config: GenesisConfig,
         contracts_config: ContractsConfig,
         secrets: Secrets,
-        consensus_config: Option<ConsensusConfig>,
     ) -> Self {
         Self {
             node: ZkStackServiceBuilder::new(),
@@ -105,7 +100,6 @@ impl MainNodeBuilder {
             genesis_config,
             contracts_config,
             secrets,
-            consensus_config,
         }
     }
 
@@ -456,6 +450,7 @@ impl MainNodeBuilder {
     fn add_consensus_layer(mut self) -> anyhow::Result<Self> {
         self.node.add_layer(MainNodeConsensusLayer {
             config: self
+                .configs
                 .consensus_config
                 .clone()
                 .context("Consensus config has to be provided")?,

diff --git a/core/lib/basic_types/src/prover_dal.rs b/core/lib/basic_types/src/prover_dal.rs
@@ -5,9 +5,7 @@ use chrono::{DateTime, Duration, NaiveDateTime, NaiveTime, Utc};
 use strum::{Display, EnumString};
 
 use crate::{
-    basic_fri_types::{AggregationRound, Eip4844Blobs},
-    protocol_version::ProtocolVersionId,
-    L1BatchNumber,
+    basic_fri_types::AggregationRound, protocol_version::ProtocolVersionId, L1BatchNumber,
 };
 
 #[derive(Debug, Clone)]
@@ -255,7 +253,6 @@ pub struct ProverJobFriInfo {
     pub created_at: NaiveDateTime,
     pub updated_at: NaiveDateTime,
     pub time_taken: Option<NaiveTime>,
-    pub is_blob_cleaned: Option<bool>,
     pub depth: u32,
     pub is_node_final_proof: bool,
     pub proof_blob_url: Option<String>,
@@ -266,7 +263,6 @@ pub struct ProverJobFriInfo {
 #[derive(Debug, Clone)]
 pub struct BasicWitnessGeneratorJobInfo {
     pub l1_batch_number: L1BatchNumber,
-    pub merkle_tree_paths_blob_url: Option<String>,
     pub witness_inputs_blob_url: Option<String>,
     pub attempts: u32,
     pub status: WitnessJobStatus,
@@ -275,10 +271,8 @@ pub struct BasicWitnessGeneratorJobInfo {
     pub updated_at: NaiveDateTime,
     pub processing_started_at: Option<NaiveDateTime>,
     pub time_taken: Option<NaiveTime>,
-    pub is_blob_cleaned: Option<bool>,
     pub protocol_version: Option<i32>,
     pub picked_by: Option<String>,
-    pub eip_4844_blobs: Option<Eip4844Blobs>,
 }
 
 #[derive(Debug, Clone)]
@@ -294,7 +288,6 @@ pub struct LeafWitnessGeneratorJobInfo {
     pub updated_at: NaiveDateTime,
     pub processing_started_at: Option<NaiveDateTime>,
     pub time_taken: Option<NaiveTime>,
-    pub is_blob_cleaned: Option<bool>,
     pub number_of_basic_circuits: Option<i32>,
     pub protocol_version: Option<i32>,
     pub picked_by: Option<String>,

diff --git a/core/lib/config/src/configs/general.rs b/core/lib/config/src/configs/general.rs
@@ -2,6 +2,7 @@ use crate::{
     configs::{
         base_token_adjuster::BaseTokenAdjusterConfig,
         chain::{CircuitBreakerConfig, MempoolConfig, OperationsManagerConfig, StateKeeperConfig},
+        consensus::ConsensusConfig,
         da_dispatcher::DADispatcherConfig,
         fri_prover_group::FriProverGroupConfig,
         house_keeper::HouseKeeperConfig,
@@ -17,7 +18,7 @@ use crate::{
     SnapshotsCreatorConfig,
 };
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub struct GeneralConfig {
     pub postgres_config: Option<PostgresConfig>,
     pub api_config: Option<ApiConfig>,
@@ -48,4 +49,5 @@ pub struct GeneralConfig {
     pub core_object_store: Option<ObjectStoreConfig>,
     pub base_token_adjuster: Option<BaseTokenAdjusterConfig>,
     pub external_price_api_client_config: Option<ExternalPriceApiClientConfig>,
+    pub consensus_config: Option<ConsensusConfig>,
 }