fix build

matter-labs · Sep 12, 2024 · 7e93082 · 7e93082
1 parent a055636
commit 7e93082
Show file tree

Hide file tree

Showing 18 changed files with 109 additions and 160 deletions.
diff --git a/prover/crates/bin/witness_generator/src/artifacts.rs b/prover/crates/bin/witness_generator/src/artifacts.rs
@@ -2,9 +2,7 @@ use std::time::Instant;
 
 use async_trait::async_trait;
 use zksync_object_store::ObjectStore;
-use zksync_prover_dal::{ConnectionPool, Prover, ProverDal};
-use zksync_prover_fri_utils::get_recursive_layer_circuit_id_for_base_layer;
-use zksync_types::{basic_fri_types::AggregationRound, L1BatchNumber};
+use zksync_prover_dal::{ConnectionPool, Prover};
 
 #[derive(Debug)]
 pub(crate) struct AggregationBlobUrls {
@@ -32,7 +30,7 @@ pub(crate) trait ArtifactsManager {
     type OutputArtifacts;
 
     async fn get_artifacts(
-        metadata: &Self::Medatadata,
+        metadata: &Self::InputMetadata,
         object_store: &dyn ObjectStore,
     ) -> Self::InputArtifacts;
 
@@ -48,5 +46,5 @@ pub(crate) trait ArtifactsManager {
         started_at: Instant,
         blob_urls: BlobUrls,
         artifacts: Self::OutputArtifacts,
-    );
+    ) -> anyhow::Result<()>;
 }
diff --git a/prover/crates/bin/witness_generator/src/basic_circuits/artifacts.rs b/prover/crates/bin/witness_generator/src/basic_circuits/artifacts.rs
@@ -1,87 +1,64 @@
 use std::time::Instant;
 
 use async_trait::async_trait;
-use circuit_definitions::zkevm_circuits::scheduler::{
-    block_header::BlockAuxilaryOutputWitness, input::SchedulerCircuitInstanceWitness,
-};
-use zksync_multivm::circuit_sequencer_api_latest::boojum::{
-    field::goldilocks::{GoldilocksExt2, GoldilocksField},
-    gadgets::recursion::recursive_tree_hasher::CircuitGoldilocksPoseidon2Sponge,
-};
 use zksync_object_store::ObjectStore;
 use zksync_prover_dal::{ConnectionPool, Prover, ProverDal};
 use zksync_prover_fri_types::AuxOutputWitnessWrapper;
 use zksync_prover_fri_utils::get_recursive_layer_circuit_id_for_base_layer;
 use zksync_types::{basic_fri_types::AggregationRound, L1BatchNumber};
 
+use crate::basic_circuits::BasicCircuitArtifacts;
 use crate::{
     artifacts::{ArtifactsManager, BlobUrls},
     basic_circuits::{BasicWitnessGenerator, BasicWitnessGeneratorJob},
     utils::SchedulerPartialInputWrapper,
 };
 
-#[derive(Debug)]
-pub struct SchedulerArtifacts<'a> {
-    pub(super) block_number: L1BatchNumber,
-    pub(super) scheduler_partial_input: SchedulerCircuitInstanceWitness<
-        GoldilocksField,
-        CircuitGoldilocksPoseidon2Sponge,
-        GoldilocksExt2,
-    >,
-    pub(super) aux_output_witness: BlockAuxilaryOutputWitness<GoldilocksField>,
-    pub(super) public_object_store: Option<&'a dyn ObjectStore>,
-    pub(super) shall_save_to_public_bucket: bool,
-}
-
 #[async_trait]
 impl ArtifactsManager for BasicWitnessGenerator {
     type InputMetadata = L1BatchNumber;
     type InputArtifacts = BasicWitnessGeneratorJob;
-    type OutputArtifacts = SchedulerArtifacts<'_>;
+    type OutputArtifacts = BasicCircuitArtifacts;
 
     async fn get_artifacts(
         metadata: &Self::InputMetadata,
         object_store: &dyn ObjectStore,
     ) -> Self::InputArtifacts {
-        let l1_batch_number = metadata;
+        let l1_batch_number = *metadata;
         let job = object_store.get(l1_batch_number).await.unwrap();
-        BasicWitnessGeneratorJob { block_number, job }
+        BasicWitnessGeneratorJob {
+            block_number: l1_batch_number,
+            job,
+        }
     }
 
     async fn save_artifacts(
-        _job_id: u32,
+        job_id: u32,
         artifacts: Self::OutputArtifacts,
         object_store: &dyn ObjectStore,
     ) -> BlobUrls {
         let aux_output_witness_wrapper = AuxOutputWitnessWrapper(artifacts.aux_output_witness);
-        if artifacts.shall_save_to_public_bucket {
-            artifacts.public_object_store
-                .expect("public_object_store shall not be empty while running with shall_save_to_public_bucket config")
-                .put(artifacts.block_number, &aux_output_witness_wrapper)
-                .await
-                .unwrap();
-        }
         object_store
-            .put(artifacts.block_number, &aux_output_witness_wrapper)
+            .put(L1BatchNumber(job_id), &aux_output_witness_wrapper)
             .await
             .unwrap();
-        let wrapper = SchedulerPartialInputWrapper(artifacts.scheduler_partial_input);
+        let wrapper = SchedulerPartialInputWrapper(artifacts.scheduler_witness);
         let url = object_store
-            .put(artifacts.block_number, &wrapper)
+            .put(L1BatchNumber(job_id), &wrapper)
             .await
             .unwrap();
 
         BlobUrls::Url(url)
     }
 
-    #[tracing::instrument(skip_all, fields(l1_batch = %block_number))]
+    #[tracing::instrument(skip_all, fields(l1_batch = %job_id))]
     async fn update_database(
         connection_pool: &ConnectionPool<Prover>,
-        _job_id: u32,
+        job_id: u32,
         started_at: Instant,
         blob_urls: BlobUrls,
-        artifacts: Self::OutputArtifacts,
-    ) {
+        _artifacts: Self::OutputArtifacts,
+    ) -> anyhow::Result<()> {
         let blob_urls = match blob_urls {
             BlobUrls::Scheduler(blobs) => blobs,
             _ => unreachable!(),
@@ -97,12 +74,12 @@ impl ArtifactsManager for BasicWitnessGenerator {
             .expect("failed to get database transaction");
         let protocol_version_id = transaction
             .fri_witness_generator_dal()
-            .protocol_version_for_l1_batch(artifacts.block_number)
+            .protocol_version_for_l1_batch(L1BatchNumber(job_id))
             .await;
         transaction
             .fri_prover_jobs_dal()
             .insert_prover_jobs(
-                artifacts.block_number,
+                L1BatchNumber(job_id),
                 blob_urls.circuit_ids_and_urls,
                 AggregationRound::BasicCircuits,
                 0,
@@ -112,7 +89,7 @@ impl ArtifactsManager for BasicWitnessGenerator {
         transaction
             .fri_witness_generator_dal()
             .create_aggregation_jobs(
-                artifacts.block_number,
+                L1BatchNumber(job_id),
                 &blob_urls.closed_form_inputs_and_urls,
                 &blob_urls.scheduler_witness_url,
                 get_recursive_layer_circuit_id_for_base_layer,
@@ -121,11 +98,12 @@ impl ArtifactsManager for BasicWitnessGenerator {
             .await;
         transaction
             .fri_witness_generator_dal()
-            .mark_witness_job_as_successful(artifacts.block_number, started_at.elapsed())
+            .mark_witness_job_as_successful(L1BatchNumber(job_id), started_at.elapsed())
             .await;
         transaction
             .commit()
             .await
             .expect("failed to commit database transaction");
+        Ok(())
     }
 }
diff --git a/prover/crates/bin/witness_generator/src/basic_circuits/job_processor.rs b/prover/crates/bin/witness_generator/src/basic_circuits/job_processor.rs
@@ -3,16 +3,13 @@ use std::{sync::Arc, time::Instant};
 use anyhow::Context as _;
 use tracing::Instrument;
 use zksync_prover_dal::ProverDal;
-use zksync_prover_fri_types::get_current_pod_name;
+use zksync_prover_fri_types::{get_current_pod_name, AuxOutputWitnessWrapper};
 use zksync_queued_job_processor::{async_trait, JobProcessor};
 use zksync_types::{basic_fri_types::AggregationRound, L1BatchNumber};
 
 use crate::{
     artifacts::{ArtifactsManager, BlobUrls, SchedulerBlobUrls},
-    basic_circuits::{
-        artifacts::SchedulerArtifacts, BasicCircuitArtifacts, BasicWitnessGenerator,
-        BasicWitnessGeneratorJob,
-    },
+    basic_circuits::{BasicCircuitArtifacts, BasicWitnessGenerator, BasicWitnessGeneratorJob},
     metrics::WITNESS_GENERATOR_METRICS,
 };
 
@@ -96,15 +93,18 @@ impl JobProcessor for BasicWitnessGenerator {
             None => Ok(()),
             Some(artifacts) => {
                 let blob_started_at = Instant::now();
-                let circuit_urls = artifacts.circuit_urls;
-                let queue_urls = artifacts.queue_urls;
-                let artifacts = SchedulerArtifacts {
-                    block_number: job_id,
-                    scheduler_partial_input: artifacts.scheduler_witness,
-                    aux_output_witness: artifacts.aux_output_witness,
-                    public_object_store: self.public_blob_store.as_deref(),
-                    shall_save_to_public_bucket: self.config.shall_save_to_public_bucket,
-                };
+                let circuit_urls = artifacts.circuit_urls.clone();
+                let queue_urls = artifacts.queue_urls.clone();
+
+                let aux_output_witness_wrapper =
+                    AuxOutputWitnessWrapper(artifacts.aux_output_witness.clone());
+                if self.config.shall_save_to_public_bucket {
+                    self.public_blob_store.as_deref()
+                        .expect("public_object_store shall not be empty while running with shall_save_to_public_bucket config")
+                        .put(job_id, &aux_output_witness_wrapper)
+                        .await
+                        .unwrap();
+                }
 
                 let scheduler_witness_url =
                     match Self::save_artifacts(job_id.0, artifacts.clone(), &*self.object_store)
@@ -128,7 +128,7 @@ impl JobProcessor for BasicWitnessGenerator {
                     }),
                     artifacts,
                 )
-                .await;
+                .await?;
                 Ok(())
             }
         }

diff --git a/prover/crates/bin/witness_generator/src/basic_circuits/mod.rs b/prover/crates/bin/witness_generator/src/basic_circuits/mod.rs
@@ -32,33 +32,29 @@ use zksync_multivm::{
     zk_evm_latest::ethereum_types::Address,
 };
 use zksync_object_store::ObjectStore;
-use zksync_prover_dal::{ConnectionPool, Prover, ProverDal};
-use zksync_prover_fri_types::{
-    get_current_pod_name, keys::ClosedFormInputKey, AuxOutputWitnessWrapper, CircuitAuxData,
-};
-use zksync_prover_fri_utils::get_recursive_layer_circuit_id_for_base_layer;
+use zksync_prover_dal::{ConnectionPool, Prover};
+use zksync_prover_fri_types::{keys::ClosedFormInputKey, CircuitAuxData};
 use zksync_prover_interface::inputs::WitnessInputData;
-use zksync_queued_job_processor::{async_trait, JobProcessor};
 use zksync_system_constants::BOOTLOADER_ADDRESS;
 use zksync_types::{
     basic_fri_types::AggregationRound, protocol_version::ProtocolSemanticVersion, L1BatchNumber,
 };
 
 use crate::{
-    artifacts::ArtifactsManager,
     metrics::WITNESS_GENERATOR_METRICS,
     precalculated_merkle_paths_provider::PrecalculatedMerklePathsProvider,
     storage_oracle::StorageOracle,
     utils::{
         expand_bootloader_contents, save_circuit, save_ram_premutation_queue_witness,
-        ClosedFormInputWrapper, SchedulerPartialInputWrapper, KZG_TRUSTED_SETUP_FILE,
+        ClosedFormInputWrapper, KZG_TRUSTED_SETUP_FILE,
     },
     witness::WitnessStorage,
 };
 
 mod artifacts;
 pub mod job_processor;
 
+#[derive(Clone)]
 pub struct BasicCircuitArtifacts {
     pub(super) circuit_urls: Vec<(u8, String)>,
     pub(super) queue_urls: Vec<(u8, String, usize)>,

diff --git a/prover/crates/bin/witness_generator/src/leaf_aggregation/artifacts.rs b/prover/crates/bin/witness_generator/src/leaf_aggregation/artifacts.rs
@@ -1,12 +1,11 @@
+use async_trait::async_trait;
 use std::time::Instant;
 
 use zksync_object_store::ObjectStore;
 use zksync_prover_dal::{ConnectionPool, Prover, ProverDal};
 use zksync_prover_fri_types::keys::ClosedFormInputKey;
 use zksync_prover_fri_utils::get_recursive_layer_circuit_id_for_base_layer;
-use zksync_types::{
-    basic_fri_types::AggregationRound, prover_dal::LeafAggregationJobMetadata, L1BatchNumber,
-};
+use zksync_types::{basic_fri_types::AggregationRound, prover_dal::LeafAggregationJobMetadata};
 
 use crate::{
     artifacts::{AggregationBlobUrls, ArtifactsManager, BlobUrls},
@@ -15,15 +14,12 @@ use crate::{
     utils::{save_node_aggregations_artifacts, ClosedFormInputWrapper},
 };
 
-pub struct LeafAggregationArtifactsMetadata {
-    pub job_id: u32,
-    pub circuit_id: u8,
-}
-
+#[async_trait]
 impl ArtifactsManager for LeafAggregationWitnessGenerator {
     type InputMetadata = LeafAggregationJobMetadata;
     type InputArtifacts = ClosedFormInputWrapper;
     type OutputArtifacts = LeafAggregationArtifacts;
+
     async fn get_artifacts(
         metadata: &Self::InputMetadata,
         object_store: &dyn ObjectStore,
@@ -68,15 +64,15 @@ impl ArtifactsManager for LeafAggregationWitnessGenerator {
 
     #[tracing::instrument(
         skip_all,
-        fields(l1_batch = %block_number, circuit_id = %circuit_id)
+        fields(l1_batch = %job_id)
     )]
     async fn update_database(
         connection_pool: &ConnectionPool<Prover>,
         job_id: u32,
         started_at: Instant,
         blob_urls: BlobUrls,
         artifacts: Self::OutputArtifacts,
-    ) {
+    ) -> anyhow::Result<()> {
         tracing::info!(
             "Updating database for job_id {}, block {} with circuit id {}",
             job_id,
@@ -146,6 +142,7 @@ impl ArtifactsManager for LeafAggregationWitnessGenerator {
             artifacts.block_number.0,
             artifacts.circuit_id,
         );
-        transaction.commit().await.unwrap();
+        transaction.commit().await?;
+        Ok(())
     }
 }
diff --git a/prover/crates/bin/witness_generator/src/leaf_aggregation/job_processor.rs b/prover/crates/bin/witness_generator/src/leaf_aggregation/job_processor.rs
@@ -8,10 +8,9 @@ use zksync_queued_job_processor::JobProcessor;
 use zksync_types::basic_fri_types::AggregationRound;
 
 use crate::{
-    artifacts::{ArtifactsManager, BlobUrls},
+    artifacts::ArtifactsManager,
     leaf_aggregation::{
-        artifacts::LeafAggregationArtifactsMetadata, prepare_leaf_aggregation_job,
-        LeafAggregationArtifacts, LeafAggregationWitnessGenerator,
+        prepare_leaf_aggregation_job, LeafAggregationArtifacts, LeafAggregationWitnessGenerator,
         LeafAggregationWitnessGeneratorJob,
     },
     metrics::WITNESS_GENERATOR_METRICS,
@@ -101,7 +100,7 @@ impl JobProcessor for LeafAggregationWitnessGenerator {
             blob_urls,
             artifacts,
         )
-        .await;
+        .await?;
         Ok(())
     }
 

diff --git a/prover/crates/bin/witness_generator/src/leaf_aggregation/mod.rs b/prover/crates/bin/witness_generator/src/leaf_aggregation/mod.rs
@@ -1,7 +1,6 @@
 use std::{sync::Arc, time::Instant};
 
 use anyhow::Context as _;
-use async_trait::async_trait;
 use circuit_definitions::circuit_definitions::recursion_layer::base_circuit_type_into_recursive_leaf_circuit_type;
 use tokio::sync::Semaphore;
 use zkevm_test_harness::{
@@ -12,7 +11,7 @@ use zkevm_test_harness::{
 };
 use zksync_config::configs::FriWitnessGeneratorConfig;
 use zksync_object_store::ObjectStore;
-use zksync_prover_dal::{ConnectionPool, Prover, ProverDal};
+use zksync_prover_dal::{ConnectionPool, Prover};
 use zksync_prover_fri_types::{
     circuit_definitions::{
         boojum::field::goldilocks::GoldilocksField,
@@ -22,24 +21,20 @@ use zksync_prover_fri_types::{
         encodings::recursion_request::RecursionQueueSimulator,
         zkevm_circuits::recursion::leaf_layer::input::RecursionLeafParametersWitness,
     },
-    get_current_pod_name,
-    keys::ClosedFormInputKey,
     FriProofWrapper,
 };
-use zksync_prover_fri_utils::get_recursive_layer_circuit_id_for_base_layer;
 use zksync_prover_keystore::keystore::Keystore;
-use zksync_queued_job_processor::JobProcessor;
 use zksync_types::{
     basic_fri_types::AggregationRound, protocol_version::ProtocolSemanticVersion,
     prover_dal::LeafAggregationJobMetadata, L1BatchNumber,
 };
 
 use crate::{
-    artifacts::{AggregationBlobUrls, ArtifactsManager, BlobUrls},
+    artifacts::ArtifactsManager,
     metrics::WITNESS_GENERATOR_METRICS,
     utils::{
-        load_proofs_for_job_ids, save_node_aggregations_artifacts,
-        save_recursive_layer_prover_input_artifacts, ClosedFormInputWrapper,
+        load_proofs_for_job_ids, save_recursive_layer_prover_input_artifacts,
+        ClosedFormInputWrapper,
     },
 };
 
@@ -64,6 +59,7 @@ pub struct LeafAggregationWitnessGenerator {
     keystore: Keystore,
 }
 
+#[derive(Clone)]
 pub struct LeafAggregationArtifacts {
     circuit_id: u8,
     block_number: L1BatchNumber,