From 01ecd2f9b7ca36ddead16536cfa54da1f943a97a Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Mon, 29 Apr 2024 13:20:34 -0300 Subject: [PATCH 1/9] add leaf query --- core/lib/basic_types/src/prover_dal.rs | 19 +++- .../prover_cli/src/commands/status/batch.rs | 22 ++++ ...5d2832571464e74b5fed92cf54617573c84ec.json | 106 ++++++++++++++++++ .../src/fri_witness_generator_dal.rs | 38 ++++++- 4 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 prover/prover_dal/.sqlx/query-21621153e545859d71188e2421f5d2832571464e74b5fed92cf54617573c84ec.json diff --git a/core/lib/basic_types/src/prover_dal.rs b/core/lib/basic_types/src/prover_dal.rs index f355932bb48f..7daea8a2205b 100644 --- a/core/lib/basic_types/src/prover_dal.rs +++ b/core/lib/basic_types/src/prover_dal.rs @@ -5,7 +5,7 @@ use chrono::{DateTime, Duration, NaiveDateTime, NaiveTime, Utc}; use strum::{Display, EnumString}; use crate::{ - basic_fri_types::{AggregationRound, Eip4844Blobs}, + basic_fri_types::{AggregationRound, CircuitIdRoundTuple, Eip4844Blobs}, protocol_version::ProtocolVersionId, L1BatchNumber, }; @@ -273,6 +273,23 @@ pub struct BasicWitnessGeneratorJobInfo { pub eip_4844_blobs: Option, } +pub struct LeafWitnessGeneratorJobInfo { + pub l1_batch_number: L1BatchNumber, + pub circuit_id: u32, + pub closed_form_inputs_blob_url: Option, + pub attempts: u32, + pub status: WitnessJobStatus, + pub error: Option, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, + pub processing_started_at: Option, + pub time_taken: Option, + pub is_blob_cleaned: Option, + pub number_of_basic_circuits: Option, + pub protocol_version: Option, + pub picked_by: Option, +} + #[derive(Debug, EnumString, Display)] pub enum ProofCompressionJobStatus { #[strum(serialize = "queued")] diff --git a/prover/prover_cli/src/commands/status/batch.rs b/prover/prover_cli/src/commands/status/batch.rs index 4b7933f0438c..d6f33c023716 100644 --- a/prover/prover_cli/src/commands/status/batch.rs +++ b/prover/prover_cli/src/commands/status/batch.rs @@ -53,6 +53,15 @@ async fn get_batches_data(batches: Vec) -> anyhow::Result( )) } +async fn get_proof_leaf_witness_generator_status_for_batch<'a>( + batch_number: L1BatchNumber, + conn: &mut Connection<'a, Prover>, +) -> TaskStatus { + conn.fri_witness_generator_dal() + .get_leaf_witness_generator_job_for_batch(batch_number) + .await + .map(|job| TaskStatus::from(job.status)) + .unwrap_or(TaskStatus::Custom( + "Leaf witness generator job not found 🚫".to_owned(), + )) +} + async fn get_proof_compression_job_status_for_batch<'a>( batch_number: L1BatchNumber, conn: &mut Connection<'a, Prover>, diff --git a/prover/prover_dal/.sqlx/query-21621153e545859d71188e2421f5d2832571464e74b5fed92cf54617573c84ec.json b/prover/prover_dal/.sqlx/query-21621153e545859d71188e2421f5d2832571464e74b5fed92cf54617573c84ec.json new file mode 100644 index 000000000000..9e750348decb --- /dev/null +++ b/prover/prover_dal/.sqlx/query-21621153e545859d71188e2421f5d2832571464e74b5fed92cf54617573c84ec.json @@ -0,0 +1,106 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n *\n FROM\n leaf_aggregation_witness_jobs_fri\n WHERE\n l1_batch_number = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "l1_batch_number", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "circuit_id", + "type_info": "Int2" + }, + { + "ordinal": 3, + "name": "closed_form_inputs_blob_url", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "attempts", + "type_info": "Int2" + }, + { + "ordinal": 5, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": "Timestamp" + }, + { + "ordinal": 8, + "name": "updated_at", + "type_info": "Timestamp" + }, + { + "ordinal": 9, + "name": "processing_started_at", + "type_info": "Timestamp" + }, + { + "ordinal": 10, + "name": "time_taken", + "type_info": "Time" + }, + { + "ordinal": 11, + "name": "is_blob_cleaned", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "number_of_basic_circuits", + "type_info": "Int4" + }, + { + "ordinal": 13, + "name": "protocol_version", + "type_info": "Int4" + }, + { + "ordinal": 14, + "name": "picked_by", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + true, + false, + false, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "21621153e545859d71188e2421f5d2832571464e74b5fed92cf54617573c84ec" +} diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index 2662183d22bf..30d9bb21c977 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -7,7 +7,7 @@ use zksync_basic_types::{ protocol_version::ProtocolVersionId, prover_dal::{ BasicWitnessGeneratorJobInfo, JobCountStatistics, LeafAggregationJobMetadata, - NodeAggregationJobMetadata, StuckJobs, WitnessJobStatus, + LeafWitnessGeneratorJobInfo, NodeAggregationJobMetadata, StuckJobs, WitnessJobStatus, }, L1BatchNumber, }; @@ -1187,4 +1187,40 @@ impl FriWitnessGeneratorDal<'_, '_> { eip_4844_blobs: row.eip_4844_blobs.map(|vec_u8| Eip4844Blobs::from(vec_u8)), }) } + + pub async fn get_leaf_witness_generator_job_for_batch( + &mut self, + l1_batch_number: L1BatchNumber, + ) -> Option { + sqlx::query!( + r#" + SELECT + * + FROM + leaf_aggregation_witness_jobs_fri + WHERE + l1_batch_number = $1 + "#, + i64::from(l1_batch_number.0) + ) + .fetch_optional(self.storage.conn()) + .await + .unwrap() + .map(|row| LeafWitnessGeneratorJobInfo { + l1_batch_number: l1_batch_number, + circuit_id: row.circuit_id as u32, + closed_form_inputs_blob_url: row.closed_form_inputs_blob_url, + attempts: row.attempts as u32, + status: WitnessJobStatus::from_str(&row.status).unwrap(), + error: row.error, + created_at: row.created_at, + updated_at: row.updated_at, + processing_started_at: row.processing_started_at, + time_taken: row.time_taken, + is_blob_cleaned: row.is_blob_cleaned, + protocol_version: row.protocol_version, + picked_by: row.picked_by, + number_of_basic_circuits: row.number_of_basic_circuits, + }) + } } From 590c58bbff5576f297e73d0cf836a440f3386f3c Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Mon, 29 Apr 2024 13:26:05 -0300 Subject: [PATCH 2/9] fix merge --- prover/prover_cli/src/commands/status/batch.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/prover/prover_cli/src/commands/status/batch.rs b/prover/prover_cli/src/commands/status/batch.rs index bb2d1a802aae..456d2b6e29d9 100644 --- a/prover/prover_cli/src/commands/status/batch.rs +++ b/prover/prover_cli/src/commands/status/batch.rs @@ -55,7 +55,7 @@ async fn get_batches_data(batches: Vec) -> anyhow::Result( .get_leaf_witness_generator_job_for_batch(batch_number) .await .map(|job| TaskStatus::from(job.status)) - .unwrap_or(TaskStatus::Custom( - "Leaf witness generator job not found 🚫".to_owned(), - )) + .unwrap_or_default() } async fn get_proof_compression_job_status_for_batch<'a>( From e08b05e383f0d35512df22fda48cbe8699dbda46 Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Mon, 29 Apr 2024 14:26:55 -0300 Subject: [PATCH 3/9] add querys --- core/lib/basic_types/src/prover_dal.rs | 33 +++++- .../prover_cli/src/commands/status/batch.rs | 41 +++++++ .../prover_cli/src/commands/status/utils.rs | 14 +-- ...dd8547a1ad20492ec37c3c0be5639e5d49952.json | 82 ++++++++++++++ ...9bfb838c787fc58d7536f9e9976e5e515431a.json | 106 ++++++++++++++++++ .../src/fri_witness_generator_dal.rs | 72 +++++++++++- 6 files changed, 339 insertions(+), 9 deletions(-) create mode 100644 prover/prover_dal/.sqlx/query-285d0ff850fa5c9af36564fcb14dd8547a1ad20492ec37c3c0be5639e5d49952.json create mode 100644 prover/prover_dal/.sqlx/query-94a75b05ecbab75d6ebf39cca029bfb838c787fc58d7536f9e9976e5e515431a.json diff --git a/core/lib/basic_types/src/prover_dal.rs b/core/lib/basic_types/src/prover_dal.rs index 7daea8a2205b..0e5963801082 100644 --- a/core/lib/basic_types/src/prover_dal.rs +++ b/core/lib/basic_types/src/prover_dal.rs @@ -5,7 +5,7 @@ use chrono::{DateTime, Duration, NaiveDateTime, NaiveTime, Utc}; use strum::{Display, EnumString}; use crate::{ - basic_fri_types::{AggregationRound, CircuitIdRoundTuple, Eip4844Blobs}, + basic_fri_types::{AggregationRound, Eip4844Blobs}, protocol_version::ProtocolVersionId, L1BatchNumber, }; @@ -290,6 +290,37 @@ pub struct LeafWitnessGeneratorJobInfo { pub picked_by: Option, } +pub struct NodeWitnessGeneratorJobInfo { + pub l1_batch_number: L1BatchNumber, + pub circuit_id: u32, + pub depth: u32, + pub status: WitnessJobStatus, + pub attempts: u32, + pub aggregations_url: Option, + pub processing_started_at: Option, + pub time_taken: Option, + pub error: Option, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, + pub number_of_dependent_jobs: Option, + pub protocol_version: Option, + pub picked_by: Option, +} + +pub struct SchedulerWitnessGeneratorJobInfo { + pub l1_batch_number: L1BatchNumber, + pub scheduler_partial_input_blob_url: String, + pub status: WitnessJobStatus, + pub processing_started_at: Option, + pub time_taken: Option, + pub error: Option, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, + pub attempts: u32, + pub protocol_version: Option, + pub picked_by: Option, +} + #[derive(Debug, EnumString, Display)] pub enum ProofCompressionJobStatus { #[strum(serialize = "queued")] diff --git a/prover/prover_cli/src/commands/status/batch.rs b/prover/prover_cli/src/commands/status/batch.rs index 456d2b6e29d9..bd1b768ba27f 100644 --- a/prover/prover_cli/src/commands/status/batch.rs +++ b/prover/prover_cli/src/commands/status/batch.rs @@ -62,6 +62,25 @@ async fn get_batches_data(batches: Vec) -> anyhow::Result( .unwrap_or_default() } +async fn get_proof_node_witness_generator_status_for_batch<'a>( + batch_number: L1BatchNumber, + conn: &mut Connection<'a, Prover>, +) -> TaskStatus { + conn.fri_witness_generator_dal() + .get_node_witness_generator_job_for_batch(batch_number) + .await + .map(|job| TaskStatus::from(job.status)) + .unwrap_or_default() +} + +async fn get_proof_scheduler_witness_generator_status_for_batch<'a>( + batch_number: L1BatchNumber, + conn: &mut Connection<'a, Prover>, +) -> TaskStatus { + conn.fri_witness_generator_dal() + .get_scheduler_witness_generator_job_for_batch(batch_number) + .await + .map(|job| TaskStatus::from(job.status)) + .unwrap_or_default() +} + async fn get_proof_compression_job_status_for_batch<'a>( batch_number: L1BatchNumber, conn: &mut Connection<'a, Prover>, diff --git a/prover/prover_cli/src/commands/status/utils.rs b/prover/prover_cli/src/commands/status/utils.rs index e2408d9c17a9..6a0ebc1b7fb9 100644 --- a/prover/prover_cli/src/commands/status/utils.rs +++ b/prover/prover_cli/src/commands/status/utils.rs @@ -27,7 +27,7 @@ pub struct BatchData { /// The recursion tip data. pub recursion_tip: Task, /// The scheduler data. - pub scheduler: Task, + pub scheduler_witness_generator: Task, /// The compressor data. pub compressor: Task, } @@ -45,7 +45,7 @@ impl Debug for BatchData { writeln!(f, "{:?}", self.leaf_witness_generator)?; writeln!(f, "{:?}", self.node_witness_generator)?; writeln!(f, "{:?}", self.recursion_tip)?; - writeln!(f, "{:?}", self.scheduler)?; + writeln!(f, "{:?}", self.scheduler_witness_generator)?; writeln!(f, "{:?}", self.compressor) } } @@ -82,7 +82,7 @@ impl Default for BatchData { prover_jobs_status: TaskStatus::default(), }, }, - scheduler: Task::Scheduler { + scheduler_witness_generator: Task::SchedulerWitnessGenerator { status: TaskStatus::default(), aggregation_round_info: AggregationRoundInfo { round: AggregationRound::Scheduler, @@ -185,7 +185,7 @@ pub enum Task { }, /// Represents the scheduler task and its status. #[strum(to_string = "Scheduler")] - Scheduler { + SchedulerWitnessGenerator { status: TaskStatus, aggregation_round_info: AggregationRoundInfo, }, @@ -201,7 +201,7 @@ impl Task { | Task::LeafWitnessGenerator { status, .. } | Task::NodeWitnessGenerator { status, .. } | Task::RecursionTip { status, .. } - | Task::Scheduler { status, .. } + | Task::SchedulerWitnessGenerator { status, .. } | Task::Compressor(status) => status.clone(), } } @@ -224,7 +224,7 @@ impl Task { aggregation_round_info, .. } - | Task::Scheduler { + | Task::SchedulerWitnessGenerator { aggregation_round_info, .. } => Some(aggregation_round_info.round), @@ -254,7 +254,7 @@ impl Task { status, aggregation_round_info, } - | Task::Scheduler { + | Task::SchedulerWitnessGenerator { status, aggregation_round_info, } => match status { diff --git a/prover/prover_dal/.sqlx/query-285d0ff850fa5c9af36564fcb14dd8547a1ad20492ec37c3c0be5639e5d49952.json b/prover/prover_dal/.sqlx/query-285d0ff850fa5c9af36564fcb14dd8547a1ad20492ec37c3c0be5639e5d49952.json new file mode 100644 index 000000000000..415b3e31c798 --- /dev/null +++ b/prover/prover_dal/.sqlx/query-285d0ff850fa5c9af36564fcb14dd8547a1ad20492ec37c3c0be5639e5d49952.json @@ -0,0 +1,82 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n *\n FROM\n scheduler_witness_jobs_fri\n WHERE\n l1_batch_number = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "l1_batch_number", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "scheduler_partial_input_blob_url", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "processing_started_at", + "type_info": "Timestamp" + }, + { + "ordinal": 4, + "name": "time_taken", + "type_info": "Time" + }, + { + "ordinal": 5, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamp" + }, + { + "ordinal": 7, + "name": "updated_at", + "type_info": "Timestamp" + }, + { + "ordinal": 8, + "name": "attempts", + "type_info": "Int2" + }, + { + "ordinal": 9, + "name": "protocol_version", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "picked_by", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + false, + false, + false, + true, + true + ] + }, + "hash": "285d0ff850fa5c9af36564fcb14dd8547a1ad20492ec37c3c0be5639e5d49952" +} diff --git a/prover/prover_dal/.sqlx/query-94a75b05ecbab75d6ebf39cca029bfb838c787fc58d7536f9e9976e5e515431a.json b/prover/prover_dal/.sqlx/query-94a75b05ecbab75d6ebf39cca029bfb838c787fc58d7536f9e9976e5e515431a.json new file mode 100644 index 000000000000..896f10a4ca3a --- /dev/null +++ b/prover/prover_dal/.sqlx/query-94a75b05ecbab75d6ebf39cca029bfb838c787fc58d7536f9e9976e5e515431a.json @@ -0,0 +1,106 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n *\n FROM\n node_aggregation_witness_jobs_fri\n WHERE\n l1_batch_number = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "l1_batch_number", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "circuit_id", + "type_info": "Int2" + }, + { + "ordinal": 3, + "name": "depth", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "attempts", + "type_info": "Int2" + }, + { + "ordinal": 6, + "name": "aggregations_url", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "processing_started_at", + "type_info": "Timestamp" + }, + { + "ordinal": 8, + "name": "time_taken", + "type_info": "Time" + }, + { + "ordinal": 9, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "created_at", + "type_info": "Timestamp" + }, + { + "ordinal": 11, + "name": "updated_at", + "type_info": "Timestamp" + }, + { + "ordinal": 12, + "name": "number_of_dependent_jobs", + "type_info": "Int4" + }, + { + "ordinal": 13, + "name": "protocol_version", + "type_info": "Int4" + }, + { + "ordinal": 14, + "name": "picked_by", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false, + false, + true, + true, + true + ] + }, + "hash": "94a75b05ecbab75d6ebf39cca029bfb838c787fc58d7536f9e9976e5e515431a" +} diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index 30d9bb21c977..62464a49d7af 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -7,7 +7,8 @@ use zksync_basic_types::{ protocol_version::ProtocolVersionId, prover_dal::{ BasicWitnessGeneratorJobInfo, JobCountStatistics, LeafAggregationJobMetadata, - LeafWitnessGeneratorJobInfo, NodeAggregationJobMetadata, StuckJobs, WitnessJobStatus, + LeafWitnessGeneratorJobInfo, NodeAggregationJobMetadata, NodeWitnessGeneratorJobInfo, + SchedulerWitnessGeneratorJobInfo, StuckJobs, WitnessJobStatus, }, L1BatchNumber, }; @@ -1223,4 +1224,73 @@ impl FriWitnessGeneratorDal<'_, '_> { number_of_basic_circuits: row.number_of_basic_circuits, }) } + + pub async fn get_node_witness_generator_job_for_batch( + &mut self, + l1_batch_number: L1BatchNumber, + ) -> Option { + sqlx::query!( + r#" + SELECT + * + FROM + node_aggregation_witness_jobs_fri + WHERE + l1_batch_number = $1 + "#, + i64::from(l1_batch_number.0) + ) + .fetch_optional(self.storage.conn()) + .await + .unwrap() + .map(|row| NodeWitnessGeneratorJobInfo { + l1_batch_number: l1_batch_number, + circuit_id: row.circuit_id as u32, + depth: row.depth as u32, + status: WitnessJobStatus::from_str(&row.status).unwrap(), + attempts: row.attempts as u32, + aggregations_url: row.aggregations_url, + processing_started_at: row.processing_started_at, + time_taken: row.time_taken, + error: row.error, + created_at: row.created_at, + updated_at: row.updated_at, + number_of_dependent_jobs: row.number_of_dependent_jobs, + protocol_version: row.protocol_version, + picked_by: row.picked_by, + }) + } + + pub async fn get_scheduler_witness_generator_job_for_batch( + &mut self, + l1_batch_number: L1BatchNumber, + ) -> Option { + sqlx::query!( + r#" + SELECT + * + FROM + scheduler_witness_jobs_fri + WHERE + l1_batch_number = $1 + "#, + i64::from(l1_batch_number.0) + ) + .fetch_optional(self.storage.conn()) + .await + .unwrap() + .map(|row| SchedulerWitnessGeneratorJobInfo { + l1_batch_number: l1_batch_number, + scheduler_partial_input_blob_url: row.scheduler_partial_input_blob_url, + status: WitnessJobStatus::from_str(&row.status).unwrap(), + processing_started_at: row.processing_started_at, + time_taken: row.time_taken, + error: row.error, + created_at: row.created_at, + updated_at: row.updated_at, + attempts: row.attempts as u32, + protocol_version: row.protocol_version, + picked_by: row.picked_by, + }) + } } From cf069b9b2def6e47e019b3dc7beeeae08e7df26c Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Tue, 30 Apr 2024 15:32:54 -0300 Subject: [PATCH 4/9] fix witnes jobs status --- Cargo.lock | 1 + core/lib/basic_types/Cargo.toml | 1 + core/lib/basic_types/src/prover_dal.rs | 31 +++++++++++-- prover/Cargo.lock | 1 + .../prover_cli/src/commands/status/batch.rs | 45 ++++++++++++------- .../prover_cli/src/commands/status/utils.rs | 25 +++++++++++ .../src/fri_witness_generator_dal.rs | 44 ++++++++++-------- 7 files changed, 110 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1d3481d0009..327543d495ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8124,6 +8124,7 @@ dependencies = [ "num_enum 0.7.2", "serde", "serde_json", + "sqlx", "strum", "web3", ] diff --git a/core/lib/basic_types/Cargo.toml b/core/lib/basic_types/Cargo.toml index 13ce9cc357d7..ef6733b0ef19 100644 --- a/core/lib/basic_types/Cargo.toml +++ b/core/lib/basic_types/Cargo.toml @@ -20,3 +20,4 @@ serde_json.workspace = true chrono.workspace = true strum = { workspace = true, features = ["derive"] } num_enum.workspace = true +sqlx = { workspace = true, feature= ["derive"]} diff --git a/core/lib/basic_types/src/prover_dal.rs b/core/lib/basic_types/src/prover_dal.rs index 0e5963801082..d81f45e31231 100644 --- a/core/lib/basic_types/src/prover_dal.rs +++ b/core/lib/basic_types/src/prover_dal.rs @@ -124,7 +124,7 @@ pub struct ProverJobStatusInProgress { pub started_at: DateTime, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct WitnessJobStatusSuccessful { pub started_at: DateTime, pub time_taken: Duration, @@ -139,7 +139,7 @@ impl Default for WitnessJobStatusSuccessful { } } -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct WitnessJobStatusFailed { pub started_at: DateTime, pub error: String, @@ -161,7 +161,7 @@ pub enum ProverJobStatus { Ignored, } -#[derive(Debug, strum::Display, strum::EnumString, strum::AsRefStr)] +#[derive(Debug, Clone, strum::Display, strum::EnumString, strum::AsRefStr)] pub enum WitnessJobStatus { #[strum(serialize = "failed")] Failed(WitnessJobStatusFailed), @@ -273,7 +273,31 @@ pub struct BasicWitnessGeneratorJobInfo { pub eip_4844_blobs: Option, } +// impl FromRow<'_, R> for BasicWitnessGeneratorJobInfo { +// fn from_row(row: &R) -> sqlx::Result { +// let l1_batch_number: i32 = row.try_get("l1_batch_number")?; +// let attempts: i32 = row.try_get("attempts")?; +// Ok(Self { +// l1_batch_number: L1BatchNumber(l1_batch_number as u32), +// merkle_tree_paths_blob_url: row.try_get("merkle_tree_paths_blob_url")?, +// attempts: attempts as u32, +// status: WitnessJobStatus::from_str(row.try_get("status")?).unwrap(), +// error: row.try_get("error")?, +// created_at: row.try_get("created_at")?, +// updated_at: row.try_get("updated_at")?, +// processing_started_at: row.try_get("processing_started_at")?, +// time_taken: row.try_get("time_taken")?, +// is_blob_cleaned: row.try_get("is_blob_cleaned")?, +// protocol_version: row.try_get("protocol_version")?, +// picked_by: row.try_get("picked_by")?, +// eip_4844_blobs: +// Some(Eip4844Blobs::from(row.try_get::, &str>("eip_4844_blobs")?)) +// }) +// } +// } + pub struct LeafWitnessGeneratorJobInfo { + pub id: u32, pub l1_batch_number: L1BatchNumber, pub circuit_id: u32, pub closed_form_inputs_blob_url: Option, @@ -291,6 +315,7 @@ pub struct LeafWitnessGeneratorJobInfo { } pub struct NodeWitnessGeneratorJobInfo { + pub id: u32, pub l1_batch_number: L1BatchNumber, pub circuit_id: u32, pub depth: u32, diff --git a/prover/Cargo.lock b/prover/Cargo.lock index e690f7ddf1fe..3edd696f5013 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -7516,6 +7516,7 @@ dependencies = [ "num_enum 0.7.2", "serde", "serde_json", + "sqlx", "strum", "web3", ] diff --git a/prover/prover_cli/src/commands/status/batch.rs b/prover/prover_cli/src/commands/status/batch.rs index bd1b768ba27f..0b11151e53e5 100644 --- a/prover/prover_cli/src/commands/status/batch.rs +++ b/prover/prover_cli/src/commands/status/batch.rs @@ -1,7 +1,9 @@ use anyhow::{ensure, Context as _}; use clap::Args as ClapArgs; use prover_dal::{Connection, ConnectionPool, Prover, ProverDal}; -use zksync_types::{basic_fri_types::AggregationRound, L1BatchNumber}; +use zksync_types::{ + basic_fri_types::AggregationRound, prover_dal::WitnessJobStatus, L1BatchNumber, +}; use super::utils::{AggregationRoundInfo, BatchData, Task, TaskStatus}; use crate::commands::status::utils::postgres_config; @@ -62,21 +64,21 @@ async fn get_batches_data(batches: Vec) -> anyhow::Result( batch_number: L1BatchNumber, conn: &mut Connection<'a, Prover>, ) -> TaskStatus { - conn.fri_witness_generator_dal() - .get_leaf_witness_generator_job_for_batch(batch_number) + let status_vec: Vec = conn + .fri_witness_generator_dal() + .get_leaf_witness_generator_jobs_for_batch(batch_number) .await - .map(|job| TaskStatus::from(job.status)) - .unwrap_or_default() + .iter() + .map(|s| s.status.clone()) + .collect(); + TaskStatus::from(status_vec) } async fn get_proof_node_witness_generator_status_for_batch<'a>( batch_number: L1BatchNumber, conn: &mut Connection<'a, Prover>, ) -> TaskStatus { - conn.fri_witness_generator_dal() - .get_node_witness_generator_job_for_batch(batch_number) + let status_vec: Vec = conn + .fri_witness_generator_dal() + .get_node_witness_generator_jobs_for_batch(batch_number) .await - .map(|job| TaskStatus::from(job.status)) - .unwrap_or_default() + .iter() + .map(|s| s.status.clone()) + .collect(); + TaskStatus::from(status_vec) } async fn get_proof_scheduler_witness_generator_status_for_batch<'a>( batch_number: L1BatchNumber, conn: &mut Connection<'a, Prover>, ) -> TaskStatus { - conn.fri_witness_generator_dal() - .get_scheduler_witness_generator_job_for_batch(batch_number) + let status_vec: Vec = conn + .fri_witness_generator_dal() + .get_scheduler_witness_generator_jobs_for_batch(batch_number) .await - .map(|job| TaskStatus::from(job.status)) - .unwrap_or_default() + .iter() + .map(|s| s.status.clone()) + .collect(); + TaskStatus::from(status_vec) } async fn get_proof_compression_job_status_for_batch<'a>( diff --git a/prover/prover_cli/src/commands/status/utils.rs b/prover/prover_cli/src/commands/status/utils.rs index 6a0ebc1b7fb9..73daffbba61a 100644 --- a/prover/prover_cli/src/commands/status/utils.rs +++ b/prover/prover_cli/src/commands/status/utils.rs @@ -157,6 +157,31 @@ impl From for TaskStatus { } } +impl From> for TaskStatus { + fn from(status_vector: Vec) -> Self { + if status_vector.is_empty() { + TaskStatus::JobsNotFound + } else if status_vector + .iter() + .all(|job| matches!(job, WitnessJobStatus::Queued)) + { + TaskStatus::Queued + } else if status_vector + .iter() + .all(|job| matches!(job, WitnessJobStatus::WaitingForProofs)) + { + TaskStatus::WaitingForProofs + } else if status_vector + .iter() + .all(|job| matches!(job, WitnessJobStatus::InProgress)) + { + TaskStatus::Successful + } else { + TaskStatus::InProgress + } + } +} + #[derive(EnumString, Clone, Display)] pub enum Task { /// Represents the basic witness generator task and its status. diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index 62464a49d7af..57f14c0742dc 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -1189,10 +1189,10 @@ impl FriWitnessGeneratorDal<'_, '_> { }) } - pub async fn get_leaf_witness_generator_job_for_batch( + pub async fn get_leaf_witness_generator_jobs_for_batch( &mut self, l1_batch_number: L1BatchNumber, - ) -> Option { + ) -> Vec { sqlx::query!( r#" SELECT @@ -1204,31 +1204,34 @@ impl FriWitnessGeneratorDal<'_, '_> { "#, i64::from(l1_batch_number.0) ) - .fetch_optional(self.storage.conn()) + .fetch_all(self.storage.conn()) .await .unwrap() + .iter() .map(|row| LeafWitnessGeneratorJobInfo { + id: row.id as u32, l1_batch_number: l1_batch_number, circuit_id: row.circuit_id as u32, - closed_form_inputs_blob_url: row.closed_form_inputs_blob_url, + closed_form_inputs_blob_url: row.closed_form_inputs_blob_url.clone(), attempts: row.attempts as u32, status: WitnessJobStatus::from_str(&row.status).unwrap(), - error: row.error, + error: row.error.clone(), created_at: row.created_at, updated_at: row.updated_at, processing_started_at: row.processing_started_at, time_taken: row.time_taken, is_blob_cleaned: row.is_blob_cleaned, protocol_version: row.protocol_version, - picked_by: row.picked_by, + picked_by: row.picked_by.clone(), number_of_basic_circuits: row.number_of_basic_circuits, }) + .collect() } - pub async fn get_node_witness_generator_job_for_batch( + pub async fn get_node_witness_generator_jobs_for_batch( &mut self, l1_batch_number: L1BatchNumber, - ) -> Option { + ) -> Vec { sqlx::query!( r#" SELECT @@ -1240,31 +1243,34 @@ impl FriWitnessGeneratorDal<'_, '_> { "#, i64::from(l1_batch_number.0) ) - .fetch_optional(self.storage.conn()) + .fetch_all(self.storage.conn()) .await .unwrap() + .iter() .map(|row| NodeWitnessGeneratorJobInfo { + id: row.id as u32, l1_batch_number: l1_batch_number, circuit_id: row.circuit_id as u32, depth: row.depth as u32, status: WitnessJobStatus::from_str(&row.status).unwrap(), attempts: row.attempts as u32, - aggregations_url: row.aggregations_url, + aggregations_url: row.aggregations_url.clone(), processing_started_at: row.processing_started_at, time_taken: row.time_taken, - error: row.error, + error: row.error.clone(), created_at: row.created_at, updated_at: row.updated_at, number_of_dependent_jobs: row.number_of_dependent_jobs, protocol_version: row.protocol_version, - picked_by: row.picked_by, + picked_by: row.picked_by.clone(), }) + .collect() } - pub async fn get_scheduler_witness_generator_job_for_batch( + pub async fn get_scheduler_witness_generator_jobs_for_batch( &mut self, l1_batch_number: L1BatchNumber, - ) -> Option { + ) -> Vec { sqlx::query!( r#" SELECT @@ -1276,21 +1282,23 @@ impl FriWitnessGeneratorDal<'_, '_> { "#, i64::from(l1_batch_number.0) ) - .fetch_optional(self.storage.conn()) + .fetch_all(self.storage.conn()) .await .unwrap() + .iter() .map(|row| SchedulerWitnessGeneratorJobInfo { l1_batch_number: l1_batch_number, - scheduler_partial_input_blob_url: row.scheduler_partial_input_blob_url, + scheduler_partial_input_blob_url: row.scheduler_partial_input_blob_url.clone(), status: WitnessJobStatus::from_str(&row.status).unwrap(), processing_started_at: row.processing_started_at, time_taken: row.time_taken, - error: row.error, + error: row.error.clone(), created_at: row.created_at, updated_at: row.updated_at, attempts: row.attempts as u32, protocol_version: row.protocol_version, - picked_by: row.picked_by, + picked_by: row.picked_by.clone(), }) + .collect() } } From 6ad55660dd55ee75e1985c0ffa6aa05e3a22310c Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Tue, 30 Apr 2024 15:35:15 -0300 Subject: [PATCH 5/9] rm comments --- core/lib/basic_types/src/prover_dal.rs | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/core/lib/basic_types/src/prover_dal.rs b/core/lib/basic_types/src/prover_dal.rs index d81f45e31231..dc106da18a7b 100644 --- a/core/lib/basic_types/src/prover_dal.rs +++ b/core/lib/basic_types/src/prover_dal.rs @@ -273,29 +273,6 @@ pub struct BasicWitnessGeneratorJobInfo { pub eip_4844_blobs: Option, } -// impl FromRow<'_, R> for BasicWitnessGeneratorJobInfo { -// fn from_row(row: &R) -> sqlx::Result { -// let l1_batch_number: i32 = row.try_get("l1_batch_number")?; -// let attempts: i32 = row.try_get("attempts")?; -// Ok(Self { -// l1_batch_number: L1BatchNumber(l1_batch_number as u32), -// merkle_tree_paths_blob_url: row.try_get("merkle_tree_paths_blob_url")?, -// attempts: attempts as u32, -// status: WitnessJobStatus::from_str(row.try_get("status")?).unwrap(), -// error: row.try_get("error")?, -// created_at: row.try_get("created_at")?, -// updated_at: row.try_get("updated_at")?, -// processing_started_at: row.try_get("processing_started_at")?, -// time_taken: row.try_get("time_taken")?, -// is_blob_cleaned: row.try_get("is_blob_cleaned")?, -// protocol_version: row.try_get("protocol_version")?, -// picked_by: row.try_get("picked_by")?, -// eip_4844_blobs: -// Some(Eip4844Blobs::from(row.try_get::, &str>("eip_4844_blobs")?)) -// }) -// } -// } - pub struct LeafWitnessGeneratorJobInfo { pub id: u32, pub l1_batch_number: L1BatchNumber, From a3f1938a9a0aef5829f81ad14ed7a8758684f701 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Tue, 30 Apr 2024 15:56:08 -0300 Subject: [PATCH 6/9] Fix & refactors --- .../prover_cli/src/commands/status/batch.rs | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/prover/prover_cli/src/commands/status/batch.rs b/prover/prover_cli/src/commands/status/batch.rs index 0b11151e53e5..661c2eef65c6 100644 --- a/prover/prover_cli/src/commands/status/batch.rs +++ b/prover/prover_cli/src/commands/status/batch.rs @@ -40,7 +40,10 @@ async fn get_batches_data(batches: Vec) -> anyhow::Result) -> anyhow::Result) -> anyhow::Result) -> anyhow::Result) -> anyhow::Result) -> anyhow::Result( +async fn get_aggregation_round_info_for_batch<'a>( batch_number: L1BatchNumber, aggregation_round: AggregationRound, conn: &mut Connection<'a, Prover>, @@ -106,7 +109,7 @@ async fn get_prover_jobs_data_for_batch<'a>( .into(); AggregationRoundInfo { - round: AggregationRound::BasicCircuits, + round: aggregation_round, prover_jobs_status: status, } } @@ -126,42 +129,39 @@ async fn get_proof_leaf_witness_generator_status_for_batch<'a>( batch_number: L1BatchNumber, conn: &mut Connection<'a, Prover>, ) -> TaskStatus { - let status_vec: Vec = conn - .fri_witness_generator_dal() + conn.fri_witness_generator_dal() .get_leaf_witness_generator_jobs_for_batch(batch_number) .await .iter() .map(|s| s.status.clone()) - .collect(); - TaskStatus::from(status_vec) + .collect::>() + .into() } async fn get_proof_node_witness_generator_status_for_batch<'a>( batch_number: L1BatchNumber, conn: &mut Connection<'a, Prover>, ) -> TaskStatus { - let status_vec: Vec = conn - .fri_witness_generator_dal() + conn.fri_witness_generator_dal() .get_node_witness_generator_jobs_for_batch(batch_number) .await .iter() .map(|s| s.status.clone()) - .collect(); - TaskStatus::from(status_vec) + .collect::>() + .into() } async fn get_proof_scheduler_witness_generator_status_for_batch<'a>( batch_number: L1BatchNumber, conn: &mut Connection<'a, Prover>, ) -> TaskStatus { - let status_vec: Vec = conn - .fri_witness_generator_dal() + conn.fri_witness_generator_dal() .get_scheduler_witness_generator_jobs_for_batch(batch_number) .await .iter() .map(|s| s.status.clone()) - .collect(); - TaskStatus::from(status_vec) + .collect::>() + .into() } async fn get_proof_compression_job_status_for_batch<'a>( From 112ae934e763215775f33a23a61b15657da9c2bc Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Tue, 30 Apr 2024 16:12:15 -0300 Subject: [PATCH 7/9] fix import --- prover/prover_fri_gateway/src/proof_submitter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prover/prover_fri_gateway/src/proof_submitter.rs b/prover/prover_fri_gateway/src/proof_submitter.rs index 1c5850d31a0b..025d79e2f8c3 100644 --- a/prover/prover_fri_gateway/src/proof_submitter.rs +++ b/prover/prover_fri_gateway/src/proof_submitter.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; -use prover_dal::{fri_proof_compressor_dal::ProofCompressionJobStatus, ProverDal}; +use prover_dal::ProverDal; use zksync_prover_interface::api::{SubmitProofRequest, SubmitProofResponse}; -use zksync_types::L1BatchNumber; +use zksync_types::{prover_dal::ProofCompressionJobStatus, L1BatchNumber}; use crate::api_data_fetcher::{PeriodicApi, PeriodicApiStruct}; From e1854fa6e7ce89a3e9667be0afdc8afbf5b7fca4 Mon Sep 17 00:00:00 2001 From: ilitteri Date: Tue, 30 Apr 2024 16:12:49 -0300 Subject: [PATCH 8/9] zk lint rust --- prover/prover_dal/src/fri_witness_generator_dal.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index a65e88b7c325..c336084e7baa 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -1210,7 +1210,7 @@ impl FriWitnessGeneratorDal<'_, '_> { .iter() .map(|row| LeafWitnessGeneratorJobInfo { id: row.id as u32, - l1_batch_number: l1_batch_number, + l1_batch_number, circuit_id: row.circuit_id as u32, closed_form_inputs_blob_url: row.closed_form_inputs_blob_url.clone(), attempts: row.attempts as u32, @@ -1249,7 +1249,7 @@ impl FriWitnessGeneratorDal<'_, '_> { .iter() .map(|row| NodeWitnessGeneratorJobInfo { id: row.id as u32, - l1_batch_number: l1_batch_number, + l1_batch_number, circuit_id: row.circuit_id as u32, depth: row.depth as u32, status: WitnessJobStatus::from_str(&row.status).unwrap(), @@ -1287,7 +1287,7 @@ impl FriWitnessGeneratorDal<'_, '_> { .unwrap() .iter() .map(|row| SchedulerWitnessGeneratorJobInfo { - l1_batch_number: l1_batch_number, + l1_batch_number, scheduler_partial_input_blob_url: row.scheduler_partial_input_blob_url.clone(), status: WitnessJobStatus::from_str(&row.status).unwrap(), processing_started_at: row.processing_started_at, From fa26c26aee21eca4fae8e13fedbb2b5fe268cefb Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Tue, 30 Apr 2024 17:26:33 -0300 Subject: [PATCH 9/9] fix wg task types --- prover/prover_cli/src/commands/status/batch.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prover/prover_cli/src/commands/status/batch.rs b/prover/prover_cli/src/commands/status/batch.rs index 661c2eef65c6..c29d25dc62e5 100644 --- a/prover/prover_cli/src/commands/status/batch.rs +++ b/prover/prover_cli/src/commands/status/batch.rs @@ -67,7 +67,7 @@ async fn get_batches_data(batches: Vec) -> anyhow::Result) -> anyhow::Result