diff --git a/core/lib/db_connection/src/utils.rs b/core/lib/db_connection/src/utils.rs index 80cf0a5cbb35..02840afaf1d1 100644 --- a/core/lib/db_connection/src/utils.rs +++ b/core/lib/db_connection/src/utils.rs @@ -20,7 +20,13 @@ pub fn duration_to_naive_time(duration: Duration) -> NaiveTime { (total_seconds / 60) % 60, total_seconds % 60, ) - .unwrap() + .expect("failed to convert Duration to NaiveTime") +} + +pub fn duration_to_naive_time_ms(duration: Duration) -> NaiveTime { + let total_ms = duration.as_millis(); + let ms = (total_ms % 1_000) as u64; + duration_to_naive_time(duration) + Duration::from_millis(ms) } pub const fn pg_interval_from_duration(processing_timeout: Duration) -> PgInterval { diff --git a/prover/crates/bin/prover_fri/src/utils.rs b/prover/crates/bin/prover_fri/src/utils.rs index 15a2a6c18bb2..e53e0810d693 100644 --- a/prover/crates/bin/prover_fri/src/utils.rs +++ b/prover/crates/bin/prover_fri/src/utils.rs @@ -67,10 +67,11 @@ pub async fn save_proof( connection: &mut Connection<'_, Prover>, protocol_version: ProtocolSemanticVersion, ) { + let time_taken = started_at.elapsed(); tracing::info!( "Successfully proven job: {}, total time taken: {:?}", job_id, - started_at.elapsed() + time_taken, ); let proof = artifacts.proof_wrapper; @@ -101,7 +102,7 @@ pub async fn save_proof( let mut transaction = connection.start_transaction().await.unwrap(); transaction .fri_prover_jobs_dal() - .save_proof(job_id, started_at.elapsed(), &blob_url) + .save_proof(job_id, time_taken, &blob_url) .await; if is_scheduler_proof { transaction diff --git a/prover/crates/bin/witness_vector_generator/src/generator.rs b/prover/crates/bin/witness_vector_generator/src/generator.rs index 5574f0f1578d..5ab5537358ff 100644 --- a/prover/crates/bin/witness_vector_generator/src/generator.rs +++ b/prover/crates/bin/witness_vector_generator/src/generator.rs @@ -145,15 +145,24 @@ impl JobProcessor for WitnessVectorGenerator { METRICS.gpu_witness_vector_generation_time[&circuit_type].observe(started_at.elapsed()); + let wvg_time_taken = started_at.elapsed(); tracing::info!( "Finished witness vector generation for job: {job_id} in zone: {:?} took: {:?}", self.zone, - started_at.elapsed() + wvg_time_taken ); let serialized: Vec = bincode::serialize(&artifacts).expect("Failed to serialize witness vector artifacts"); + self.pool + .connection() + .await + .unwrap() + .fri_prover_jobs_dal() + .save_wvg_time_taken(job_id, wvg_time_taken) + .await; + let now = Instant::now(); let mut attempts = 0; diff --git a/prover/crates/lib/prover_dal/.sqlx/query-ada952c0172767d1ba41ab9b812f37e379c9a7ff62fa84eeaa8df7749803466c.json b/prover/crates/lib/prover_dal/.sqlx/query-ada952c0172767d1ba41ab9b812f37e379c9a7ff62fa84eeaa8df7749803466c.json new file mode 100644 index 000000000000..3ced718c7500 --- /dev/null +++ b/prover/crates/lib/prover_dal/.sqlx/query-ada952c0172767d1ba41ab9b812f37e379c9a7ff62fa84eeaa8df7749803466c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE prover_jobs_fri\n SET\n updated_at = NOW(),\n wvg_time_taken = $1\n WHERE\n id = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Time", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "ada952c0172767d1ba41ab9b812f37e379c9a7ff62fa84eeaa8df7749803466c" +} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-c2c140d136df5303d7b3a66ccd0d34a5baece02812f8c950fc84d37eeebd33a4.json b/prover/crates/lib/prover_dal/.sqlx/query-c2c140d136df5303d7b3a66ccd0d34a5baece02812f8c950fc84d37eeebd33a4.json index 25a49e191f6e..90d8dcd47f8f 100644 --- a/prover/crates/lib/prover_dal/.sqlx/query-c2c140d136df5303d7b3a66ccd0d34a5baece02812f8c950fc84d37eeebd33a4.json +++ b/prover/crates/lib/prover_dal/.sqlx/query-c2c140d136df5303d7b3a66ccd0d34a5baece02812f8c950fc84d37eeebd33a4.json @@ -97,6 +97,11 @@ "ordinal": 18, "name": "protocol_version_patch", "type_info": "Int4" + }, + { + "ordinal": 20, + "name": "wvg_time_taken", + "type_info": "Time" } ], "parameters": { @@ -124,7 +129,8 @@ true, true, true, - false + false, + true ] }, "hash": "c2c140d136df5303d7b3a66ccd0d34a5baece02812f8c950fc84d37eeebd33a4" diff --git a/prover/crates/lib/prover_dal/migrations/20240708090440_add_wvg_time_taken.down.sql b/prover/crates/lib/prover_dal/migrations/20240708090440_add_wvg_time_taken.down.sql new file mode 100644 index 000000000000..55c7b1bd8f84 --- /dev/null +++ b/prover/crates/lib/prover_dal/migrations/20240708090440_add_wvg_time_taken.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE prover_jobs_fri + DROP COLUMN IF EXISTS wvg_time_taken; + +ALTER TABLE prover_jobs_fri_archive + DROP COLUMN IF EXISTS wvg_time_taken; diff --git a/prover/crates/lib/prover_dal/migrations/20240708090440_add_wvg_time_taken.up.sql b/prover/crates/lib/prover_dal/migrations/20240708090440_add_wvg_time_taken.up.sql new file mode 100644 index 000000000000..0530f66873b9 --- /dev/null +++ b/prover/crates/lib/prover_dal/migrations/20240708090440_add_wvg_time_taken.up.sql @@ -0,0 +1,5 @@ +ALTER TABLE prover_jobs_fri + ADD COLUMN IF NOT EXISTS wvg_time_taken TIME; + +ALTER TABLE prover_jobs_fri_archive + ADD COLUMN IF NOT EXISTS wvg_time_taken TIME; diff --git a/prover/crates/lib/prover_dal/src/fri_prover_dal.rs b/prover/crates/lib/prover_dal/src/fri_prover_dal.rs index f6efc6afa6ad..37f65bf68718 100644 --- a/prover/crates/lib/prover_dal/src/fri_prover_dal.rs +++ b/prover/crates/lib/prover_dal/src/fri_prover_dal.rs @@ -13,7 +13,7 @@ use zksync_db_connection::{ connection::Connection, instrument::InstrumentExt, metrics::MethodLatency, }; -use crate::{duration_to_naive_time, pg_interval_from_duration, Prover}; +use crate::{duration_to_naive_time_ms, pg_interval_from_duration, Prover}; #[derive(Debug)] pub struct FriProverDal<'a, 'c> { @@ -265,7 +265,7 @@ impl FriProverDal<'_, '_> { prover_jobs_fri.depth, prover_jobs_fri.is_node_final_proof "#, - duration_to_naive_time(time_taken), + duration_to_naive_time_ms(time_taken), blob_url, i64::from(id) ) @@ -288,6 +288,24 @@ impl FriProverDal<'_, '_> { .unwrap() } + pub async fn save_wvg_time_taken(&mut self, id: u32, wvg_time_taken: Duration) { + sqlx::query!( + r#" + UPDATE prover_jobs_fri + SET + updated_at = NOW(), + wvg_time_taken = $1 + WHERE + id = $2 + "#, + duration_to_naive_time_ms(wvg_time_taken), + i64::from(id) + ) + .execute(self.storage.conn()) + .await + .unwrap(); + } + pub async fn requeue_stuck_jobs( &mut self, processing_timeout: Duration, diff --git a/prover/crates/lib/prover_dal/src/lib.rs b/prover/crates/lib/prover_dal/src/lib.rs index bb552b899e90..3df1e4098e9b 100644 --- a/prover/crates/lib/prover_dal/src/lib.rs +++ b/prover/crates/lib/prover_dal/src/lib.rs @@ -2,7 +2,7 @@ use zksync_db_connection::connection::DbMarker; pub use zksync_db_connection::{ connection::Connection, connection_pool::ConnectionPool, - utils::{duration_to_naive_time, pg_interval_from_duration}, + utils::{duration_to_naive_time, duration_to_naive_time_ms, pg_interval_from_duration}, }; use crate::{