From b84fd00f8e86c9c6c49b0e53957a115e96053091 Mon Sep 17 00:00:00 2001 From: Thomas Knauth Date: Thu, 8 Aug 2024 20:15:07 +0000 Subject: [PATCH 1/4] feat: remove TEE from tree TEE input producer jobs are now created by monitoring the db instead of hooking directly into the tree logic. --- .../src/tee_verifier_input_producer_dal.rs | 48 +++++++++++++++++++ core/node/metadata_calculator/src/updater.rs | 4 -- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/core/lib/dal/src/tee_verifier_input_producer_dal.rs b/core/lib/dal/src/tee_verifier_input_producer_dal.rs index 4adee62e7aa6..86ef6cd0f50f 100644 --- a/core/lib/dal/src/tee_verifier_input_producer_dal.rs +++ b/core/lib/dal/src/tee_verifier_input_producer_dal.rs @@ -72,9 +72,57 @@ impl TeeVerifierInputProducerDal<'_, '_> { Ok(()) } + // Returns inclusive range, i.e. [low,high] + pub async fn get_new_l1_batches( + &mut self, + ) -> DalResult> { + // COALESCE(1, ...) covers the case where the table is initially empty. NB: We start from batch 1, since genesis batch 0 has no proof. + let row_low = sqlx::query!( + r#" + SELECT + COALESCE(1, MAX(l1_batch_number) + 1) AS "number" + FROM + tee_verifier_input_producer_jobs + "# + ) + .instrument("get_latest_tee_job_l1_batch_number") + .report_latency() + .fetch_one(self.storage) + .await?; + + // Since we depend on Merkle paths, we use the proof_generation_details table to inform us of newly available to-be-proven batches. + let row_high = sqlx::query!( + r#" + SELECT + MAX(l1_batch_number) AS "number" + FROM + proof_generation_details + "# + ) + .instrument("get_sealed_l1_batch_number") + .report_latency() + .fetch_one(self.storage) + .await?; + + match (row_low.number, row_high.number) { + (Some(low), Some(high)) => Ok(Some(( + L1BatchNumber(low as u32), + L1BatchNumber(high as u32), + ))), + _ => Ok(None), + } + } + pub async fn get_next_tee_verifier_input_producer_job( &mut self, ) -> DalResult> { + if let Some((low, high)) = self.get_new_l1_batches().await? { + for i in low.0..=high.0 { + self.create_tee_verifier_input_producer_job(L1BatchNumber(i)) + .await? + } + } + let l1_batch_number = sqlx::query!( r#" UPDATE tee_verifier_input_producer_jobs diff --git a/core/node/metadata_calculator/src/updater.rs b/core/node/metadata_calculator/src/updater.rs index e2acf62dea8a..17fd5d900eab 100644 --- a/core/node/metadata_calculator/src/updater.rs +++ b/core/node/metadata_calculator/src/updater.rs @@ -152,10 +152,6 @@ impl TreeUpdater { // right away without having to implement dedicated code. if let Some(object_key) = &object_key { - storage - .tee_verifier_input_producer_dal() - .create_tee_verifier_input_producer_job(l1_batch_number) - .await?; // Save the proof generation details to Postgres storage .proof_generation_dal() From e6d702d90c71c75bf77e2f4bb6134c436c583329 Mon Sep 17 00:00:00 2001 From: Thomas Knauth Date: Fri, 9 Aug 2024 11:17:09 +0000 Subject: [PATCH 2/4] fixup! feat: remove TEE from tree --- .../src/tee_verifier_input_producer_dal.rs | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/core/lib/dal/src/tee_verifier_input_producer_dal.rs b/core/lib/dal/src/tee_verifier_input_producer_dal.rs index 86ef6cd0f50f..56950f5cfdb5 100644 --- a/core/lib/dal/src/tee_verifier_input_producer_dal.rs +++ b/core/lib/dal/src/tee_verifier_input_producer_dal.rs @@ -76,41 +76,42 @@ impl TeeVerifierInputProducerDal<'_, '_> { pub async fn get_new_l1_batches( &mut self, ) -> DalResult> { - // COALESCE(1, ...) covers the case where the table is initially empty. NB: We start from batch 1, since genesis batch 0 has no proof. - let row_low = sqlx::query!( + // Since we depend on Merkle paths, we use the proof_generation_details table to inform us of newly available to-be-proven batches. + let row_high = sqlx::query!( r#" SELECT - COALESCE(1, MAX(l1_batch_number) + 1) AS "number" + MAX(l1_batch_number) AS "number" FROM - tee_verifier_input_producer_jobs + proof_generation_details "# ) - .instrument("get_latest_tee_job_l1_batch_number") + .instrument("get_sealed_l1_batch_number") .report_latency() .fetch_one(self.storage) - .await?; + .await? + .number + .unwrap(); - // Since we depend on Merkle paths, we use the proof_generation_details table to inform us of newly available to-be-proven batches. - let row_high = sqlx::query!( + let row_low = sqlx::query!( r#" SELECT - MAX(l1_batch_number) AS "number" + MAX(l1_batch_number) + 1 AS "number" FROM - proof_generation_details + tee_verifier_input_producer_jobs "# ) - .instrument("get_sealed_l1_batch_number") + .instrument("get_latest_tee_job_l1_batch_number") .report_latency() .fetch_one(self.storage) - .await?; + .await? + .number + // If the table is empty, we start with the most recent batch. + .unwrap_or(row_high); - match (row_low.number, row_high.number) { - (Some(low), Some(high)) => Ok(Some(( - L1BatchNumber(low as u32), - L1BatchNumber(high as u32), - ))), - _ => Ok(None), - } + Ok(Some(( + L1BatchNumber(row_low as u32), + L1BatchNumber(row_high as u32), + ))) } pub async fn get_next_tee_verifier_input_producer_job( From 7244e4024757902bc0ed04c70846593ce54ae6e8 Mon Sep 17 00:00:00 2001 From: Thomas Knauth Date: Mon, 12 Aug 2024 08:22:42 +0000 Subject: [PATCH 3/4] fixup! feat: remove TEE from tree --- core/lib/dal/src/tee_verifier_input_producer_dal.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/lib/dal/src/tee_verifier_input_producer_dal.rs b/core/lib/dal/src/tee_verifier_input_producer_dal.rs index 56950f5cfdb5..dad93197d7a9 100644 --- a/core/lib/dal/src/tee_verifier_input_producer_dal.rs +++ b/core/lib/dal/src/tee_verifier_input_producer_dal.rs @@ -89,8 +89,12 @@ impl TeeVerifierInputProducerDal<'_, '_> { .report_latency() .fetch_one(self.storage) .await? - .number - .unwrap(); + .number; + + let row_high = match row_high { + Some(number) => number, + None => return Ok(None), + }; let row_low = sqlx::query!( r#" From c1988f7acef5c8171f60706be56e6e209da6c20b Mon Sep 17 00:00:00 2001 From: Thomas Knauth Date: Mon, 19 Aug 2024 10:32:47 +0000 Subject: [PATCH 4/4] Address review comments. Change get_new_l1_batches() to only return a single value. --- .../src/tee_verifier_input_producer_dal.rs | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/core/lib/dal/src/tee_verifier_input_producer_dal.rs b/core/lib/dal/src/tee_verifier_input_producer_dal.rs index dad93197d7a9..50f7011917dc 100644 --- a/core/lib/dal/src/tee_verifier_input_producer_dal.rs +++ b/core/lib/dal/src/tee_verifier_input_producer_dal.rs @@ -72,17 +72,16 @@ impl TeeVerifierInputProducerDal<'_, '_> { Ok(()) } - // Returns inclusive range, i.e. [low,high] - pub async fn get_new_l1_batches( - &mut self, - ) -> DalResult> { + pub async fn get_new_l1_batch(&mut self) -> DalResult> { // Since we depend on Merkle paths, we use the proof_generation_details table to inform us of newly available to-be-proven batches. - let row_high = sqlx::query!( + let row = sqlx::query!( r#" SELECT MAX(l1_batch_number) AS "number" FROM proof_generation_details + WHERE + proof_gen_data_blob_url IS NOT NULL "# ) .instrument("get_sealed_l1_batch_number") @@ -91,41 +90,43 @@ impl TeeVerifierInputProducerDal<'_, '_> { .await? .number; - let row_high = match row_high { + let max_sealed = match row { Some(number) => number, None => return Ok(None), }; - let row_low = sqlx::query!( + let row = sqlx::query!( r#" SELECT - MAX(l1_batch_number) + 1 AS "number" + MAX(l1_batch_number) AS "number" FROM tee_verifier_input_producer_jobs "# ) - .instrument("get_latest_tee_job_l1_batch_number") + .instrument("get_latest_tee_verifier_input_producer_jobs") .report_latency() .fetch_one(self.storage) .await? - .number - // If the table is empty, we start with the most recent batch. - .unwrap_or(row_high); + .number; - Ok(Some(( - L1BatchNumber(row_low as u32), - L1BatchNumber(row_high as u32), - ))) + match row { + // If no batches have been processed by TEE so far, i.e., table is empty, we start with the most recent L1 batch. + None => Ok(Some(L1BatchNumber(max_sealed as u32))), + Some(max_tee_batch_number) => { + if max_sealed > max_tee_batch_number { + Ok(Some(L1BatchNumber(max_tee_batch_number as u32 + 1))) + } else { + Ok(None) + } + } + } } pub async fn get_next_tee_verifier_input_producer_job( &mut self, ) -> DalResult> { - if let Some((low, high)) = self.get_new_l1_batches().await? { - for i in low.0..=high.0 { - self.create_tee_verifier_input_producer_job(L1BatchNumber(i)) - .await? - } + if let Some(n) = self.get_new_l1_batch().await? { + self.create_tee_verifier_input_producer_job(n).await?; } let l1_batch_number = sqlx::query!(