From 3df911121ec3b72a5e7c01e1cbca18549c17a54a Mon Sep 17 00:00:00 2001 From: Akash Date: Fri, 27 Oct 2023 11:50:23 +0400 Subject: [PATCH 1/6] fix(db-index): Add missing index from FRI prover job retry --- ...0231027074517_add_index_for_fri_prover_job_retry.down.sql | 5 +++++ .../20231027074517_add_index_for_fri_prover_job_retry.up.sql | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.down.sql create mode 100644 core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql diff --git a/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.down.sql b/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.down.sql new file mode 100644 index 000000000000..c3cecf9c572b --- /dev/null +++ b/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.down.sql @@ -0,0 +1,5 @@ +CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_status_processing_attempts + ON prover_jobs_fri (processing_started_at, attempts) + WHERE status IN ('in_progress', 'failed'); + +DROP INDEX IF EXISTS idx_prover_jobs_fri_status_processing_attempts_2; diff --git a/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql b/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql new file mode 100644 index 000000000000..1693d5efe52a --- /dev/null +++ b/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql @@ -0,0 +1,5 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_prover_jobs_fri_status_processing_attempts_2 + ON prover_jobs_fri (processing_started_at, attempts) + WHERE status IN ('in_progress', 'failed', 'in_gpu_proof'); + +DROP INDEX IF EXISTS idx_prover_jobs_fri_status_processing_attempts; From fa4005d0e74281c0da27fd94df64261d988193d8 Mon Sep 17 00:00:00 2001 From: Akash <112477155+akash-chandrakar@users.noreply.github.com> Date: Fri, 27 Oct 2023 11:59:17 +0400 Subject: [PATCH 2/6] Update 20231027074517_add_index_for_fri_prover_job_retry.up.sql --- .../20231027074517_add_index_for_fri_prover_job_retry.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql b/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql index 1693d5efe52a..5fc84e7524a4 100644 --- a/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql +++ b/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql @@ -1,4 +1,4 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_prover_jobs_fri_status_processing_attempts_2 +CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_status_processing_attempts_2 ON prover_jobs_fri (processing_started_at, attempts) WHERE status IN ('in_progress', 'failed', 'in_gpu_proof'); From 17a5096f17e8d86ab27d63bca9c6192638b85bd0 Mon Sep 17 00:00:00 2001 From: Akash Date: Fri, 27 Oct 2023 14:45:09 +0400 Subject: [PATCH 3/6] update index --- ..._add_index_for_fri_prover_job_retry.up.sql | 5 -- ...sing_index_for_fri_prover_tables.down.sql} | 0 ...missing_index_for_fri_prover_tables.up.sql | 11 +++ core/lib/dal/sqlx-data.json | 72 +++++++++---------- core/lib/dal/src/fri_prover_dal.rs | 1 + 5 files changed, 48 insertions(+), 41 deletions(-) delete mode 100644 core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql rename core/lib/dal/migrations/{20231027074517_add_index_for_fri_prover_job_retry.down.sql => 20231027074517_add_missing_index_for_fri_prover_tables.down.sql} (100%) create mode 100644 core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql diff --git a/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql b/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql deleted file mode 100644 index 5fc84e7524a4..000000000000 --- a/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.up.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_status_processing_attempts_2 - ON prover_jobs_fri (processing_started_at, attempts) - WHERE status IN ('in_progress', 'failed', 'in_gpu_proof'); - -DROP INDEX IF EXISTS idx_prover_jobs_fri_status_processing_attempts; diff --git a/core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.down.sql b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql similarity index 100% rename from core/lib/dal/migrations/20231027074517_add_index_for_fri_prover_job_retry.down.sql rename to core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql diff --git a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql new file mode 100644 index 000000000000..f500b334db9f --- /dev/null +++ b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql @@ -0,0 +1,11 @@ +CREATE INDEX IF NOT EXISTS prover_jobs_fri_status_processing_started_at_idx ON prover_jobs_fri (status, processing_started_at) + WHERE (attempts < 20); + +DROP INDEX IF EXISTS idx_prover_jobs_fri_status_processing_attempts; + +CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_status ON prover_jobs_fri (circuit_id, aggregation_round) + WHERE (status != 'successful' and status != 'skipped'); + +CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_circuit_id_agg_batch_num + ON prover_jobs_fri (circuit_id, aggregation_round, l1_batch_number) + WHERE status IN ('queued', 'in_progress', 'in_gpu_proof', 'failed'); diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json index 29aea31db1f2..a673b348a639 100644 --- a/core/lib/dal/sqlx-data.json +++ b/core/lib/dal/sqlx-data.json @@ -1374,42 +1374,6 @@ }, "query": "\n UPDATE node_aggregation_witness_jobs_fri\n SET status='queued'\n WHERE (l1_batch_number, circuit_id, depth) IN\n (SELECT prover_jobs_fri.l1_batch_number, prover_jobs_fri.circuit_id, prover_jobs_fri.depth\n FROM prover_jobs_fri\n JOIN node_aggregation_witness_jobs_fri nawj ON\n prover_jobs_fri.l1_batch_number = nawj.l1_batch_number\n AND prover_jobs_fri.circuit_id = nawj.circuit_id\n AND prover_jobs_fri.depth = nawj.depth\n WHERE nawj.status = 'waiting_for_proofs'\n AND prover_jobs_fri.status = 'successful'\n AND prover_jobs_fri.aggregation_round = 1\n AND prover_jobs_fri.depth = 0\n GROUP BY prover_jobs_fri.l1_batch_number, prover_jobs_fri.circuit_id, prover_jobs_fri.depth, nawj.number_of_dependent_jobs\n HAVING COUNT(*) = nawj.number_of_dependent_jobs)\n RETURNING l1_batch_number, circuit_id, depth;\n " }, - "1e68306cbd83eb6b5de59fb8f638c8f3252732b9074e2455f7b5aedf6fdc886f": { - "describe": { - "columns": [ - { - "name": "count!", - "ordinal": 0, - "type_info": "Int8" - }, - { - "name": "circuit_id!", - "ordinal": 1, - "type_info": "Int2" - }, - { - "name": "aggregation_round!", - "ordinal": 2, - "type_info": "Int2" - }, - { - "name": "status!", - "ordinal": 3, - "type_info": "Text" - } - ], - "nullable": [ - null, - false, - false, - false - ], - "parameters": { - "Left": [] - } - }, - "query": "\n SELECT COUNT(*) as \"count!\", circuit_id as \"circuit_id!\", aggregation_round as \"aggregation_round!\", status as \"status!\"\n FROM prover_jobs_fri\n GROUP BY circuit_id, aggregation_round, status\n " - }, "1ed353a16e8d0abaf426e5c235b20a79c727c08bc23fb1708a833a6930131691": { "describe": { "columns": [], @@ -4038,6 +4002,42 @@ }, "query": "SELECT bootloader_code_hash, default_account_code_hash FROM protocol_versions\n WHERE id = $1\n " }, + "51d02f2e314ebf78c27949cc10997bd2171755400cc3a13c63994c85e15cb3df": { + "describe": { + "columns": [ + { + "name": "count!", + "ordinal": 0, + "type_info": "Int8" + }, + { + "name": "circuit_id!", + "ordinal": 1, + "type_info": "Int2" + }, + { + "name": "aggregation_round!", + "ordinal": 2, + "type_info": "Int2" + }, + { + "name": "status!", + "ordinal": 3, + "type_info": "Text" + } + ], + "nullable": [ + null, + false, + false, + false + ], + "parameters": { + "Left": [] + } + }, + "query": "\n SELECT COUNT(*) as \"count!\", circuit_id as \"circuit_id!\", aggregation_round as \"aggregation_round!\", status as \"status!\"\n FROM prover_jobs_fri\n WHERE status <> 'skipped' and status <> 'successful'\n GROUP BY circuit_id, aggregation_round, status\n " + }, "52eeb8c529efb796fdefb30a381fcf6c931512f30e55e24c155f6c649e662909": { "describe": { "columns": [ diff --git a/core/lib/dal/src/fri_prover_dal.rs b/core/lib/dal/src/fri_prover_dal.rs index af1c218195cf..3362c6bd166f 100644 --- a/core/lib/dal/src/fri_prover_dal.rs +++ b/core/lib/dal/src/fri_prover_dal.rs @@ -268,6 +268,7 @@ impl FriProverDal<'_, '_> { r#" SELECT COUNT(*) as "count!", circuit_id as "circuit_id!", aggregation_round as "aggregation_round!", status as "status!" FROM prover_jobs_fri + WHERE status <> 'skipped' and status <> 'successful' GROUP BY circuit_id, aggregation_round, status "# ) From 18ff7739b2b388b7dcd8f59959c5c174384b79a1 Mon Sep 17 00:00:00 2001 From: Akash Date: Fri, 27 Oct 2023 14:50:29 +0400 Subject: [PATCH 4/6] fix revert and add missing --- ...ssing_index_for_fri_prover_tables.down.sql | 4 ++ core/lib/dal/sqlx-data.json | 60 +++++++++---------- core/lib/dal/src/fri_prover_dal.rs | 2 +- 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql index c3cecf9c572b..2efc91386ec8 100644 --- a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql +++ b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql @@ -3,3 +3,7 @@ CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_status_processing_attempts WHERE status IN ('in_progress', 'failed'); DROP INDEX IF EXISTS idx_prover_jobs_fri_status_processing_attempts_2; + +DROP INDEX IF EXISTS idx_prover_jobs_fri_status; + +DROP INDEX IF EXISTS idx_prover_jobs_fri_circuit_id_agg_batch_num; diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json index a673b348a639..7fc1026a3d5a 100644 --- a/core/lib/dal/sqlx-data.json +++ b/core/lib/dal/sqlx-data.json @@ -285,36 +285,6 @@ }, "query": "\n INSERT INTO node_aggregation_witness_jobs\n (l1_batch_number, protocol_version, status, created_at, updated_at)\n VALUES ($1, $2, 'waiting_for_artifacts', now(), now())\n " }, - "0c729d441aceba247e36c08a89661c35b476d4d7c73882147699009affe78472": { - "describe": { - "columns": [ - { - "name": "l1_batch_number!", - "ordinal": 0, - "type_info": "Int8" - }, - { - "name": "circuit_id", - "ordinal": 1, - "type_info": "Int2" - }, - { - "name": "aggregation_round", - "ordinal": 2, - "type_info": "Int2" - } - ], - "nullable": [ - null, - false, - false - ], - "parameters": { - "Left": [] - } - }, - "query": "\n SELECT MIN(l1_batch_number) as \"l1_batch_number!\", circuit_id, aggregation_round\n FROM prover_jobs_fri\n WHERE status IN('queued', 'in_progress', 'failed')\n GROUP BY circuit_id, aggregation_round\n " - }, "0d1bed183c38304ff1a6c8c78dca03964e2e188a6d01f98eaf0c6b24f19b8b6f": { "describe": { "columns": [], @@ -7762,6 +7732,36 @@ }, "query": "UPDATE prover_jobs_fri SET status = $1, updated_at = now() WHERE id = $2" }, + "98c81ee6f73859c6cd6ba54ab438c900dda646b70a700f936e5218d9ba3bd0ec": { + "describe": { + "columns": [ + { + "name": "l1_batch_number!", + "ordinal": 0, + "type_info": "Int8" + }, + { + "name": "circuit_id", + "ordinal": 1, + "type_info": "Int2" + }, + { + "name": "aggregation_round", + "ordinal": 2, + "type_info": "Int2" + } + ], + "nullable": [ + null, + false, + false + ], + "parameters": { + "Left": [] + } + }, + "query": "\n SELECT MIN(l1_batch_number) as \"l1_batch_number!\", circuit_id, aggregation_round\n FROM prover_jobs_fri\n WHERE status IN('queued', 'in_gpu_proof', 'in_progress', 'failed')\n GROUP BY circuit_id, aggregation_round\n " + }, "9970bb69f5ca9ab9f103e1547eb40c1d4f5dd3a540ff6f1b9724821350c9501a": { "describe": { "columns": [ diff --git a/core/lib/dal/src/fri_prover_dal.rs b/core/lib/dal/src/fri_prover_dal.rs index 3362c6bd166f..845e793948d1 100644 --- a/core/lib/dal/src/fri_prover_dal.rs +++ b/core/lib/dal/src/fri_prover_dal.rs @@ -302,7 +302,7 @@ impl FriProverDal<'_, '_> { r#" SELECT MIN(l1_batch_number) as "l1_batch_number!", circuit_id, aggregation_round FROM prover_jobs_fri - WHERE status IN('queued', 'in_progress', 'failed') + WHERE status IN('queued', 'in_gpu_proof', 'in_progress', 'failed') GROUP BY circuit_id, aggregation_round "# ) From 95d0ca0cb35d2cbe9a8522e272e8fdf65774d887 Mon Sep 17 00:00:00 2001 From: Akash Date: Fri, 27 Oct 2023 14:53:44 +0400 Subject: [PATCH 5/6] update name --- ...0231027074517_add_missing_index_for_fri_prover_tables.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql index f500b334db9f..1a9eaece6a34 100644 --- a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql +++ b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql @@ -1,4 +1,4 @@ -CREATE INDEX IF NOT EXISTS prover_jobs_fri_status_processing_started_at_idx ON prover_jobs_fri (status, processing_started_at) +CREATE INDEX IF NOT EXISTS prover_jobs_fri_status_processing_started_at_idx_2 ON prover_jobs_fri (status, processing_started_at) WHERE (attempts < 20); DROP INDEX IF EXISTS idx_prover_jobs_fri_status_processing_attempts; From a3572d9c8deb5bf99a9adf27dedac72db69edb98 Mon Sep 17 00:00:00 2001 From: Akash <112477155+akash-chandrakar@users.noreply.github.com> Date: Fri, 27 Oct 2023 14:54:35 +0400 Subject: [PATCH 6/6] Update 20231027074517_add_missing_index_for_fri_prover_tables.down.sql --- ...31027074517_add_missing_index_for_fri_prover_tables.down.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql index 2efc91386ec8..6c80db7472a2 100644 --- a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql +++ b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql @@ -2,7 +2,7 @@ CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_status_processing_attempts ON prover_jobs_fri (processing_started_at, attempts) WHERE status IN ('in_progress', 'failed'); -DROP INDEX IF EXISTS idx_prover_jobs_fri_status_processing_attempts_2; +DROP INDEX IF EXISTS prover_jobs_fri_status_processing_started_at_idx_2; DROP INDEX IF EXISTS idx_prover_jobs_fri_status;