From 2488a767a362ea3b40a348ae9822bed77d4b8de9 Mon Sep 17 00:00:00 2001 From: EmilLuta Date: Thu, 13 Jun 2024 17:58:44 +0200 Subject: [PATCH] fix(prover): Disallow state changes from successful (#2233) This PR is done as a fix for boojnet outage. TL;DR; of outage -- race condition caused by prover jobs moving from 'successful` state to `in_progress`/`in_gpu_proving`. The PR addresses: - no job can move from successful state (considered final state) - fix local development (contracts were pointing to 0.24.0 instead of 0.24.1) -- can be split to a different PR, if this is problematic. - add table constraint -- again, can be split in different PR - add checks for recursion_tip number of jobs (post outage check, should not happen ever, but better to verify) --- core/lib/basic_types/src/prover_dal.rs | 2 +- ...270e25815ca2ab720a59567da3b3b5bcedd63.json | 15 ------- etc/env/base/contracts.toml | 2 +- ...254a457665179d9cf0a3c0b18c3fe09e4838.json} | 4 +- ...601d35fd2881ac1fd070f0f1a8add4bc388d.json} | 10 ++++- ...a1a04821495487a80595cc9b523dac6ac8e9.json} | 4 +- ...0fc9ac9a7f3dce2ba48c2d0e6f38b6ba455a.json} | 4 +- ...c39ae8a6e053a0e03afd3fb5e02ee17157067.json | 2 +- ...66e8f67a380302762c272bfb27307682d62e.json} | 8 ++-- ...f724216807ffd481cd6f7f19968e42e52b284.json | 14 ------ ...0222e177262292241bd8cb89dbb9c1e74c2d.json} | 4 +- ...263556f258565f79cbb40f5ecc1a4f6402f5.json} | 4 +- ...775c6f7414c7bed75d33b61de00fdbabc349.json} | 4 +- ...b02c44b099e27e3c45c5c810cd5fcd8884ed.json} | 4 +- ...71ababa66e4a443fbefbfffca72b7540b075b.json | 15 ------- ...d6a8a6de1e3a56e2a95963d933c21485c9939.json | 28 ------------ ...67878f347bdaf36294e9b24ee9c0aa1e861b.json} | 4 +- ...mber_of_final_node_jobs_mandatory.down.sql | 1 + ...number_of_final_node_jobs_mandatory.up.sql | 1 + .../src/fri_proof_compressor_dal.rs | 23 +++------- prover/prover_dal/src/fri_prover_dal.rs | 44 ++----------------- .../src/fri_witness_generator_dal.rs | 18 ++++++-- prover/witness_generator/src/recursion_tip.rs | 9 +++- 23 files changed, 65 insertions(+), 159 deletions(-) delete mode 100644 core/lib/dal/.sqlx/query-6b7f66422078e9880b002da3175270e25815ca2ab720a59567da3b3b5bcedd63.json rename prover/prover_dal/.sqlx/{query-12ab208f416e2875f89e558f0d4aff3a06b7a9c1866132d62e4449fa9436c7c4.json => query-16548daf69e9ff0528904be2e142254a457665179d9cf0a3c0b18c3fe09e4838.json} (67%) rename prover/prover_dal/.sqlx/{query-147e61e0ff8ce225b7fadc1ea0ef63b24a5d95e45908be338c00a034f7a82083.json => query-41af30620f8a1f20b8a6c46be162601d35fd2881ac1fd070f0f1a8add4bc388d.json} (80%) rename prover/prover_dal/.sqlx/{query-9ef2f43e6201cc00a0e1425a666a36532fee1450733849852dfd20e18ded1f03.json => query-5db868e03dc6901a0afa06f82a37a1a04821495487a80595cc9b523dac6ac8e9.json} (66%) rename prover/prover_dal/.sqlx/{query-afc24bd1407dba82cd3dc9e7ee71ac4ab2d73bda6022700aeb0a630a2563a4b4.json => query-67f5f3a015dc478f02f4f701c90d0fc9ac9a7f3dce2ba48c2d0e6f38b6ba455a.json} (67%) rename prover/prover_dal/.sqlx/{query-f4362a61ab05af3d71a3232d2f017db60405a887f9f7fa0ca60aa7fc879ce630.json => query-93b9706aa8eb840d574d7c156cc866e8f67a380302762c272bfb27307682d62e.json} (59%) delete mode 100644 prover/prover_dal/.sqlx/query-af72fabd90eb43fb315f46d7fe9f724216807ffd481cd6f7f19968e42e52b284.json rename prover/prover_dal/.sqlx/{query-5821f1446983260168cec366af26009503182c300877e74a8539f231050e6f85.json => query-b25c66b9705b3f2fb8a3492f1bd20222e177262292241bd8cb89dbb9c1e74c2d.json} (72%) rename prover/prover_dal/.sqlx/{query-06a8b9028125d81f1d83180dbe23fc5acbb7569ef424d98a232f1bfc0bf0a6b1.json => query-bf06bd08d8ccf67fc00bbc364715263556f258565f79cbb40f5ecc1a4f6402f5.json} (66%) rename prover/prover_dal/.sqlx/{query-c23d5ff919ade5898c6a912780ae899e360650afccb34f5cc301b5cbac4a3d36.json => query-c340c043c938bf5f4b63d57a1654775c6f7414c7bed75d33b61de00fdbabc349.json} (61%) rename prover/prover_dal/.sqlx/{query-a0e2b2c034cc5f668f0b3d43b94d2e2326d7ace079b095def52723a45b65d3f3.json => query-d4949debfe0dc5112204cd196c68b02c44b099e27e3c45c5c810cd5fcd8884ed.json} (67%) delete mode 100644 prover/prover_dal/.sqlx/query-e74a34a59e6afda689b0ec9e19071ababa66e4a443fbefbfffca72b7540b075b.json delete mode 100644 prover/prover_dal/.sqlx/query-e78e94239dc10c5560f239a71e4d6a8a6de1e3a56e2a95963d933c21485c9939.json rename prover/prover_dal/.sqlx/{query-35b87a3b7db0af87c6a95e9fe7ef9044ae85b579c7051301b40bd5f94df1f530.json => query-e8066db420e075306235f728d57567878f347bdaf36294e9b24ee9c0aa1e861b.json} (76%) create mode 100644 prover/prover_dal/migrations/20240613111518_make_number_of_final_node_jobs_mandatory.down.sql create mode 100644 prover/prover_dal/migrations/20240613111518_make_number_of_final_node_jobs_mandatory.up.sql diff --git a/core/lib/basic_types/src/prover_dal.rs b/core/lib/basic_types/src/prover_dal.rs index 2d3d6f085e06..1d741fac508c 100644 --- a/core/lib/basic_types/src/prover_dal.rs +++ b/core/lib/basic_types/src/prover_dal.rs @@ -332,7 +332,7 @@ pub struct RecursionTipWitnessGeneratorJobInfo { pub error: Option, pub created_at: NaiveDateTime, pub updated_at: NaiveDateTime, - pub number_of_final_node_jobs: Option, + pub number_of_final_node_jobs: i32, pub protocol_version: Option, pub picked_by: Option, } diff --git a/core/lib/dal/.sqlx/query-6b7f66422078e9880b002da3175270e25815ca2ab720a59567da3b3b5bcedd63.json b/core/lib/dal/.sqlx/query-6b7f66422078e9880b002da3175270e25815ca2ab720a59567da3b3b5bcedd63.json deleted file mode 100644 index b7b84c323b2e..000000000000 --- a/core/lib/dal/.sqlx/query-6b7f66422078e9880b002da3175270e25815ca2ab720a59567da3b3b5bcedd63.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE tee_proof_generation_details\n SET\n status = $1,\n updated_at = NOW()\n WHERE\n l1_batch_number = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "6b7f66422078e9880b002da3175270e25815ca2ab720a59567da3b3b5bcedd63" -} diff --git a/etc/env/base/contracts.toml b/etc/env/base/contracts.toml index 91f25a41e80d..b88a3e179eae 100644 --- a/etc/env/base/contracts.toml +++ b/etc/env/base/contracts.toml @@ -32,7 +32,7 @@ PRIORITY_TX_MAX_GAS_LIMIT = 72000000 DEPLOY_L2_BRIDGE_COUNTERPART_GAS_LIMIT = 10000000 GENESIS_ROLLUP_LEAF_INDEX = "54" GENESIS_PROTOCOL_VERSION = "24" -GENESIS_PROTOCOL_SEMANTIC_VERSION = "0.24.0" +GENESIS_PROTOCOL_SEMANTIC_VERSION = "0.24.1" L1_WETH_BRIDGE_IMPL_ADDR = "0x5E6D086F5eC079ADFF4FB3774CDf3e8D6a34F7E9" L1_WETH_BRIDGE_PROXY_ADDR = "0x5E6D086F5eC079ADFF4FB3774CDf3e8D6a34F7E9" L1_WETH_TOKEN_ADDR = "0x5E6D086F5eC079ADFF4FB3774CDf3e8D6a34F7E9" diff --git a/prover/prover_dal/.sqlx/query-12ab208f416e2875f89e558f0d4aff3a06b7a9c1866132d62e4449fa9436c7c4.json b/prover/prover_dal/.sqlx/query-16548daf69e9ff0528904be2e142254a457665179d9cf0a3c0b18c3fe09e4838.json similarity index 67% rename from prover/prover_dal/.sqlx/query-12ab208f416e2875f89e558f0d4aff3a06b7a9c1866132d62e4449fa9436c7c4.json rename to prover/prover_dal/.sqlx/query-16548daf69e9ff0528904be2e142254a457665179d9cf0a3c0b18c3fe09e4838.json index 5441bce3e016..5fe5032746e9 100644 --- a/prover/prover_dal/.sqlx/query-12ab208f416e2875f89e558f0d4aff3a06b7a9c1866132d62e4449fa9436c7c4.json +++ b/prover/prover_dal/.sqlx/query-16548daf69e9ff0528904be2e142254a457665179d9cf0a3c0b18c3fe09e4838.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE node_aggregation_witness_jobs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n id = $2\n ", + "query": "\n UPDATE node_aggregation_witness_jobs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n id = $2\n AND status != 'successful'\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "12ab208f416e2875f89e558f0d4aff3a06b7a9c1866132d62e4449fa9436c7c4" + "hash": "16548daf69e9ff0528904be2e142254a457665179d9cf0a3c0b18c3fe09e4838" } diff --git a/prover/prover_dal/.sqlx/query-147e61e0ff8ce225b7fadc1ea0ef63b24a5d95e45908be338c00a034f7a82083.json b/prover/prover_dal/.sqlx/query-41af30620f8a1f20b8a6c46be162601d35fd2881ac1fd070f0f1a8add4bc388d.json similarity index 80% rename from prover/prover_dal/.sqlx/query-147e61e0ff8ce225b7fadc1ea0ef63b24a5d95e45908be338c00a034f7a82083.json rename to prover/prover_dal/.sqlx/query-41af30620f8a1f20b8a6c46be162601d35fd2881ac1fd070f0f1a8add4bc388d.json index e681ac6a1a34..b65633a904e3 100644 --- a/prover/prover_dal/.sqlx/query-147e61e0ff8ce225b7fadc1ea0ef63b24a5d95e45908be338c00a034f7a82083.json +++ b/prover/prover_dal/.sqlx/query-41af30620f8a1f20b8a6c46be162601d35fd2881ac1fd070f0f1a8add4bc388d.json @@ -1,12 +1,17 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE recursion_tip_witness_jobs_fri\n SET\n status = 'in_progress',\n attempts = attempts + 1,\n updated_at = NOW(),\n processing_started_at = NOW(),\n picked_by = $3\n WHERE\n l1_batch_number = (\n SELECT\n l1_batch_number\n FROM\n recursion_tip_witness_jobs_fri\n WHERE\n status = 'queued'\n AND protocol_version = $1\n AND protocol_version_patch = $2\n ORDER BY\n l1_batch_number ASC\n LIMIT\n 1\n FOR UPDATE\n SKIP LOCKED\n )\n RETURNING\n recursion_tip_witness_jobs_fri.l1_batch_number\n ", + "query": "\n UPDATE recursion_tip_witness_jobs_fri\n SET\n status = 'in_progress',\n attempts = attempts + 1,\n updated_at = NOW(),\n processing_started_at = NOW(),\n picked_by = $3\n WHERE\n l1_batch_number = (\n SELECT\n l1_batch_number\n FROM\n recursion_tip_witness_jobs_fri\n WHERE\n status = 'queued'\n AND protocol_version = $1\n AND protocol_version_patch = $2\n ORDER BY\n l1_batch_number ASC\n LIMIT\n 1\n FOR UPDATE\n SKIP LOCKED\n )\n RETURNING\n recursion_tip_witness_jobs_fri.l1_batch_number,\n recursion_tip_witness_jobs_fri.number_of_final_node_jobs\n ", "describe": { "columns": [ { "ordinal": 0, "name": "l1_batch_number", "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "number_of_final_node_jobs", + "type_info": "Int4" } ], "parameters": { @@ -17,8 +22,9 @@ ] }, "nullable": [ + false, false ] }, - "hash": "147e61e0ff8ce225b7fadc1ea0ef63b24a5d95e45908be338c00a034f7a82083" + "hash": "41af30620f8a1f20b8a6c46be162601d35fd2881ac1fd070f0f1a8add4bc388d" } diff --git a/prover/prover_dal/.sqlx/query-9ef2f43e6201cc00a0e1425a666a36532fee1450733849852dfd20e18ded1f03.json b/prover/prover_dal/.sqlx/query-5db868e03dc6901a0afa06f82a37a1a04821495487a80595cc9b523dac6ac8e9.json similarity index 66% rename from prover/prover_dal/.sqlx/query-9ef2f43e6201cc00a0e1425a666a36532fee1450733849852dfd20e18ded1f03.json rename to prover/prover_dal/.sqlx/query-5db868e03dc6901a0afa06f82a37a1a04821495487a80595cc9b523dac6ac8e9.json index fd770071cf86..94dbaa80a10e 100644 --- a/prover/prover_dal/.sqlx/query-9ef2f43e6201cc00a0e1425a666a36532fee1450733849852dfd20e18ded1f03.json +++ b/prover/prover_dal/.sqlx/query-5db868e03dc6901a0afa06f82a37a1a04821495487a80595cc9b523dac6ac8e9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE scheduler_witness_jobs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n l1_batch_number = $2\n ", + "query": "\n UPDATE scheduler_witness_jobs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n l1_batch_number = $2\n AND status != 'successful'\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "9ef2f43e6201cc00a0e1425a666a36532fee1450733849852dfd20e18ded1f03" + "hash": "5db868e03dc6901a0afa06f82a37a1a04821495487a80595cc9b523dac6ac8e9" } diff --git a/prover/prover_dal/.sqlx/query-afc24bd1407dba82cd3dc9e7ee71ac4ab2d73bda6022700aeb0a630a2563a4b4.json b/prover/prover_dal/.sqlx/query-67f5f3a015dc478f02f4f701c90d0fc9ac9a7f3dce2ba48c2d0e6f38b6ba455a.json similarity index 67% rename from prover/prover_dal/.sqlx/query-afc24bd1407dba82cd3dc9e7ee71ac4ab2d73bda6022700aeb0a630a2563a4b4.json rename to prover/prover_dal/.sqlx/query-67f5f3a015dc478f02f4f701c90d0fc9ac9a7f3dce2ba48c2d0e6f38b6ba455a.json index ede2995ff558..29838881a52c 100644 --- a/prover/prover_dal/.sqlx/query-afc24bd1407dba82cd3dc9e7ee71ac4ab2d73bda6022700aeb0a630a2563a4b4.json +++ b/prover/prover_dal/.sqlx/query-67f5f3a015dc478f02f4f701c90d0fc9ac9a7f3dce2ba48c2d0e6f38b6ba455a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE leaf_aggregation_witness_jobs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n id = $2\n ", + "query": "\n UPDATE leaf_aggregation_witness_jobs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n id = $2\n AND status != 'successful'\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "afc24bd1407dba82cd3dc9e7ee71ac4ab2d73bda6022700aeb0a630a2563a4b4" + "hash": "67f5f3a015dc478f02f4f701c90d0fc9ac9a7f3dce2ba48c2d0e6f38b6ba455a" } diff --git a/prover/prover_dal/.sqlx/query-85a69b433c08847876bf6e7af9bc39ae8a6e053a0e03afd3fb5e02ee17157067.json b/prover/prover_dal/.sqlx/query-85a69b433c08847876bf6e7af9bc39ae8a6e053a0e03afd3fb5e02ee17157067.json index 58b9116faaa2..9631bd54d394 100644 --- a/prover/prover_dal/.sqlx/query-85a69b433c08847876bf6e7af9bc39ae8a6e053a0e03afd3fb5e02ee17157067.json +++ b/prover/prover_dal/.sqlx/query-85a69b433c08847876bf6e7af9bc39ae8a6e053a0e03afd3fb5e02ee17157067.json @@ -78,7 +78,7 @@ true, false, false, - true, + false, true, true, false diff --git a/prover/prover_dal/.sqlx/query-f4362a61ab05af3d71a3232d2f017db60405a887f9f7fa0ca60aa7fc879ce630.json b/prover/prover_dal/.sqlx/query-93b9706aa8eb840d574d7c156cc866e8f67a380302762c272bfb27307682d62e.json similarity index 59% rename from prover/prover_dal/.sqlx/query-f4362a61ab05af3d71a3232d2f017db60405a887f9f7fa0ca60aa7fc879ce630.json rename to prover/prover_dal/.sqlx/query-93b9706aa8eb840d574d7c156cc866e8f67a380302762c272bfb27307682d62e.json index 59c28852a03e..90eface5350b 100644 --- a/prover/prover_dal/.sqlx/query-f4362a61ab05af3d71a3232d2f017db60405a887f9f7fa0ca60aa7fc879ce630.json +++ b/prover/prover_dal/.sqlx/query-93b9706aa8eb840d574d7c156cc866e8f67a380302762c272bfb27307682d62e.json @@ -1,16 +1,18 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE proof_compression_jobs_fri\n SET\n status = $1,\n error = $2,\n updated_at = NOW()\n WHERE\n l1_batch_number = $3\n ", + "query": "\n UPDATE proof_compression_jobs_fri\n SET\n status = $1,\n error = $2,\n updated_at = NOW()\n WHERE\n l1_batch_number = $3\n AND status != $4\n AND status != $5\n ", "describe": { "columns": [], "parameters": { "Left": [ "Text", "Text", - "Int8" + "Int8", + "Text", + "Text" ] }, "nullable": [] }, - "hash": "f4362a61ab05af3d71a3232d2f017db60405a887f9f7fa0ca60aa7fc879ce630" + "hash": "93b9706aa8eb840d574d7c156cc866e8f67a380302762c272bfb27307682d62e" } diff --git a/prover/prover_dal/.sqlx/query-af72fabd90eb43fb315f46d7fe9f724216807ffd481cd6f7f19968e42e52b284.json b/prover/prover_dal/.sqlx/query-af72fabd90eb43fb315f46d7fe9f724216807ffd481cd6f7f19968e42e52b284.json deleted file mode 100644 index 6674fab59eab..000000000000 --- a/prover/prover_dal/.sqlx/query-af72fabd90eb43fb315f46d7fe9f724216807ffd481cd6f7f19968e42e52b284.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE prover_jobs_fri\n SET\n status = 'sent_to_server',\n updated_at = NOW()\n WHERE\n l1_batch_number = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "af72fabd90eb43fb315f46d7fe9f724216807ffd481cd6f7f19968e42e52b284" -} diff --git a/prover/prover_dal/.sqlx/query-5821f1446983260168cec366af26009503182c300877e74a8539f231050e6f85.json b/prover/prover_dal/.sqlx/query-b25c66b9705b3f2fb8a3492f1bd20222e177262292241bd8cb89dbb9c1e74c2d.json similarity index 72% rename from prover/prover_dal/.sqlx/query-5821f1446983260168cec366af26009503182c300877e74a8539f231050e6f85.json rename to prover/prover_dal/.sqlx/query-b25c66b9705b3f2fb8a3492f1bd20222e177262292241bd8cb89dbb9c1e74c2d.json index 86877a48dd4d..d0c5d31aa3ec 100644 --- a/prover/prover_dal/.sqlx/query-5821f1446983260168cec366af26009503182c300877e74a8539f231050e6f85.json +++ b/prover/prover_dal/.sqlx/query-b25c66b9705b3f2fb8a3492f1bd20222e177262292241bd8cb89dbb9c1e74c2d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE witness_inputs_fri\n SET\n status = $1,\n updated_at = NOW()\n WHERE\n l1_batch_number = $2\n ", + "query": "\n UPDATE witness_inputs_fri\n SET\n status = $1,\n updated_at = NOW()\n WHERE\n l1_batch_number = $2\n AND status != 'successful'\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "5821f1446983260168cec366af26009503182c300877e74a8539f231050e6f85" + "hash": "b25c66b9705b3f2fb8a3492f1bd20222e177262292241bd8cb89dbb9c1e74c2d" } diff --git a/prover/prover_dal/.sqlx/query-06a8b9028125d81f1d83180dbe23fc5acbb7569ef424d98a232f1bfc0bf0a6b1.json b/prover/prover_dal/.sqlx/query-bf06bd08d8ccf67fc00bbc364715263556f258565f79cbb40f5ecc1a4f6402f5.json similarity index 66% rename from prover/prover_dal/.sqlx/query-06a8b9028125d81f1d83180dbe23fc5acbb7569ef424d98a232f1bfc0bf0a6b1.json rename to prover/prover_dal/.sqlx/query-bf06bd08d8ccf67fc00bbc364715263556f258565f79cbb40f5ecc1a4f6402f5.json index af6d49ba049b..c1f9806625dd 100644 --- a/prover/prover_dal/.sqlx/query-06a8b9028125d81f1d83180dbe23fc5acbb7569ef424d98a232f1bfc0bf0a6b1.json +++ b/prover/prover_dal/.sqlx/query-bf06bd08d8ccf67fc00bbc364715263556f258565f79cbb40f5ecc1a4f6402f5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE recursion_tip_witness_jobs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n l1_batch_number = $2\n ", + "query": "\n UPDATE recursion_tip_witness_jobs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n l1_batch_number = $2\n AND status != 'successful'\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "06a8b9028125d81f1d83180dbe23fc5acbb7569ef424d98a232f1bfc0bf0a6b1" + "hash": "bf06bd08d8ccf67fc00bbc364715263556f258565f79cbb40f5ecc1a4f6402f5" } diff --git a/prover/prover_dal/.sqlx/query-c23d5ff919ade5898c6a912780ae899e360650afccb34f5cc301b5cbac4a3d36.json b/prover/prover_dal/.sqlx/query-c340c043c938bf5f4b63d57a1654775c6f7414c7bed75d33b61de00fdbabc349.json similarity index 61% rename from prover/prover_dal/.sqlx/query-c23d5ff919ade5898c6a912780ae899e360650afccb34f5cc301b5cbac4a3d36.json rename to prover/prover_dal/.sqlx/query-c340c043c938bf5f4b63d57a1654775c6f7414c7bed75d33b61de00fdbabc349.json index 8922816c7e15..41edb649e7c9 100644 --- a/prover/prover_dal/.sqlx/query-c23d5ff919ade5898c6a912780ae899e360650afccb34f5cc301b5cbac4a3d36.json +++ b/prover/prover_dal/.sqlx/query-c340c043c938bf5f4b63d57a1654775c6f7414c7bed75d33b61de00fdbabc349.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE prover_jobs_fri\n SET\n status = $1,\n updated_at = NOW()\n WHERE\n id = $2\n ", + "query": "\n UPDATE prover_jobs_fri\n SET\n status = $1,\n updated_at = NOW()\n WHERE\n id = $2\n AND status != 'successful'\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "c23d5ff919ade5898c6a912780ae899e360650afccb34f5cc301b5cbac4a3d36" + "hash": "c340c043c938bf5f4b63d57a1654775c6f7414c7bed75d33b61de00fdbabc349" } diff --git a/prover/prover_dal/.sqlx/query-a0e2b2c034cc5f668f0b3d43b94d2e2326d7ace079b095def52723a45b65d3f3.json b/prover/prover_dal/.sqlx/query-d4949debfe0dc5112204cd196c68b02c44b099e27e3c45c5c810cd5fcd8884ed.json similarity index 67% rename from prover/prover_dal/.sqlx/query-a0e2b2c034cc5f668f0b3d43b94d2e2326d7ace079b095def52723a45b65d3f3.json rename to prover/prover_dal/.sqlx/query-d4949debfe0dc5112204cd196c68b02c44b099e27e3c45c5c810cd5fcd8884ed.json index 7dc19564f7fa..9121539b3179 100644 --- a/prover/prover_dal/.sqlx/query-a0e2b2c034cc5f668f0b3d43b94d2e2326d7ace079b095def52723a45b65d3f3.json +++ b/prover/prover_dal/.sqlx/query-d4949debfe0dc5112204cd196c68b02c44b099e27e3c45c5c810cd5fcd8884ed.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE witness_inputs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n l1_batch_number = $2\n ", + "query": "\n UPDATE witness_inputs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n l1_batch_number = $2\n AND status != 'successful'\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "a0e2b2c034cc5f668f0b3d43b94d2e2326d7ace079b095def52723a45b65d3f3" + "hash": "d4949debfe0dc5112204cd196c68b02c44b099e27e3c45c5c810cd5fcd8884ed" } diff --git a/prover/prover_dal/.sqlx/query-e74a34a59e6afda689b0ec9e19071ababa66e4a443fbefbfffca72b7540b075b.json b/prover/prover_dal/.sqlx/query-e74a34a59e6afda689b0ec9e19071ababa66e4a443fbefbfffca72b7540b075b.json deleted file mode 100644 index 54ea6b6eb03a..000000000000 --- a/prover/prover_dal/.sqlx/query-e74a34a59e6afda689b0ec9e19071ababa66e4a443fbefbfffca72b7540b075b.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO\n proof_compression_jobs_fri (l1_batch_number, status, created_at, updated_at)\n VALUES\n ($1, $2, NOW(), NOW())\n ON CONFLICT (l1_batch_number) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "e74a34a59e6afda689b0ec9e19071ababa66e4a443fbefbfffca72b7540b075b" -} diff --git a/prover/prover_dal/.sqlx/query-e78e94239dc10c5560f239a71e4d6a8a6de1e3a56e2a95963d933c21485c9939.json b/prover/prover_dal/.sqlx/query-e78e94239dc10c5560f239a71e4d6a8a6de1e3a56e2a95963d933c21485c9939.json deleted file mode 100644 index 35cec4af0682..000000000000 --- a/prover/prover_dal/.sqlx/query-e78e94239dc10c5560f239a71e4d6a8a6de1e3a56e2a95963d933c21485c9939.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n protocol_version,\n protocol_version_patch\n FROM\n prover_jobs_fri\n WHERE\n id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "protocol_version", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "protocol_version_patch", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - true, - false - ] - }, - "hash": "e78e94239dc10c5560f239a71e4d6a8a6de1e3a56e2a95963d933c21485c9939" -} diff --git a/prover/prover_dal/.sqlx/query-35b87a3b7db0af87c6a95e9fe7ef9044ae85b579c7051301b40bd5f94df1f530.json b/prover/prover_dal/.sqlx/query-e8066db420e075306235f728d57567878f347bdaf36294e9b24ee9c0aa1e861b.json similarity index 76% rename from prover/prover_dal/.sqlx/query-35b87a3b7db0af87c6a95e9fe7ef9044ae85b579c7051301b40bd5f94df1f530.json rename to prover/prover_dal/.sqlx/query-e8066db420e075306235f728d57567878f347bdaf36294e9b24ee9c0aa1e861b.json index a11e154326ec..422036ebb115 100644 --- a/prover/prover_dal/.sqlx/query-35b87a3b7db0af87c6a95e9fe7ef9044ae85b579c7051301b40bd5f94df1f530.json +++ b/prover/prover_dal/.sqlx/query-e8066db420e075306235f728d57567878f347bdaf36294e9b24ee9c0aa1e861b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE prover_jobs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n id = $2\n ", + "query": "\n UPDATE prover_jobs_fri\n SET\n status = 'failed',\n error = $1,\n updated_at = NOW()\n WHERE\n id = $2\n AND status != 'successful'\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "35b87a3b7db0af87c6a95e9fe7ef9044ae85b579c7051301b40bd5f94df1f530" + "hash": "e8066db420e075306235f728d57567878f347bdaf36294e9b24ee9c0aa1e861b" } diff --git a/prover/prover_dal/migrations/20240613111518_make_number_of_final_node_jobs_mandatory.down.sql b/prover/prover_dal/migrations/20240613111518_make_number_of_final_node_jobs_mandatory.down.sql new file mode 100644 index 000000000000..2bd4cea7b9ef --- /dev/null +++ b/prover/prover_dal/migrations/20240613111518_make_number_of_final_node_jobs_mandatory.down.sql @@ -0,0 +1 @@ +ALTER TABLE recursion_tip_witness_jobs_fri ALTER COLUMN number_of_final_node_jobs DROP NOT NULL; diff --git a/prover/prover_dal/migrations/20240613111518_make_number_of_final_node_jobs_mandatory.up.sql b/prover/prover_dal/migrations/20240613111518_make_number_of_final_node_jobs_mandatory.up.sql new file mode 100644 index 000000000000..ed3a1703b00a --- /dev/null +++ b/prover/prover_dal/migrations/20240613111518_make_number_of_final_node_jobs_mandatory.up.sql @@ -0,0 +1 @@ +ALTER TABLE recursion_tip_witness_jobs_fri ALTER COLUMN number_of_final_node_jobs SET NOT NULL; diff --git a/prover/prover_dal/src/fri_proof_compressor_dal.rs b/prover/prover_dal/src/fri_proof_compressor_dal.rs index 38f09114f2bf..7adc08b680dc 100644 --- a/prover/prover_dal/src/fri_proof_compressor_dal.rs +++ b/prover/prover_dal/src/fri_proof_compressor_dal.rs @@ -51,23 +51,6 @@ impl FriProofCompressorDal<'_, '_> { .unwrap(); } - pub async fn skip_proof_compression_job(&mut self, block_number: L1BatchNumber) { - sqlx::query!( - r#" - INSERT INTO - proof_compression_jobs_fri (l1_batch_number, status, created_at, updated_at) - VALUES - ($1, $2, NOW(), NOW()) - ON CONFLICT (l1_batch_number) DO NOTHING - "#, - i64::from(block_number.0), - ProofCompressionJobStatus::Skipped.to_string(), - ) - .fetch_optional(self.storage.conn()) - .await - .unwrap(); - } - pub async fn get_next_proof_compression_job( &mut self, picked_by: &str, @@ -177,10 +160,14 @@ impl FriProofCompressorDal<'_, '_> { updated_at = NOW() WHERE l1_batch_number = $3 + AND status != $4 + AND status != $5 "#, ProofCompressionJobStatus::Failed.to_string(), error, - i64::from(block_number.0) + i64::from(block_number.0), + ProofCompressionJobStatus::Successful.to_string(), + ProofCompressionJobStatus::SentToServer.to_string(), ) .execute(self.storage.conn()) .await diff --git a/prover/prover_dal/src/fri_prover_dal.rs b/prover/prover_dal/src/fri_prover_dal.rs index 2dfb0f7e0baa..f6c0379ee8a0 100644 --- a/prover/prover_dal/src/fri_prover_dal.rs +++ b/prover/prover_dal/src/fri_prover_dal.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, convert::TryFrom, str::FromStr, time::Duration}; use zksync_basic_types::{ basic_fri_types::{AggregationRound, CircuitIdRoundTuple, JobIdentifiers}, - protocol_version::{ProtocolSemanticVersion, ProtocolVersionId, VersionPatch}, + protocol_version::{ProtocolSemanticVersion, ProtocolVersionId}, prover_dal::{ correct_circuit_id, FriProverJobMetadata, JobCountStatistics, ProverJobFriInfo, ProverJobStatus, StuckJobs, @@ -211,6 +211,7 @@ impl FriProverDal<'_, '_> { updated_at = NOW() WHERE id = $2 + AND status != 'successful' "#, error, i64::from(id) @@ -520,6 +521,7 @@ impl FriProverDal<'_, '_> { updated_at = NOW() WHERE id = $2 + AND status != 'successful' "#, status, i64::from(id) @@ -529,23 +531,6 @@ impl FriProverDal<'_, '_> { .unwrap(); } - pub async fn save_successful_sent_proof(&mut self, l1_batch_number: L1BatchNumber) { - sqlx::query!( - r#" - UPDATE prover_jobs_fri - SET - status = 'sent_to_server', - updated_at = NOW() - WHERE - l1_batch_number = $1 - "#, - i64::from(l1_batch_number.0) - ) - .execute(self.storage.conn()) - .await - .unwrap(); - } - pub async fn get_scheduler_proof_job_id( &mut self, l1_batch_number: L1BatchNumber, @@ -698,29 +683,6 @@ impl FriProverDal<'_, '_> { .collect() } - pub async fn protocol_version_for_job(&mut self, job_id: u32) -> ProtocolSemanticVersion { - let result = sqlx::query!( - r#" - SELECT - protocol_version, - protocol_version_patch - FROM - prover_jobs_fri - WHERE - id = $1 - "#, - job_id as i32 - ) - .fetch_one(self.storage.conn()) - .await - .unwrap(); - - ProtocolSemanticVersion::new( - ProtocolVersionId::try_from(result.protocol_version.unwrap() as u16).unwrap(), - VersionPatch(result.protocol_version_patch as u32), - ) - } - pub async fn delete_prover_jobs_fri_batch_data( &mut self, l1_batch_number: L1BatchNumber, diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index 3c733623e477..14d47beed1a0 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -171,6 +171,7 @@ impl FriWitnessGeneratorDal<'_, '_> { updated_at = NOW() WHERE l1_batch_number = $2 + AND status != 'successful' "#, status.to_string(), i64::from(block_number.0) @@ -213,6 +214,7 @@ impl FriWitnessGeneratorDal<'_, '_> { updated_at = NOW() WHERE l1_batch_number = $2 + AND status != 'successful' "#, error, i64::from(block_number.0) @@ -232,6 +234,7 @@ impl FriWitnessGeneratorDal<'_, '_> { updated_at = NOW() WHERE id = $2 + AND status != 'successful' "#, error, i64::from(id) @@ -719,6 +722,7 @@ impl FriWitnessGeneratorDal<'_, '_> { updated_at = NOW() WHERE id = $2 + AND status != 'successful' "#, error, i64::from(id) @@ -1084,7 +1088,7 @@ impl FriWitnessGeneratorDal<'_, '_> { &mut self, protocol_version: ProtocolSemanticVersion, picked_by: &str, - ) -> Option { + ) -> Option<(L1BatchNumber, i32)> { sqlx::query!( r#" UPDATE recursion_tip_witness_jobs_fri @@ -1112,7 +1116,8 @@ impl FriWitnessGeneratorDal<'_, '_> { SKIP LOCKED ) RETURNING - recursion_tip_witness_jobs_fri.l1_batch_number + recursion_tip_witness_jobs_fri.l1_batch_number, + recursion_tip_witness_jobs_fri.number_of_final_node_jobs "#, protocol_version.minor as i32, protocol_version.patch.0 as i32, @@ -1121,7 +1126,12 @@ impl FriWitnessGeneratorDal<'_, '_> { .fetch_optional(self.storage.conn()) .await .unwrap() - .map(|row| L1BatchNumber(row.l1_batch_number as u32)) + .map(|row| { + ( + L1BatchNumber(row.l1_batch_number as u32), + row.number_of_final_node_jobs, + ) + }) } pub async fn mark_scheduler_jobs_as_queued(&mut self, l1_batch_number: i64) { @@ -1334,6 +1344,7 @@ impl FriWitnessGeneratorDal<'_, '_> { updated_at = NOW() WHERE l1_batch_number = $2 + AND status != 'successful' "#, error, l1_batch_number.0 as i64 @@ -1353,6 +1364,7 @@ impl FriWitnessGeneratorDal<'_, '_> { updated_at = NOW() WHERE l1_batch_number = $2 + AND status != 'successful' "#, error, i64::from(block_number.0) diff --git a/prover/witness_generator/src/recursion_tip.rs b/prover/witness_generator/src/recursion_tip.rs index e9291b5b182f..a44661d55aaf 100644 --- a/prover/witness_generator/src/recursion_tip.rs +++ b/prover/witness_generator/src/recursion_tip.rs @@ -143,7 +143,7 @@ impl JobProcessor for RecursionTipWitnessGenerator { async fn get_next_job(&self) -> anyhow::Result> { let mut prover_connection = self.prover_connection_pool.connection().await?; let pod_name = get_current_pod_name(); - let Some(l1_batch_number) = prover_connection + let Some((l1_batch_number, number_of_final_node_jobs)) = prover_connection .fri_witness_generator_dal() .get_next_recursion_tip_witness_job(self.protocol_version, &pod_name) .await @@ -156,6 +156,13 @@ impl JobProcessor for RecursionTipWitnessGenerator { .get_final_node_proof_job_ids_for(l1_batch_number) .await; + assert_eq!( + final_node_proof_job_ids.len(), + number_of_final_node_jobs as usize, + "recursion tip witness job was scheduled without all final node jobs being completed; expected {}, got {}", + number_of_final_node_jobs, final_node_proof_job_ids.len() + ); + Ok(Some(( l1_batch_number, prepare_job(