diff --git a/barretenberg/cpp/.gitignore b/barretenberg/cpp/.gitignore index 3af23ba15cd..5bd083b2335 100644 --- a/barretenberg/cpp/.gitignore +++ b/barretenberg/cpp/.gitignore @@ -3,9 +3,6 @@ build*/ src/wasi-sdk-* src/aztec/proof_system/proving_key/fixtures src/aztec/rollup/proofs/*/fixtures -srs_db/ignition/transcript* -srs_db/lagrange -srs_db/coset_lagrange -srs_db/modified_lagrange +srs_db/*/*/transcript* # to be unignored when we agree on clang-tidy rules .clangd \ No newline at end of file diff --git a/barretenberg/cpp/src/aztec/ecc/curves/bn254/scalar_multiplication/pippenger.cpp b/barretenberg/cpp/src/aztec/ecc/curves/bn254/scalar_multiplication/pippenger.cpp index 4b7d0f2a485..4c4b672210a 100644 --- a/barretenberg/cpp/src/aztec/ecc/curves/bn254/scalar_multiplication/pippenger.cpp +++ b/barretenberg/cpp/src/aztec/ecc/curves/bn254/scalar_multiplication/pippenger.cpp @@ -7,24 +7,16 @@ Pippenger::Pippenger(g1::affine_element* points, size_t num_points) : monomials_(points) , num_points_(num_points) { - monomials_[0] = barretenberg::g1::affine_one; - io::byteswap(&monomials_[1], (num_points - 1) * 64); - + io::byteswap(&monomials_[0], num_points * 64); scalar_multiplication::generate_pippenger_point_table(monomials_, monomials_, num_points); } -Pippenger::Pippenger(uint8_t const* points, size_t num_points, bool is_lagrange) +Pippenger::Pippenger(uint8_t const* points, size_t num_points, bool) : num_points_(num_points) { monomials_ = point_table_alloc(num_points); - size_t index = 0; - if (!is_lagrange) { - monomials_[0] = barretenberg::g1::affine_one; - index = 1; - } - - barretenberg::io::read_g1_elements_from_buffer(&monomials_[index], (char*)points, (num_points - index) * 64); + barretenberg::io::read_g1_elements_from_buffer(&monomials_[0], (char*)points, num_points * 64); barretenberg::scalar_multiplication::generate_pippenger_point_table(monomials_, monomials_, num_points); } diff --git a/barretenberg/cpp/src/aztec/lagrange_base_gen/lagrange_base_processor.cpp b/barretenberg/cpp/src/aztec/lagrange_base_gen/lagrange_base_processor.cpp index c2a7e7e1b94..24ca874a35b 100644 --- a/barretenberg/cpp/src/aztec/lagrange_base_gen/lagrange_base_processor.cpp +++ b/barretenberg/cpp/src/aztec/lagrange_base_gen/lagrange_base_processor.cpp @@ -5,6 +5,7 @@ #include #include #include +#include int main(int argc, char** argv) { @@ -44,10 +45,26 @@ int main(int argc, char** argv) g2_elements.push_back(verifier_ref_string->get_g2x()); g2_elements.push_back(barretenberg::g2::affine_one); - barretenberg::io::Manifest manifest{ - 0, 1, static_cast(subgroup_size), 2, static_cast(subgroup_size), 2, 0 - }; - barretenberg::io::write_transcript(&lagrange_base_srs[0], &g2_elements[0], manifest, lagrange_srs_path, true); + const uint32_t subgroup_size_ = static_cast(subgroup_size); + const uint32_t max_lagrange_transcript_size = static_cast((size_t(1) << 24)); + + // If the transcript size is > 2^24, write the Lagrange srs into individual transcripts each containing 2^24 + // points. This means that the individual lagrange transcripts will have a max size of 1.1 GB. + const uint32_t total_transcripts = std::max(uint32_t(1), subgroup_size_ / max_lagrange_transcript_size); + const uint32_t num_g1_points_ = std::min(subgroup_size_, max_lagrange_transcript_size); + size_t start_idx = 0; + for (uint32_t i = 0; i < total_transcripts; i++) { + barretenberg::io::Manifest manifest{ .transcript_number = i, + .total_transcripts = total_transcripts, + .total_g1_points = subgroup_size_, + .total_g2_points = 2, + .num_g1_points = num_g1_points_, + .num_g2_points = 2, + .start_from = 0 }; + barretenberg::io::write_transcript( + &lagrange_base_srs[start_idx], &g2_elements[0], manifest, lagrange_srs_path, true); + start_idx += max_lagrange_transcript_size; + } return 0; -} \ No newline at end of file +} diff --git a/barretenberg/cpp/src/aztec/srs/io.cpp b/barretenberg/cpp/src/aztec/srs/io.cpp index a84b5d24b76..91cd81a7ef8 100644 --- a/barretenberg/cpp/src/aztec/srs/io.cpp +++ b/barretenberg/cpp/src/aztec/srs/io.cpp @@ -128,14 +128,15 @@ void read_file_into_buffer( std::string get_transcript_path(std::string const& dir, size_t num) { - return dir + "/transcript" + (num < 10 ? "0" : "") + std::to_string(num) + ".dat"; + return format(dir, "/monomial/transcript", (num < 10) ? "0" : "", std::to_string(num), ".dat"); }; -std::string get_lagrange_transcript_path(std::string const& dir, size_t degree) +std::string get_lagrange_transcript_path(std::string const& dir, size_t degree, size_t num = 0) { auto log2_n = static_cast(numeric::get_msb(degree)); auto new_degree = (1 << log2_n); - return dir + "/transcript_" + std::to_string(new_degree) + ".dat"; + std::string suffix_str = (num > 0) ? format("_", std::to_string(num)) : ""; + return format(dir, "/lagrange/transcript_", std::to_string(new_degree), suffix_str, ".dat"); }; bool is_file_exist(std::string const& fileName) @@ -148,14 +149,7 @@ void read_transcript_g1(g1::affine_element* monomials, size_t degree, std::strin { size_t num = 0; size_t num_read = 0; - std::string path = get_lagrange_transcript_path(dir, degree); - if (!is_lagrange) { - // read g1 elements at second array position - first point is the basic generato - // This is true for monomial base srs. We start from index 0 for lagrange base transcript. - monomials[0] = g1::affine_one; - num_read = 1; - path = get_transcript_path(dir, num); - } + std::string path = is_lagrange ? get_lagrange_transcript_path(dir, degree, num) : get_transcript_path(dir, num); while (is_file_exist(path) && num_read < degree) { Manifest manifest; @@ -174,14 +168,16 @@ void read_transcript_g1(g1::affine_element* monomials, size_t degree, std::strin byteswap(&monomials[num_read], size); num_read += num_to_read; - path = get_transcript_path(dir, ++num); + path = is_lagrange ? get_lagrange_transcript_path(dir, degree, ++num) : get_transcript_path(dir, ++num); } const bool monomial_srs_condition = (num_read < degree) & (!is_lagrange); if (monomial_srs_condition) { throw_or_abort(format("Only read ", num_read, - " points but require ", + " points from ", + path, + ", but require ", degree, ". Is your srs large enough? Either run bootstrap.sh to download the transcript.dat " "files to `srs_db/ignition/`, or you might need to download extra transcript.dat files " @@ -194,7 +190,7 @@ void read_transcript_g2(g2::affine_element& g2_x, std::string const& dir, bool i { const size_t g2_size = sizeof(fq2) * 2; - std::string path = dir + "/g2.dat"; + std::string path = format(dir, "/g2.dat"); if (is_file_exist(path)) { char* buffer = (char*)&g2_x; @@ -320,13 +316,13 @@ void write_transcript(g1::affine_element const* g1_x, { const size_t num_g1_x = manifest.num_g1_points; const size_t num_g2_x = manifest.num_g2_points; + const size_t transcript_num = manifest.transcript_number; const size_t manifest_size = sizeof(Manifest); const size_t g1_buffer_size = sizeof(fq) * 2 * num_g1_x; const size_t g2_buffer_size = sizeof(fq) * 4 * num_g2_x; const size_t transcript_size = manifest_size + g1_buffer_size + g2_buffer_size; - std::string path = is_lagrange ? get_lagrange_transcript_path(dir, num_g1_x) : get_transcript_path(dir, 0); - // const size_t transcript_size = manifest_size + g1_buffer_size + g2_buffer_size + - // checksum::BLAKE2B_CHECKSUM_LENGTH; + std::string path = is_lagrange ? get_lagrange_transcript_path(dir, num_g1_x, transcript_num) + : get_transcript_path(dir, transcript_num); std::vector buffer(transcript_size); Manifest net_manifest; @@ -342,7 +338,6 @@ void write_transcript(g1::affine_element const* g1_x, write_g1_elements_to_buffer(g1_x, &buffer[manifest_size], num_g1_x); write_g2_elements_to_buffer(g2_x, &buffer[manifest_size + g1_buffer_size], num_g2_x); - // add_checksum_to_buffer(&buffer[0], manifest_size + g1_buffer_size + g2_buffer_size); write_buffer_to_file(path, &buffer[0], transcript_size); } diff --git a/barretenberg/cpp/src/aztec/srs/io.test.cpp b/barretenberg/cpp/src/aztec/srs/io.test.cpp index 078079d6c60..e0c71695531 100644 --- a/barretenberg/cpp/src/aztec/srs/io.test.cpp +++ b/barretenberg/cpp/src/aztec/srs/io.test.cpp @@ -13,6 +13,8 @@ TEST(io, read_transcript_loads_well_formed_srs) g2::affine_element g2_x; io::read_transcript(monomials, g2_x, degree, "../srs_db/ignition"); + EXPECT_EQ(g1::affine_one, monomials[0]); + g1::affine_element P[2]; g2::affine_element Q[2]; P[0] = monomials[1]; diff --git a/barretenberg/cpp/src/aztec/srs/lagrange_base_transformation/lagrange_base.cpp b/barretenberg/cpp/src/aztec/srs/lagrange_base_transformation/lagrange_base.cpp index 3afb9707448..ff46b1a16bc 100644 --- a/barretenberg/cpp/src/aztec/srs/lagrange_base_transformation/lagrange_base.cpp +++ b/barretenberg/cpp/src/aztec/srs/lagrange_base_transformation/lagrange_base.cpp @@ -35,15 +35,15 @@ void transform_srs(g1::affine_element* monomials, g1::affine_element* lagrange_b barretenberg::fr root = domain.root_inverse; std::vector monomials_jac(degree); for (size_t i = 0; i < degree; ++i) { - monomials_jac[i] = g1::element(monomials[i].x, monomials[i].y, g1::one.z ); + monomials_jac[i] = g1::element(monomials[i].x, monomials[i].y, g1::one.z); } auto lagrange_jac = g1fft(&monomials_jac[0], degree, root, 1); for (size_t i = 0; i < degree - 1; ++i) { lagrange_base_affine[i + 1] = static_cast(lagrange_jac[i] * domain.domain_inverse); } - lagrange_base_affine[0] = static_cast(lagrange_jac[degree-1] * domain.domain_inverse); + lagrange_base_affine[0] = static_cast(lagrange_jac[degree - 1] * domain.domain_inverse); } } // namespace lagrange_base -} \ No newline at end of file +} // namespace barretenberg \ No newline at end of file diff --git a/barretenberg/cpp/src/aztec/srs/lagrange_base_transformation/lagrange_base.test.cpp b/barretenberg/cpp/src/aztec/srs/lagrange_base_transformation/lagrange_base.test.cpp index 28d471186ec..98a81ee32fb 100644 --- a/barretenberg/cpp/src/aztec/srs/lagrange_base_transformation/lagrange_base.test.cpp +++ b/barretenberg/cpp/src/aztec/srs/lagrange_base_transformation/lagrange_base.test.cpp @@ -154,12 +154,10 @@ TEST(lagrange_base, verify_lagrange_base_import_srs) EXPECT_EQ(result, expected); } -void test_lagrange_transcripts_helper(const size_t degree, - const std::string monomial_path, - const std::string lagrange_path) +void test_lagrange_transcripts_helper(const size_t degree, const std::string path) { - auto lagrange_reference_string = scalar_multiplication::Pippenger(lagrange_path, degree, true); - auto monomial_reference_string = scalar_multiplication::Pippenger(monomial_path, degree, false); + auto lagrange_reference_string = scalar_multiplication::Pippenger(path, degree, true); + auto monomial_reference_string = scalar_multiplication::Pippenger(path, degree, false); barretenberg::evaluation_domain domain(degree); domain.compute_lookup_table(); @@ -178,8 +176,8 @@ void test_lagrange_transcripts_helper(const size_t degree, g2::affine_element lagrange_g2x; g2::affine_element monomial_g2x; - barretenberg::io::read_transcript_g2(lagrange_g2x, lagrange_path, true); - barretenberg::io::read_transcript_g2(monomial_g2x, monomial_path, false); + barretenberg::io::read_transcript_g2(lagrange_g2x, path, true); + barretenberg::io::read_transcript_g2(monomial_g2x, path, false); EXPECT_EQ(lagrange_g2x, monomial_g2x); } @@ -200,7 +198,7 @@ HEAVY_TEST(lagrange_base, test_local_lagrange_transcripts) for (size_t i = 0; i < num_files; i++) { const size_t degree = static_cast(1 << (i + 1)); auto begin = std::chrono::steady_clock::now(); - test_lagrange_transcripts_helper(degree, "../srs_db/ignition", "../srs_db/lagrange"); + test_lagrange_transcripts_helper(degree, "../srs_db/ignition"); auto end = std::chrono::steady_clock::now(); std::cout << "Verified Lagrange transcript of size " << degree << " in " diff --git a/barretenberg/cpp/src/aztec/srs/reference_string/mem_reference_string.test.cpp b/barretenberg/cpp/src/aztec/srs/reference_string/mem_reference_string.test.cpp index 642d3b2066b..35ee1a28ef6 100644 --- a/barretenberg/cpp/src/aztec/srs/reference_string/mem_reference_string.test.cpp +++ b/barretenberg/cpp/src/aztec/srs/reference_string/mem_reference_string.test.cpp @@ -10,8 +10,8 @@ TEST(reference_string, mem_file_consistency) { std::ifstream transcript; - int NUM_POINTS_IN_TRANSCRIPT = 5040000; - transcript.open("../srs_db/ignition/transcript00.dat", std::ifstream::binary); + int NUM_POINTS_IN_TRANSCRIPT = 5040001; + transcript.open("../srs_db/ignition/monomial/transcript00.dat", std::ifstream::binary); std::vector monomials(32768 * 64); std::vector g2x(128); transcript.seekg(28); diff --git a/barretenberg/cpp/srs_db/download_ignition.sh b/barretenberg/cpp/srs_db/download_ignition.sh index b7035ba60d3..1632c035781 100755 --- a/barretenberg/cpp/srs_db/download_ignition.sh +++ b/barretenberg/cpp/srs_db/download_ignition.sh @@ -17,6 +17,8 @@ set -e mkdir -p ignition cd ignition +mkdir -p monomial +cd monomial NUM=${1:-19} if command -v sha256sum > /dev/null; then @@ -31,7 +33,7 @@ checksum() { } download() { - curl https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/sealed/transcript${1}.dat > transcript${1}.dat + curl https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat > transcript${1}.dat } for TRANSCRIPT in $(seq 0 $NUM); do diff --git a/barretenberg/cpp/srs_db/download_ignition_lagrange.sh b/barretenberg/cpp/srs_db/download_ignition_lagrange.sh index 855bb51c707..8dc9cb76f62 100755 --- a/barretenberg/cpp/srs_db/download_ignition_lagrange.sh +++ b/barretenberg/cpp/srs_db/download_ignition_lagrange.sh @@ -1,7 +1,7 @@ #!/bin/sh # Downloads the Lagrange transcripts generated from ignition trusted setup. # -# To download all transcripts upto 2^{24}. +# To download all transcripts upto 2^{25}. # ./download_ignition_lagrange.sh # # To download select transcripts upto size 2^m, @@ -12,9 +12,10 @@ # whatever is requested but does not check the validity of the downloads. set -e +cd ignition mkdir -p lagrange cd lagrange -num_transcripts=${1:-24} +num_transcripts=${1:-25} ARGS=$(seq 1 $num_transcripts) checksum() { @@ -26,13 +27,29 @@ download() { curl https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/lagrange/transcript_${1}.dat > transcript_${1}.dat } -for TRANSCRIPT in $ARGS; do - NUM=$(printf %2d $((1 << $TRANSCRIPT))) +get_transcript() { if [ -f checksums ]; then - checksum $NUM && continue - download $NUM - checksum $NUM || exit 1 + checksum ${1} && return 0 + download ${1} + checksum ${1} || return 1 else - download $NUM + download ${1} + fi +} + +for TRANSCRIPT in $ARGS; do + NUM=$(printf %2d $((1 << $TRANSCRIPT))) + get_transcript $NUM + + # We need to handle Lagrange transcripts with > 2^24 points differently + # as they're split across multiple transcript files. + if [ $TRANSCRIPT -gt 24 ]; then + diff=$(($TRANSCRIPT - 24)) + num_sub_transcripts=$((1 << $diff - 1)) + for i in $(seq $num_sub_transcripts) + do + NUM_SUB="${NUM}_${i}" + get_transcript $NUM_SUB + done fi done diff --git a/barretenberg/cpp/srs_db/ignition/lagrange/checksums b/barretenberg/cpp/srs_db/ignition/lagrange/checksums new file mode 100644 index 00000000000..b8ae121997b --- /dev/null +++ b/barretenberg/cpp/srs_db/ignition/lagrange/checksums @@ -0,0 +1,27 @@ +227512f4233e1b068e5dbfcf1a7fbb8b03d18050844eec29cb34af283fa6a7c9 g2.dat +22bfbaf553eafc920078b26d41ab93ee1cb6fd333857f6080dd17f669e3f9f9b transcript_2.dat +83954bf47ecadb3bb05d8904fb65bca717d5c29ed55e720dd633a956bf101f8f transcript_4.dat +6caac617d66a61103389e4ff42f282817cb3a5bb42a7ebb7a82193cf87945aca transcript_8.dat +480b03a01fbdff93eff15277092e76d0f96552d7a95a6cf8155db9010e166401 transcript_16.dat +b85e4431eb83c7f09dce2be7da7bc8658565f2d7a49768b9367d2b46b5accc9f transcript_32.dat +5b9f1440bc74dad80865bdce14f0e1a0bf332a8612697a251df28ab018e1d73d transcript_64.dat +0e32cccf645a0f39196bde9df334b6db08249da8ed229049c5823e5ca96eb273 transcript_128.dat +95c99d976f455b555b48fab9dffbead96d21b73222761b1b24f27af82dcb7d16 transcript_256.dat +9b86ec33e7d025f3d72161320dfde80d1b24ff58212276bfc6b64afec0140d92 transcript_512.dat +408e690b0c2eeaea757af1e56388ca98b80edb96a795b6e84447c4711c6acc47 transcript_1024.dat +8a008912a54763e3f4ede6e5e5290c592709ca29f6f01100530fb48f7e6b27b6 transcript_2048.dat +0a691a36c92262677b2fe4416b7570c161b732c73f9083b95821468dbb160d16 transcript_4096.dat +2ad1e2d3ef8a6c4cfea5034903b62d31ed5a8d3234d5465b3463d2e403f5752d transcript_8192.dat +b8a6bf124112e882871a69b2ee1065c3b384ffc920af56edcc8475a7a8432d86 transcript_16384.dat +f7e22fd549bab9e669405fc7146630c688a87ea52ba5574d7c55db4ecb5f6b27 transcript_32768.dat +37dee3a7e792c211b2acfc767f24700044ae311756e4328306071ce410fa2f08 transcript_65536.dat +9caa2c2ff519cc4afd9b1e8865a604669f31026766c2fa24f7c29a113aa14c8c transcript_131072.dat +9e72008d43dda29e13bfd7c84044063f91a31e418a609ed4678b80815252282f transcript_262144.dat +53b8137464da802eec2acc637335be965c449d81ac415ad42c250cba7e0364c7 transcript_524288.dat +0a87714dbb5bba388eeb085c2eb6744215a5b48835a192a1a485c7e00ac763ce transcript_1048576.dat +8bc47bb1e36b5aea7a4a3d30c7c3c32529f5c4fb29dd8f906b3c17bfacf34ab6 transcript_2097152.dat +9b82696d6dbaf95cce3f5e2dcf04d83dd82987cdd84cbbf3a6ff8e42d6ae7c9e transcript_4194304.dat +06ed904322d468ae2412be7ee482b9149968627a21d592861d2c415051479be7 transcript_8388608.dat +998238ccac69a20620793a1a13cc43d4706c4859f2f5576ca29c645a9d2e666b transcript_16777216.dat +e3e1de0a97a6cd50a3959c795edd97253641621f867895199fca6b6852f22941 transcript_33554432.dat +6b580183d05b650517567b9c196ccec83f86bdefea021cbe885122ff41dda97a transcript_33554432_1.dat diff --git a/barretenberg/cpp/srs_db/ignition/g2.dat b/barretenberg/cpp/srs_db/ignition/lagrange/g2.dat similarity index 100% rename from barretenberg/cpp/srs_db/ignition/g2.dat rename to barretenberg/cpp/srs_db/ignition/lagrange/g2.dat diff --git a/barretenberg/cpp/srs_db/ignition/checksums b/barretenberg/cpp/srs_db/ignition/monomial/checksums similarity index 96% rename from barretenberg/cpp/srs_db/ignition/checksums rename to barretenberg/cpp/srs_db/ignition/monomial/checksums index 854d5d6e3fc..177b9441eb1 100644 --- a/barretenberg/cpp/srs_db/ignition/checksums +++ b/barretenberg/cpp/srs_db/ignition/monomial/checksums @@ -1,5 +1,5 @@ 227512f4233e1b068e5dbfcf1a7fbb8b03d18050844eec29cb34af283fa6a7c9 g2.dat -af247f77ebe9382c5ade03349737b651550a3548a78f49cddd27027f2b0df389 transcript00.dat +0f94b39426f5a57d1a17742624f7d3170a32e19d6c5e16f202281439dbe48695 transcript00.dat fe8919733b0ef279d31787f2c80e67a014dc50ad2790640157a7d411307784a3 transcript01.dat 0602f52fa0dd81a26cc5e1db5c068caf49118c30921238f41d8d16169041e15f transcript02.dat b02f5c2b6d6017f90f3389ba2796465a11f582b49bbc900e76583a64bc8b32a5 transcript03.dat diff --git a/barretenberg/cpp/srs_db/ignition/monomial/g2.dat b/barretenberg/cpp/srs_db/ignition/monomial/g2.dat new file mode 100644 index 00000000000..22bc78828b3 --- /dev/null +++ b/barretenberg/cpp/srs_db/ignition/monomial/g2.dat @@ -0,0 +1 @@ +~#쓈ƒ°ŸYD;2‹¼‰µ³˜µ—NÄÕ¸7¼ÂNþ0úÀ“ƒÁêQØz5Ž‹çÿNX‘Þè&²QöñÇ…J‡ÔÚÌ^UæÝ?–æ΢VG[Båa^"þ½£ÀÀc*îA<€Új_äœò FAù›¤ÒQVÁ»šr…üci÷ã \ No newline at end of file