SRS Transcript Cleanup (AztecProtocol/barretenberg#93)

* Remove conditional logic on [1]_1 in reading monomial transcript.

* Enable read/write of Lagrange transcripts in multiple files for sizes > 2^24.

* Enable >2^24 Lagrange transcript downloading.

Use new monomial transcript00.dat form new folder in s3 bucket.

* Fix download lagrange igniton script.

* attempt to fix gcc build

* Luke's suggestion.

* fix

---------

Co-authored-by: Suyash Bagad <[email protected]>

barretenberg/cpp/.gitignore

@@ -3,9 +3,6 @@ build*/
 src/wasi-sdk-*
 src/aztec/proof_system/proving_key/fixtures
 src/aztec/rollup/proofs/*/fixtures
-srs_db/ignition/transcript*
-srs_db/lagrange
-srs_db/coset_lagrange
-srs_db/modified_lagrange
+srs_db/*/*/transcript*
 # to be unignored when we agree on clang-tidy rules
 .clangd

barretenberg/cpp/src/aztec/ecc/curves/bn254/scalar_multiplication/pippenger.cpp

@@ -7,24 +7,16 @@ Pippenger::Pippenger(g1::affine_element* points, size_t num_points)
     : monomials_(points)
     , num_points_(num_points)
 {
-    monomials_[0] = barretenberg::g1::affine_one;
-    io::byteswap(&monomials_[1], (num_points - 1) * 64);
-
+    io::byteswap(&monomials_[0], num_points * 64);
     scalar_multiplication::generate_pippenger_point_table(monomials_, monomials_, num_points);
 }
 
-Pippenger::Pippenger(uint8_t const* points, size_t num_points, bool is_lagrange)
+Pippenger::Pippenger(uint8_t const* points, size_t num_points, bool)
     : num_points_(num_points)
 {
     monomials_ = point_table_alloc<g1::affine_element>(num_points);
 
-    size_t index = 0;
-    if (!is_lagrange) {
-        monomials_[0] = barretenberg::g1::affine_one;
-        index = 1;
-    }
-
-    barretenberg::io::read_g1_elements_from_buffer(&monomials_[index], (char*)points, (num_points - index) * 64);
+    barretenberg::io::read_g1_elements_from_buffer(&monomials_[0], (char*)points, num_points * 64);
     barretenberg::scalar_multiplication::generate_pippenger_point_table(monomials_, monomials_, num_points);
 }
 

barretenberg/cpp/src/aztec/lagrange_base_gen/lagrange_base_processor.cpp

@@ -5,6 +5,7 @@
 #include <ecc/curves/bn254/g2.hpp>
 #include <srs/io.hpp>
 #include <iostream>
+#include <algorithm>
 
 int main(int argc, char** argv)
 {
@@ -44,10 +45,26 @@ int main(int argc, char** argv)
     g2_elements.push_back(verifier_ref_string->get_g2x());
     g2_elements.push_back(barretenberg::g2::affine_one);
 
-    barretenberg::io::Manifest manifest{
-        0, 1, static_cast<uint32_t>(subgroup_size), 2, static_cast<uint32_t>(subgroup_size), 2, 0
-    };
-    barretenberg::io::write_transcript(&lagrange_base_srs[0], &g2_elements[0], manifest, lagrange_srs_path, true);
+    const uint32_t subgroup_size_ = static_cast<uint32_t>(subgroup_size);
+    const uint32_t max_lagrange_transcript_size = static_cast<uint32_t>((size_t(1) << 24));
+
+    // If the transcript size is > 2^24, write the Lagrange srs into individual transcripts each containing 2^24
+    // points. This means that the individual lagrange transcripts will have a max size of 1.1 GB.
+    const uint32_t total_transcripts = std::max(uint32_t(1), subgroup_size_ / max_lagrange_transcript_size);
+    const uint32_t num_g1_points_ = std::min(subgroup_size_, max_lagrange_transcript_size);
+    size_t start_idx = 0;
+    for (uint32_t i = 0; i < total_transcripts; i++) {
+        barretenberg::io::Manifest manifest{ .transcript_number = i,
+                                             .total_transcripts = total_transcripts,
+                                             .total_g1_points = subgroup_size_,
+                                             .total_g2_points = 2,
+                                             .num_g1_points = num_g1_points_,
+                                             .num_g2_points = 2,
+                                             .start_from = 0 };
+        barretenberg::io::write_transcript(
+            &lagrange_base_srs[start_idx], &g2_elements[0], manifest, lagrange_srs_path, true);
+        start_idx += max_lagrange_transcript_size;
+    }
 
     return 0;
-}
+}

barretenberg/cpp/src/aztec/srs/io.cpp

@@ -128,14 +128,15 @@ void read_file_into_buffer(
 
 std::string get_transcript_path(std::string const& dir, size_t num)
 {
-    return dir + "/transcript" + (num < 10 ? "0" : "") + std::to_string(num) + ".dat";
+    return format(dir, "/monomial/transcript", (num < 10) ? "0" : "", std::to_string(num), ".dat");
 };
 
-std::string get_lagrange_transcript_path(std::string const& dir, size_t degree)
+std::string get_lagrange_transcript_path(std::string const& dir, size_t degree, size_t num = 0)
 {
     auto log2_n = static_cast<uint32_t>(numeric::get_msb(degree));
     auto new_degree = (1 << log2_n);
-    return dir + "/transcript_" + std::to_string(new_degree) + ".dat";
+    std::string suffix_str = (num > 0) ? format("_", std::to_string(num)) : "";
+    return format(dir, "/lagrange/transcript_", std::to_string(new_degree), suffix_str, ".dat");
 };
 
 bool is_file_exist(std::string const& fileName)
@@ -148,14 +149,7 @@ void read_transcript_g1(g1::affine_element* monomials, size_t degree, std::strin
 {
     size_t num = 0;
     size_t num_read = 0;
-    std::string path = get_lagrange_transcript_path(dir, degree);
-    if (!is_lagrange) {
-        // read g1 elements at second array position - first point is the basic generato
-        // This is true for monomial base srs. We start from index 0 for lagrange base transcript.
-        monomials[0] = g1::affine_one;
-        num_read = 1;
-        path = get_transcript_path(dir, num);
-    }
+    std::string path = is_lagrange ? get_lagrange_transcript_path(dir, degree, num) : get_transcript_path(dir, num);
 
     while (is_file_exist(path) && num_read < degree) {
         Manifest manifest;
@@ -174,14 +168,16 @@ void read_transcript_g1(g1::affine_element* monomials, size_t degree, std::strin
         byteswap(&monomials[num_read], size);
 
         num_read += num_to_read;
-        path = get_transcript_path(dir, ++num);
+        path = is_lagrange ? get_lagrange_transcript_path(dir, degree, ++num) : get_transcript_path(dir, ++num);
     }
 
     const bool monomial_srs_condition = (num_read < degree) & (!is_lagrange);
     if (monomial_srs_condition) {
         throw_or_abort(format("Only read ",
                               num_read,
-                              " points but require ",
+                              " points from ",
+                              path,
+                              ", but require ",
                               degree,
                               ". Is your srs large enough? Either run bootstrap.sh to download the transcript.dat "
                               "files to `srs_db/ignition/`, or you might need to download extra transcript.dat files "
@@ -194,7 +190,7 @@ void read_transcript_g2(g2::affine_element& g2_x, std::string const& dir, bool i
 {
 
     const size_t g2_size = sizeof(fq2) * 2;
-    std::string path = dir + "/g2.dat";
+    std::string path = format(dir, "/g2.dat");
 
     if (is_file_exist(path)) {
         char* buffer = (char*)&g2_x;
@@ -320,13 +316,13 @@ void write_transcript(g1::affine_element const* g1_x,
 {
     const size_t num_g1_x = manifest.num_g1_points;
     const size_t num_g2_x = manifest.num_g2_points;
+    const size_t transcript_num = manifest.transcript_number;
     const size_t manifest_size = sizeof(Manifest);
     const size_t g1_buffer_size = sizeof(fq) * 2 * num_g1_x;
     const size_t g2_buffer_size = sizeof(fq) * 4 * num_g2_x;
     const size_t transcript_size = manifest_size + g1_buffer_size + g2_buffer_size;
-    std::string path = is_lagrange ? get_lagrange_transcript_path(dir, num_g1_x) : get_transcript_path(dir, 0);
-    // const size_t transcript_size = manifest_size + g1_buffer_size + g2_buffer_size +
-    // checksum::BLAKE2B_CHECKSUM_LENGTH;
+    std::string path = is_lagrange ? get_lagrange_transcript_path(dir, num_g1_x, transcript_num)
+                                   : get_transcript_path(dir, transcript_num);
     std::vector<char> buffer(transcript_size);
 
     Manifest net_manifest;
@@ -342,7 +338,6 @@ void write_transcript(g1::affine_element const* g1_x,
 
     write_g1_elements_to_buffer(g1_x, &buffer[manifest_size], num_g1_x);
     write_g2_elements_to_buffer(g2_x, &buffer[manifest_size + g1_buffer_size], num_g2_x);
-    // add_checksum_to_buffer(&buffer[0], manifest_size + g1_buffer_size + g2_buffer_size);
     write_buffer_to_file(path, &buffer[0], transcript_size);
 }
 

barretenberg/cpp/src/aztec/srs/io.test.cpp

@@ -13,6 +13,8 @@ TEST(io, read_transcript_loads_well_formed_srs)
     g2::affine_element g2_x;
     io::read_transcript(monomials, g2_x, degree, "../srs_db/ignition");
 
+    EXPECT_EQ(g1::affine_one, monomials[0]);
+
     g1::affine_element P[2];
     g2::affine_element Q[2];
     P[0] = monomials[1];

barretenberg/cpp/src/aztec/srs/lagrange_base_transformation/lagrange_base.cpp

@@ -35,15 +35,15 @@ void transform_srs(g1::affine_element* monomials, g1::affine_element* lagrange_b
     barretenberg::fr root = domain.root_inverse;
     std::vector<g1::element> monomials_jac(degree);
     for (size_t i = 0; i < degree; ++i) {
-        monomials_jac[i] = g1::element(monomials[i].x, monomials[i].y, g1::one.z );
+        monomials_jac[i] = g1::element(monomials[i].x, monomials[i].y, g1::one.z);
     }
 
     auto lagrange_jac = g1fft(&monomials_jac[0], degree, root, 1);
     for (size_t i = 0; i < degree - 1; ++i) {
         lagrange_base_affine[i + 1] = static_cast<g1::affine_element>(lagrange_jac[i] * domain.domain_inverse);
     }
-    lagrange_base_affine[0] = static_cast<g1::affine_element>(lagrange_jac[degree-1] * domain.domain_inverse);
+    lagrange_base_affine[0] = static_cast<g1::affine_element>(lagrange_jac[degree - 1] * domain.domain_inverse);
 }
 
 } // namespace lagrange_base
-}
+} // namespace barretenberg

barretenberg/cpp/src/aztec/srs/lagrange_base_transformation/lagrange_base.test.cpp

@@ -154,12 +154,10 @@ TEST(lagrange_base, verify_lagrange_base_import_srs)
     EXPECT_EQ(result, expected);
 }
 
-void test_lagrange_transcripts_helper(const size_t degree,
-                                      const std::string monomial_path,
-                                      const std::string lagrange_path)
+void test_lagrange_transcripts_helper(const size_t degree, const std::string path)
 {
-    auto lagrange_reference_string = scalar_multiplication::Pippenger(lagrange_path, degree, true);
-    auto monomial_reference_string = scalar_multiplication::Pippenger(monomial_path, degree, false);
+    auto lagrange_reference_string = scalar_multiplication::Pippenger(path, degree, true);
+    auto monomial_reference_string = scalar_multiplication::Pippenger(path, degree, false);
 
     barretenberg::evaluation_domain domain(degree);
     domain.compute_lookup_table();
@@ -178,8 +176,8 @@ void test_lagrange_transcripts_helper(const size_t degree,
 
     g2::affine_element lagrange_g2x;
     g2::affine_element monomial_g2x;
-    barretenberg::io::read_transcript_g2(lagrange_g2x, lagrange_path, true);
-    barretenberg::io::read_transcript_g2(monomial_g2x, monomial_path, false);
+    barretenberg::io::read_transcript_g2(lagrange_g2x, path, true);
+    barretenberg::io::read_transcript_g2(monomial_g2x, path, false);
 
     EXPECT_EQ(lagrange_g2x, monomial_g2x);
 }
@@ -200,7 +198,7 @@ HEAVY_TEST(lagrange_base, test_local_lagrange_transcripts)
     for (size_t i = 0; i < num_files; i++) {
         const size_t degree = static_cast<size_t>(1 << (i + 1));
         auto begin = std::chrono::steady_clock::now();
-        test_lagrange_transcripts_helper(degree, "../srs_db/ignition", "../srs_db/lagrange");
+        test_lagrange_transcripts_helper(degree, "../srs_db/ignition");
         auto end = std::chrono::steady_clock::now();
 
         std::cout << "Verified Lagrange transcript of size " << degree << " in "

barretenberg/cpp/src/aztec/srs/reference_string/mem_reference_string.test.cpp

@@ -10,8 +10,8 @@
 TEST(reference_string, mem_file_consistency)
 {
     std::ifstream transcript;
-    int NUM_POINTS_IN_TRANSCRIPT = 5040000;
-    transcript.open("../srs_db/ignition/transcript00.dat", std::ifstream::binary);
+    int NUM_POINTS_IN_TRANSCRIPT = 5040001;
+    transcript.open("../srs_db/ignition/monomial/transcript00.dat", std::ifstream::binary);
     std::vector<uint8_t> monomials(32768 * 64);
     std::vector<uint8_t> g2x(128);
     transcript.seekg(28);

barretenberg/cpp/srs_db/download_ignition.sh

@@ -17,6 +17,8 @@ set -e
 
 mkdir -p ignition
 cd ignition
+mkdir -p monomial
+cd monomial
 NUM=${1:-19}
 
 if command -v sha256sum > /dev/null; then
@@ -31,7 +33,7 @@ checksum() {
 }
 
 download() {
-  curl https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/sealed/transcript${1}.dat > transcript${1}.dat
+  curl https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat > transcript${1}.dat
 }
 
 for TRANSCRIPT in $(seq 0 $NUM); do

barretenberg/cpp/srs_db/download_ignition_lagrange.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 # Downloads the Lagrange transcripts generated from ignition trusted setup.
 #
-# To download all transcripts upto 2^{24}.
+# To download all transcripts upto 2^{25}.
 #  ./download_ignition_lagrange.sh
 #
 # To download select transcripts upto size 2^m,
@@ -12,9 +12,10 @@
 # whatever is requested but does not check the validity of the downloads.
 set -e
 
+cd ignition
 mkdir -p lagrange
 cd lagrange
-num_transcripts=${1:-24}
+num_transcripts=${1:-25}
 ARGS=$(seq 1 $num_transcripts)
 
 checksum() {
@@ -26,13 +27,29 @@ download() {
   curl https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/lagrange/transcript_${1}.dat > transcript_${1}.dat
 }
 
-for TRANSCRIPT in $ARGS; do
-  NUM=$(printf %2d $((1 << $TRANSCRIPT)))
+get_transcript() {
   if [ -f checksums ]; then
-    checksum $NUM && continue
-    download $NUM
-    checksum $NUM || exit 1
+    checksum ${1} && return 0
+    download ${1}
+    checksum ${1} || return 1
   else
-    download $NUM
+    download ${1}
+  fi
+}
+
+for TRANSCRIPT in $ARGS; do
+  NUM=$(printf %2d $((1 << $TRANSCRIPT)))
+  get_transcript $NUM
+
+  # We need to handle Lagrange transcripts with > 2^24 points differently 
+  # as they're split across multiple transcript files.
+  if [ $TRANSCRIPT -gt 24 ]; then
+    diff=$(($TRANSCRIPT - 24)) 
+    num_sub_transcripts=$((1 << $diff - 1))
+    for i in $(seq $num_sub_transcripts)
+      do
+      NUM_SUB="${NUM}_${i}"
+      get_transcript $NUM_SUB
+    done
   fi
 done

barretenberg/cpp/srs_db/ignition/lagrange/checksums

@@ -0,0 +1,27 @@
+227512f4233e1b068e5dbfcf1a7fbb8b03d18050844eec29cb34af283fa6a7c9  g2.dat
+22bfbaf553eafc920078b26d41ab93ee1cb6fd333857f6080dd17f669e3f9f9b  transcript_2.dat
+83954bf47ecadb3bb05d8904fb65bca717d5c29ed55e720dd633a956bf101f8f  transcript_4.dat
+6caac617d66a61103389e4ff42f282817cb3a5bb42a7ebb7a82193cf87945aca  transcript_8.dat
+480b03a01fbdff93eff15277092e76d0f96552d7a95a6cf8155db9010e166401  transcript_16.dat
+b85e4431eb83c7f09dce2be7da7bc8658565f2d7a49768b9367d2b46b5accc9f  transcript_32.dat
+5b9f1440bc74dad80865bdce14f0e1a0bf332a8612697a251df28ab018e1d73d  transcript_64.dat
+0e32cccf645a0f39196bde9df334b6db08249da8ed229049c5823e5ca96eb273  transcript_128.dat
+95c99d976f455b555b48fab9dffbead96d21b73222761b1b24f27af82dcb7d16  transcript_256.dat
+9b86ec33e7d025f3d72161320dfde80d1b24ff58212276bfc6b64afec0140d92  transcript_512.dat
+408e690b0c2eeaea757af1e56388ca98b80edb96a795b6e84447c4711c6acc47  transcript_1024.dat
+8a008912a54763e3f4ede6e5e5290c592709ca29f6f01100530fb48f7e6b27b6  transcript_2048.dat
+0a691a36c92262677b2fe4416b7570c161b732c73f9083b95821468dbb160d16  transcript_4096.dat
+2ad1e2d3ef8a6c4cfea5034903b62d31ed5a8d3234d5465b3463d2e403f5752d  transcript_8192.dat
+b8a6bf124112e882871a69b2ee1065c3b384ffc920af56edcc8475a7a8432d86  transcript_16384.dat
+f7e22fd549bab9e669405fc7146630c688a87ea52ba5574d7c55db4ecb5f6b27  transcript_32768.dat
+37dee3a7e792c211b2acfc767f24700044ae311756e4328306071ce410fa2f08  transcript_65536.dat
+9caa2c2ff519cc4afd9b1e8865a604669f31026766c2fa24f7c29a113aa14c8c  transcript_131072.dat
+9e72008d43dda29e13bfd7c84044063f91a31e418a609ed4678b80815252282f  transcript_262144.dat
+53b8137464da802eec2acc637335be965c449d81ac415ad42c250cba7e0364c7  transcript_524288.dat
+0a87714dbb5bba388eeb085c2eb6744215a5b48835a192a1a485c7e00ac763ce  transcript_1048576.dat
+8bc47bb1e36b5aea7a4a3d30c7c3c32529f5c4fb29dd8f906b3c17bfacf34ab6  transcript_2097152.dat
+9b82696d6dbaf95cce3f5e2dcf04d83dd82987cdd84cbbf3a6ff8e42d6ae7c9e  transcript_4194304.dat
+06ed904322d468ae2412be7ee482b9149968627a21d592861d2c415051479be7  transcript_8388608.dat
+998238ccac69a20620793a1a13cc43d4706c4859f2f5576ca29c645a9d2e666b  transcript_16777216.dat
+e3e1de0a97a6cd50a3959c795edd97253641621f867895199fca6b6852f22941  transcript_33554432.dat
+6b580183d05b650517567b9c196ccec83f86bdefea021cbe885122ff41dda97a  transcript_33554432_1.dat

barretenberg/cpp/srs_db/ignition/checksums → barretenberg/cpp/srs_db/ignition/monomial/checksums

@@ -1,5 +1,5 @@
 227512f4233e1b068e5dbfcf1a7fbb8b03d18050844eec29cb34af283fa6a7c9  g2.dat
-af247f77ebe9382c5ade03349737b651550a3548a78f49cddd27027f2b0df389  transcript00.dat
+0f94b39426f5a57d1a17742624f7d3170a32e19d6c5e16f202281439dbe48695  transcript00.dat
 fe8919733b0ef279d31787f2c80e67a014dc50ad2790640157a7d411307784a3  transcript01.dat
 0602f52fa0dd81a26cc5e1db5c068caf49118c30921238f41d8d16169041e15f  transcript02.dat
 b02f5c2b6d6017f90f3389ba2796465a11f582b49bbc900e76583a64bc8b32a5  transcript03.dat

barretenberg/cpp/srs_db/ignition/monomial/g2.dat

@@ -0,0 +1 @@
+~#쓈���YD;2��������N
�ո7��N�0������Q�z5����NX���&
�Q��DžJ����^U��?��΢VG[B�a^"�����c*�A<��j_��FA����QV���r��ci��