From c5d8457d859ff09a58e9b559c6c46ff7df37d658 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 21 Dec 2023 16:41:04 +0000 Subject: [PATCH] chore: Use simple "flat" CRS. (#3748) There's no need for us to have such a convoluted set of transcripts for our CRS in a complicated format. This: * Uses a single 6GB "flat" CRS with no header data in the ignition s3 bucket, in normalised big-endian form. * Allows us to use our normal deserialization functions for the fields, as opposed to specific custom SRS reading code. * Means we can now download, without complexity, any size CRS we need. * Changes names to eg `bn254_g1.dat` so doesn't conflict with old cache. * Get's rid of `size` file and just directly analyses file sizes. It doesn't address: * The new grumpkin CRS that arrived yesterday. Someone else can tackle that one. We also need to figure out how to generate a real one. Another trusted setup??.... --- .../cpp/src/barretenberg/bb/file_io.hpp | 27 ++++-- .../cpp/src/barretenberg/bb/get_bn254_crs.cpp | 94 ++++++------------- .../src/barretenberg/bb/get_grumpkin_crs.cpp | 5 +- barretenberg/cpp/src/barretenberg/bb/main.cpp | 11 ++- .../ecc/fields/field2_declarations.hpp | 12 +++ .../cpp/src/barretenberg/srs/c_bind.cpp | 20 ++-- barretenberg/ts/src/crs/net_crs.ts | 15 +-- barretenberg/ts/src/crs/node/index.ts | 36 +++---- 8 files changed, 108 insertions(+), 112 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/bb/file_io.hpp b/barretenberg/cpp/src/barretenberg/bb/file_io.hpp index 31796e57ffa..f91bb7712bc 100644 --- a/barretenberg/cpp/src/barretenberg/bb/file_io.hpp +++ b/barretenberg/cpp/src/barretenberg/bb/file_io.hpp @@ -1,29 +1,42 @@ #pragma once #include +#include #include #include #include -inline std::vector read_file(const std::string& filename, size_t bytes = 0) +inline size_t get_file_size(std::string const& filename) { // Open the file in binary mode and move to the end. std::ifstream file(filename, std::ios::binary | std::ios::ate); if (!file) { - throw std::runtime_error("Unable to open file: " + filename); + return 0; } + file.seekg(0, std::ios::end); + return (size_t)file.tellg(); +} + +inline std::vector read_file(const std::string& filename, size_t bytes = 0) +{ // Get the file size. - std::streamsize size = bytes == 0 ? (std::streamsize)file.tellg() : (std::streamsize)bytes; + auto size = get_file_size(filename); if (size <= 0) { throw std::runtime_error("File is empty or there's an error reading it: " + filename); } + auto to_read = bytes == 0 ? size : bytes; + + std::ifstream file(filename, std::ios::binary); + if (!file) { + throw std::runtime_error("Unable to open file: " + filename); + } + // Create a vector with enough space for the file data. - std::vector fileData((size_t)size); + std::vector fileData(to_read); - // Go back to the start of the file and read all its contents. - file.seekg(0, std::ios::beg); - file.read(reinterpret_cast(fileData.data()), size); + // Read all its contents. + file.read(reinterpret_cast(fileData.data()), (std::streamsize)to_read); return fileData; } diff --git a/barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.cpp b/barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.cpp index 06ec36faff6..5f181f117ac 100644 --- a/barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.cpp @@ -1,34 +1,17 @@ #include "get_bn254_crs.hpp" - -// Gets the transcript URL from the BARRETENBERG_TRANSCRIPT_URL environment variable, if set. -// Otherwise returns the default URL. -namespace { -std::string get_bn254_transcript_url() -{ - const char* ENV_VAR_NAME = "BARRETENBERG_TRANSCRIPT_URL"; - const std::string DEFAULT_URL = "https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/monomial/transcript00.dat"; - - const char* env_url = std::getenv(ENV_VAR_NAME); - - auto environment_variable_exists = ((env_url != nullptr) && *env_url); - - return environment_variable_exists ? std::string(env_url) : DEFAULT_URL; -} -} // namespace +#include "barretenberg/bb/file_io.hpp" std::vector download_bn254_g1_data(size_t num_points) { - size_t g1_start = 28; - size_t g1_end = g1_start + num_points * 64 - 1; + size_t g1_end = num_points * 64 - 1; - std::string url = get_bn254_transcript_url(); + std::string url = "https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g1.dat"; - std::string command = - "curl -s -H \"Range: bytes=" + std::to_string(g1_start) + "-" + std::to_string(g1_end) + "\" '" + url + "'"; + std::string command = "curl -s -H \"Range: bytes=0-" + std::to_string(g1_end) + "\" '" + url + "'"; auto data = exec_pipe(command); // Header + num_points * sizeof point. - if (data.size() < g1_end - g1_start) { + if (data.size() < g1_end) { throw std::runtime_error("Failed to download g1 data."); } @@ -37,50 +20,36 @@ std::vector download_bn254_g1_data(size_t num_points) std::vector download_bn254_g2_data() { - size_t g2_start = 28 + 5040001 * 64; - size_t g2_end = g2_start + 128 - 1; - - std::string url = get_bn254_transcript_url(); - - std::string command = - "curl -s -H \"Range: bytes=" + std::to_string(g2_start) + "-" + std::to_string(g2_end) + "\" '" + url + "'"; - + std::string url = "https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g2.dat"; + std::string command = "curl -s '" + url + "'"; return exec_pipe(command); } std::vector get_bn254_g1_data(const std::filesystem::path& path, size_t num_points) { std::filesystem::create_directories(path); - std::ifstream size_file(path / "size"); - size_t size = 0; - if (size_file) { - size_file >> size; - size_file.close(); - } - if (size >= num_points) { - vinfo("using cached crs at: ", path); - auto data = read_file(path / "g1.dat", 28 + num_points * 64); + + auto g1_path = path / "bn254_g1.dat"; + size_t g1_file_size = get_file_size(g1_path); + + if (g1_file_size >= num_points * 64 && g1_file_size % 64 == 0) { + vinfo("using cached crs of size ", std::to_string(g1_file_size / 64), " at ", g1_path); + auto data = read_file(g1_path, g1_file_size); auto points = std::vector(num_points); - auto size_of_points_in_bytes = num_points * 64; - barretenberg::srs::IO::read_affine_elements_from_buffer( - points.data(), (char*)data.data(), size_of_points_in_bytes); + for (size_t i = 0; i < num_points; ++i) { + points[i] = from_buffer(data, i * 64); + } return points; } vinfo("downloading crs..."); auto data = download_bn254_g1_data(num_points); - write_file(path / "g1.dat", data); - - std::ofstream new_size_file(path / "size"); - if (!new_size_file) { - throw std::runtime_error("Failed to open size file for writing"); - } - new_size_file << num_points; - new_size_file.close(); + write_file(g1_path, data); auto points = std::vector(num_points); - barretenberg::srs::IO::read_affine_elements_from_buffer( - points.data(), (char*)data.data(), data.size()); + for (size_t i = 0; i < num_points; ++i) { + points[i] = from_buffer(data, i * 64); + } return points; } @@ -88,16 +57,15 @@ barretenberg::g2::affine_element get_bn254_g2_data(const std::filesystem::path& { std::filesystem::create_directories(path); - try { - auto data = read_file(path / "g2.dat"); - barretenberg::g2::affine_element g2_point; - barretenberg::srs::IO::read_affine_elements_from_buffer(&g2_point, (char*)data.data(), 128); - return g2_point; - } catch (std::exception&) { - auto data = download_bn254_g2_data(); - write_file(path / "g2.dat", data); - barretenberg::g2::affine_element g2_point; - barretenberg::srs::IO::read_affine_elements_from_buffer(&g2_point, (char*)data.data(), 128); - return g2_point; + auto g2_path = path / "bn254_g2.dat"; + size_t g2_file_size = get_file_size(g2_path); + + if (g2_file_size == 128) { + auto data = read_file(g2_path); + return from_buffer(data.data()); } + + auto data = download_bn254_g2_data(); + write_file(g2_path, data); + return from_buffer(data.data()); } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/bb/get_grumpkin_crs.cpp b/barretenberg/cpp/src/barretenberg/bb/get_grumpkin_crs.cpp index 2cb7ba42320..2d3394a498d 100644 --- a/barretenberg/cpp/src/barretenberg/bb/get_grumpkin_crs.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/get_grumpkin_crs.cpp @@ -45,8 +45,9 @@ std::vector get_grumpkin_g1_data(const std::file size_file.close(); } if (size >= num_points) { - vinfo("using cached crs at: ", path); - auto data = read_file(path / "grumpkin_g1.dat", 28 + num_points * 64); + auto file = path / "grumpkin_g1.dat"; + vinfo("using cached crs at: ", file); + auto data = read_file(file, 28 + num_points * 64); auto points = std::vector(num_points); auto size_of_points_in_bytes = num_points * 64; barretenberg::srs::IO::read_affine_elements_from_buffer( diff --git a/barretenberg/cpp/src/barretenberg/bb/main.cpp b/barretenberg/cpp/src/barretenberg/bb/main.cpp index 35c260f8969..ba0e3ad543b 100644 --- a/barretenberg/cpp/src/barretenberg/bb/main.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/main.cpp @@ -19,7 +19,14 @@ #include using namespace barretenberg; -std::string CRS_PATH = "./crs"; + +std::string getHomeDir() +{ + char* home = std::getenv("HOME"); + return home != nullptr ? std::string(home) : "./"; +} + +std::string CRS_PATH = getHomeDir() + "/.bb-crs"; bool verbose = false; const std::filesystem::path current_path = std::filesystem::current_path(); @@ -441,7 +448,7 @@ int main(int argc, char* argv[]) std::string proof_path = get_option(args, "-p", "./proofs/proof"); std::string vk_path = get_option(args, "-k", "./target/vk"); std::string pk_path = get_option(args, "-r", "./target/pk"); - CRS_PATH = get_option(args, "-c", "./crs"); + CRS_PATH = get_option(args, "-c", CRS_PATH); bool recursive = flag_present(args, "-r") || flag_present(args, "--recursive"); // Skip CRS initialization for any command which doesn't require the CRS. diff --git a/barretenberg/cpp/src/barretenberg/ecc/fields/field2_declarations.hpp b/barretenberg/cpp/src/barretenberg/ecc/fields/field2_declarations.hpp index e99c92aed13..e1196d23c10 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/fields/field2_declarations.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/fields/field2_declarations.hpp @@ -138,4 +138,16 @@ template struct alignas(32) field2 { } }; +template void read(B& it, field2& value) +{ + using serialize::read; + read(it, value.c0); + read(it, value.c1); +} +template void write(B& buf, field2 const& value) +{ + using serialize::write; + write(buf, value.c0); + write(buf, value.c1); +} } // namespace barretenberg diff --git a/barretenberg/cpp/src/barretenberg/srs/c_bind.cpp b/barretenberg/cpp/src/barretenberg/srs/c_bind.cpp index ca42da6202a..43e3ea04153 100644 --- a/barretenberg/cpp/src/barretenberg/srs/c_bind.cpp +++ b/barretenberg/cpp/src/barretenberg/srs/c_bind.cpp @@ -9,18 +9,18 @@ using namespace barretenberg; /** - * WARNING: The SRS is not encoded the same way as all the read/write methods encode. - * Have to use the old school io functions to parse the buffers. + * We are not passed a vector (length prefixed), but the buffer and num points independently. + * Saves on having the generate the vector awkwardly calling side after downloading crs. */ -WASM_EXPORT void srs_init_srs(uint8_t const* points_buf, uint32_t const* num_points, uint8_t const* g2_point_buf) +WASM_EXPORT void srs_init_srs(uint8_t const* points_buf, uint32_t const* num_points_buf, uint8_t const* g2_point_buf) { - auto points = std::vector(ntohl(*num_points)); - srs::IO::read_affine_elements_from_buffer(points.data(), (char*)points_buf, points.size() * 64); - - g2::affine_element g2_point; - srs::IO::read_affine_elements_from_buffer(&g2_point, (char*)g2_point_buf, 128); - - barretenberg::srs::init_crs_factory(points, g2_point); + auto num_points = ntohl(*num_points_buf); + auto g1_points = std::vector(num_points); + for (size_t i = 0; i < num_points; ++i) { + g1_points[i] = from_buffer(points_buf, i * 64); + } + auto g2_point = from_buffer(g2_point_buf); + barretenberg::srs::init_crs_factory(g1_points, g2_point); } /** diff --git a/barretenberg/ts/src/crs/net_crs.ts b/barretenberg/ts/src/crs/net_crs.ts index 20396849924..7cca5c887ef 100644 --- a/barretenberg/ts/src/crs/net_crs.ts +++ b/barretenberg/ts/src/crs/net_crs.ts @@ -21,12 +21,11 @@ export class NetCrs { } async downloadG1Data() { - const g1Start = 28; - const g1End = g1Start + this.numPoints * 64 - 1; + const g1End = this.numPoints * 64 - 1; - const response = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/monomial/transcript00.dat', { + const response = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g1.dat', { headers: { - Range: `bytes=${g1Start}-${g1End}`, + Range: `bytes=0-${g1End}`, }, cache: 'force-cache', }); @@ -38,13 +37,7 @@ export class NetCrs { * Download the G2 points data. */ async downloadG2Data() { - const g2Start = 28 + 5040001 * 64; - const g2End = g2Start + 128 - 1; - - const response2 = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/monomial/transcript00.dat', { - headers: { - Range: `bytes=${g2Start}-${g2End}`, - }, + const response2 = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g2.dat', { cache: 'force-cache', }); diff --git a/barretenberg/ts/src/crs/node/index.ts b/barretenberg/ts/src/crs/node/index.ts index 5552029ed7b..75ef0878495 100644 --- a/barretenberg/ts/src/crs/node/index.ts +++ b/barretenberg/ts/src/crs/node/index.ts @@ -1,8 +1,9 @@ import { NetCrs } from '../net_crs.js'; import { GRUMPKIN_SRS_DEV_PATH, IgnitionFilesCrs } from './ignition_files_crs.js'; import { mkdirSync, readFileSync, writeFileSync } from 'fs'; -import { readFile } from 'fs/promises'; +import { readFile, stat } from 'fs/promises'; import createDebug from 'debug'; +import { homedir } from 'os'; const debug = createDebug('bb.js:crs'); @@ -12,7 +13,7 @@ const debug = createDebug('bb.js:crs'); export class Crs { constructor(public readonly numPoints: number, public readonly path: string) {} - static async new(numPoints: number, crsPath = './crs') { + static async new(numPoints: number, crsPath = homedir() + '/.bb-crs') { const crs = new Crs(numPoints, crsPath); await crs.init(); return crs; @@ -20,23 +21,24 @@ export class Crs { async init() { mkdirSync(this.path, { recursive: true }); - const size = await readFile(this.path + '/size', 'ascii').catch(() => undefined); - if (size && +size >= this.numPoints) { - debug(`using cached crs of size: ${size}`); + + const g1FileSize = await stat(this.path + '/bn254_g1.dat') + .then(stats => stats.size) + .catch(() => 0); + const g2FileSize = await stat(this.path + '/bn254_g2.dat') + .then(stats => stats.size) + .catch(() => 0); + + if (g1FileSize >= this.numPoints * 64 && g1FileSize % 64 == 0 && g2FileSize == 128) { + debug(`using cached crs of size: ${g1FileSize / 64}`); return; } - const ignitionCrs = new IgnitionFilesCrs(this.numPoints); - const crs = ignitionCrs.pathExists() ? new IgnitionFilesCrs(this.numPoints) : new NetCrs(this.numPoints); - if (crs instanceof NetCrs) { - debug(`downloading crs of size: ${this.numPoints}`); - } else { - debug(`loading igntion file crs of size: ${this.numPoints}`); - } + debug(`downloading crs of size: ${this.numPoints}`); + const crs = new NetCrs(this.numPoints); await crs.init(); - writeFileSync(this.path + '/size', this.numPoints.toString()); - writeFileSync(this.path + '/g1.dat', crs.getG1Data()); - writeFileSync(this.path + '/g2.dat', crs.getG2Data()); + writeFileSync(this.path + '/bn254_g1.dat', crs.getG1Data()); + writeFileSync(this.path + '/bn254_g2.dat', crs.getG2Data()); } /** @@ -44,7 +46,7 @@ export class Crs { * @returns The points data. */ getG1Data(): Uint8Array { - return readFileSync(this.path + '/g1.dat'); + return readFileSync(this.path + '/bn254_g1.dat'); } /** @@ -52,7 +54,7 @@ export class Crs { * @returns The points data. */ getG2Data(): Uint8Array { - return readFileSync(this.path + '/g2.dat'); + return readFileSync(this.path + '/bn254_g2.dat'); } }