Changes in stdlib necessary for aztec3.

Handpicked changes necessary for aztec3.

More handpicked changes and a fix.

Fix.

Port `copy_as_new_witness`.

Port `must_imply`.

`operator++`.

Port changes from `common`.

Co-authored-by: iAmMichaelConnor <[email protected]>

Minor pedersen fix.

Port `ecc/groups`.

Co-authored-by: Michael Connor <[email protected]>

Port remaining `stdlib` changes.

Co-authored-by: Michael Connor <[email protected]>

Latest fixes.

Co-authored-by: Michael Connor <[email protected]>

grumpkin conversion fix.

Co-authored-by: Michael Connor <[email protected]>

cpp/src/aztec/common/container.hpp

@@ -48,4 +48,16 @@ InnerCont flatten(Cont<InnerCont, Args...> const& in)
         result.insert(result.end(), e.begin(), e.end());
     }
     return result;
+}
+
+// Return the first index at which a given item can be found in the vector.
+// Only safe for vectors with length less than the size_t overflow size.
+template <typename T> long index_of(std::vector<T> const& vec, T const& item)
+{
+    auto const& begin = vec.begin();
+    auto const& end = vec.end();
+
+    auto const& itr = std::find(begin, end, item);
+
+    return itr == end ? -1 : std::distance(begin, itr);
 }

cpp/src/aztec/common/map.hpp

@@ -10,13 +10,29 @@ template <template <typename, typename...> typename Cont,
           typename... Args,
           typename F,
           typename OutElem = typename std::invoke_result<F, InElem const&>::type>
-Cont<OutElem> map(Cont<InElem, Args...> const& in, F op)
+Cont<OutElem> map(Cont<InElem, Args...> const& in, F&& op)
 {
     Cont<OutElem> result;
     std::transform(in.begin(), in.end(), std::back_inserter(result), op);
     return result;
 }
 
+/*
+ * Generic map function for mapping a std::array's elements to another type.
+ * TODO: this has only been added because I (Mike) couldn't get the above to work
+ * with an array.
+ */
+template <std::size_t SIZE,
+          typename InElem,
+          typename F,
+          typename OutElem = typename std::invoke_result<F, InElem const&>::type>
+std::array<OutElem, SIZE> map(std::array<InElem, SIZE> const& in, F&& op)
+{
+    std::array<OutElem, SIZE> result;
+    std::transform(in.begin(), in.end(), result.begin(), op);
+    return result;
+}
+
 /*
  * Generic map function for mapping a containers element to another type.
  * This version passes the element index as a second argument to the operator function.

cpp/src/aztec/common/serialize.hpp

@@ -34,6 +34,7 @@
 #include <map>
 #include <iostream>
 #include <common/log.hpp>
+#include <optional>
 
 #ifndef __i386__
 __extension__ using uint128_t = unsigned __int128;
@@ -311,6 +312,33 @@ template <typename B, typename T, typename U> inline void write(B& buf, std::map
     }
 }
 
+// Read std::optional<T>.
+template <typename B, typename T> inline void read(B& it, std::optional<T>& opt_value)
+{
+    bool is_nullopt;
+    read(it, is_nullopt);
+    if (is_nullopt) {
+        opt_value = std::nullopt;
+        return;
+    }
+    T value;
+    read(it, value);
+    opt_value = T(value);
+}
+
+// Write std::optional<T>.
+// Note: It takes up a different amount of space, depending on whether it's std::nullopt or populated with an actual
+// value.
+template <typename B, typename T> inline void write(B& buf, std::optional<T> const& opt_value)
+{
+    if (opt_value) {
+        write(buf, false); // is not nullopt
+        write(buf, *opt_value);
+        return;
+    }
+    write(buf, true); // is nullopt
+}
+
 } // namespace std
 
 // Helper functions that have return values.

cpp/src/aztec/common/streams.hpp

@@ -1,6 +1,7 @@
 #pragma once
 #include <iomanip>
 #include <ostream>
+#include <map>
 #include <vector>
 
 namespace std {
@@ -35,7 +36,7 @@ inline std::ostream& operator<<(std::ostream& os, std::vector<T> const& arr)
     for (auto element : arr) {
         os << ' ' << element << '\n';
     }
-    os << "]\n";
+    os << "]";
     return os;
 }
 
@@ -63,4 +64,25 @@ template <typename T, size_t S> inline std::ostream& operator<<(std::ostream& os
     return os;
 }
 
+template <typename T, typename U> inline std::ostream& operator<<(std::ostream& os, std::pair<T, U> const& pair)
+{
+    os << "(" << pair.first << ", " << pair.second << ")";
+    return os;
+}
+
+template <typename T> inline std::ostream& operator<<(std::ostream& os, std::optional<T> const& opt)
+{
+    return opt ? os << *opt : os << "std::nullopt";
+}
+
+template <typename T, typename U> inline std::ostream& operator<<(std::ostream& os, std::map<T, U> const& map)
+{
+    os << "[\n";
+    for (const auto& elem : map) {
+        os << " " << elem.first << ": " << elem.second << "\n";
+    }
+    os << "]";
+    return os;
+}
+
 } // namespace std

cpp/src/aztec/crypto/pedersen_commitment/pedersen.cpp

@@ -68,6 +68,27 @@ grumpkin::g1::affine_element commit_native(const std::vector<grumpkin::fq>& inpu
     return r.is_point_at_infinity() ? grumpkin::g1::affine_element(0, 0) : grumpkin::g1::affine_element(r);
 }
 
+grumpkin::g1::affine_element commit_native(const std::vector<std::pair<grumpkin::fq, generator_index_t>>& input_pairs)
+{
+    ASSERT((input_pairs.size() < (1 << 16)) && "too many inputs for 16 bit index");
+    std::vector<grumpkin::g1::element> out(input_pairs.size());
+
+#ifndef NO_MULTITHREADING
+    // Ensure generator data is initialized before threading...
+    init_generator_data();
+#pragma omp parallel for num_threads(input_pairs.size())
+#endif
+    for (size_t i = 0; i < input_pairs.size(); ++i) {
+        out[i] = commit_single(input_pairs[i].first, input_pairs[i].second);
+    }
+
+    grumpkin::g1::element r = out[0];
+    for (size_t i = 1; i < input_pairs.size(); ++i) {
+        r = out[i] + r;
+    }
+    return r.is_point_at_infinity() ? grumpkin::g1::affine_element(0, 0) : grumpkin::g1::affine_element(r);
+}
+
 /**
  * The same as commit_native, but only return the resultant x coordinate (i.e. compress).
  */
@@ -76,6 +97,11 @@ grumpkin::fq compress_native(const std::vector<grumpkin::fq>& inputs, const size
     return commit_native(inputs, hash_index).x;
 }
 
+grumpkin::fq compress_native(const std::vector<std::pair<grumpkin::fq, generator_index_t>>& input_pairs)
+{
+    return commit_native(input_pairs).x;
+}
+
 /**
  * Given an arbitrary length of bytes, convert them to fields and compress the result using the default generators.
  */

cpp/src/aztec/crypto/pedersen_commitment/pedersen.hpp

@@ -11,7 +11,9 @@ namespace pedersen_commitment {
 
 grumpkin::g1::element commit_single(const barretenberg::fr& in, generator_index_t const& index);
 
-grumpkin::g1::affine_element commit_native(const std::vector<grumpkin::fq>& elements, const size_t hash_index = 0);
+grumpkin::g1::affine_element commit_native(const std::vector<grumpkin::fq>& inputs, const size_t hash_index = 0);
+
+grumpkin::g1::affine_element commit_native(const std::vector<std::pair<grumpkin::fq, generator_index_t>>& input_pairs);
 
 grumpkin::fq compress_native(const std::vector<grumpkin::fq>& inputs, const size_t hash_index = 0);
 
@@ -23,5 +25,7 @@ template <size_t T> grumpkin::fq compress_native(const std::array<grumpkin::fq,
 
 grumpkin::fq compress_native(const std::vector<uint8_t>& input);
 
+grumpkin::fq compress_native(const std::vector<std::pair<grumpkin::fq, generator_index_t>>& input_pairs);
+
 } // namespace pedersen_commitment
 } // namespace crypto

cpp/src/aztec/crypto/pedersen_commitment/pedersen_lookup.cpp

@@ -3,6 +3,7 @@
 #include "./convert_buffer_to_field.hpp"
 
 #include <ecc/curves/grumpkin/grumpkin.hpp>
+#include <numeric/bitop/pow.hpp>
 
 using namespace crypto::pedersen_hash::lookup;
 
@@ -34,16 +35,82 @@ grumpkin::g1::element merkle_damgard_compress(const std::vector<grumpkin::fq>& i
     return (hash_single(result, false) + hash_single(grumpkin::fq(num_inputs), true));
 }
 
+grumpkin::g1::element merkle_damgard_compress(const std::vector<grumpkin::fq>& inputs, const std::vector<size_t>& ivs)
+{
+    if (inputs.size() == 0) {
+        auto result = grumpkin::g1::affine_one;
+        result.self_set_infinity();
+        return result;
+    }
+    init();
+    const size_t num_inputs = inputs.size();
+
+    grumpkin::fq result = (pedersen_iv_table[0]).x;
+    for (size_t i = 0; i < 2 * num_inputs; i++) {
+        if ((i & 1) == 0) {
+            grumpkin::fq iv_result = (pedersen_iv_table[ivs[i >> 1]]).x;
+            result = hash_pair(result, iv_result);
+        } else {
+            result = hash_pair(result, inputs[i >> 1]);
+        }
+    }
+
+    return (hash_single(result, false) + hash_single(grumpkin::fq(num_inputs), true));
+}
+
+grumpkin::g1::element merkle_damgard_tree_compress(const std::vector<grumpkin::fq>& inputs,
+                                                   const std::vector<size_t>& ivs)
+{
+    const size_t num_inputs = inputs.size();
+    ASSERT(num_inputs == ivs.size());
+    ASSERT(numeric::is_power_of_two(num_inputs));
+    if (inputs.size() == 0) {
+        auto result = grumpkin::g1::affine_one;
+        result.self_set_infinity();
+        return result;
+    }
+    init();
+
+    // Process height 0 of the tree.
+    std::vector<grumpkin::fq> temp_storage;
+    for (size_t i = 0; i < num_inputs; i++) {
+        grumpkin::fq iv_result = (pedersen_iv_table[ivs[i]]).x;
+        temp_storage.push_back(hash_pair(iv_result, inputs[i]));
+    }
+
+    // Process heights 1, 2, ..., log2(m) of the tree.
+    const size_t total_height = numeric::get_msb(num_inputs);
+    for (size_t height = 1; height <= total_height; height++) {
+        const size_t leaf_count = 1UL << (total_height - height);
+        for (size_t i = 0; i < leaf_count; i++) {
+            temp_storage[i] = hash_pair(temp_storage[2 * i], temp_storage[2 * i + 1]);
+        }
+    }
+
+    return (hash_single(temp_storage[0], false) + hash_single(grumpkin::fq(num_inputs), true));
+}
+
 grumpkin::g1::affine_element commit_native(const std::vector<grumpkin::fq>& inputs, const size_t hash_index)
 {
     return grumpkin::g1::affine_element(merkle_damgard_compress(inputs, hash_index));
 }
 
+grumpkin::g1::affine_element commit_native(const std::vector<grumpkin::fq>& inputs,
+                                           const std::vector<size_t>& hash_indices)
+{
+    return grumpkin::g1::affine_element(merkle_damgard_compress(inputs, hash_indices));
+}
+
 grumpkin::fq compress_native(const std::vector<grumpkin::fq>& inputs, const size_t hash_index)
 {
     return commit_native(inputs, hash_index).x;
 }
 
+grumpkin::fq compress_native(const std::vector<grumpkin::fq>& inputs, const std::vector<size_t>& hash_indices)
+{
+    return commit_native(inputs, hash_indices).x;
+}
+
 grumpkin::fq compress_native_buffer_to_field(const std::vector<uint8_t>& input)
 {
     const auto elements = convert_buffer_to_field(input);

cpp/src/aztec/crypto/pedersen_commitment/pedersen_lookup.hpp

@@ -7,8 +7,12 @@ namespace pedersen_commitment {
 namespace lookup {
 
 grumpkin::g1::element merkle_damgard_compress(const std::vector<grumpkin::fq>& inputs, const size_t iv);
+grumpkin::g1::element merkle_damgard_compress(const std::vector<grumpkin::fq>& inputs, const std::vector<size_t>& ivs);
+grumpkin::g1::element merkle_damgard_tree_compress(const std::vector<grumpkin::fq>& inputs,
+                                                   const std::vector<size_t>& ivs);
 
 grumpkin::fq compress_native(const std::vector<grumpkin::fq>& inputs, const size_t hash_index = 0);
+grumpkin::fq compress_native(const std::vector<grumpkin::fq>& inputs, const std::vector<size_t>& hash_indices);
 std::vector<uint8_t> compress_native(const std::vector<uint8_t>& input);
 
 grumpkin::fq compress_native_buffer_to_field(const std::vector<uint8_t>& input);
@@ -20,6 +24,8 @@ template <size_t T> grumpkin::fq compress_native(const std::array<grumpkin::fq,
 }
 
 grumpkin::g1::affine_element commit_native(const std::vector<grumpkin::fq>& inputs, const size_t hash_index = 0);
+grumpkin::g1::affine_element commit_native(const std::vector<grumpkin::fq>& inputs,
+                                           const std::vector<size_t>& hash_indices);
 
 } // namespace lookup
 } // namespace pedersen_commitment

cpp/src/aztec/crypto/pedersen_commitment/pedersen_lookup.test.cpp

@@ -142,3 +142,67 @@ TEST(pedersen_lookup, merkle_damgard_compress)
     EXPECT_EQ(affine_element(result).x,
               affine_element(compute_expected(intermediate, 0) + compute_expected(fq(m), 15)).x);
 }
+
+TEST(pedersen_lookup, merkle_damgard_compress_multiple_iv)
+{
+    typedef grumpkin::fq fq;
+    typedef grumpkin::fr fr;
+    typedef grumpkin::g1::affine_element affine_element;
+
+    const size_t m = 10;
+    std::vector<size_t> ivs;
+    std::vector<fq> inputs;
+    for (size_t i = 0; i < m; i++) {
+        inputs.push_back(engine.get_random_uint256());
+        ivs.push_back(engine.get_random_uint8());
+    }
+
+    const auto result = crypto::pedersen_commitment::lookup::merkle_damgard_compress(inputs, ivs);
+
+    const size_t initial_iv = 0;
+    fq intermediate = (grumpkin::g1::affine_one * fr(initial_iv + 1)).x;
+    for (size_t i = 0; i < 2 * m; i++) {
+        if ((i & 1) == 0) {
+            const auto iv = (grumpkin::g1::affine_one * fr(ivs[i >> 1] + 1)).x;
+            intermediate = affine_element(compute_expected(intermediate, 0) + compute_expected(iv, 15)).x;
+        } else {
+            intermediate = affine_element(compute_expected(intermediate, 0) + compute_expected(inputs[i >> 1], 15)).x;
+        }
+    }
+
+    EXPECT_EQ(affine_element(result).x,
+              affine_element(compute_expected(intermediate, 0) + compute_expected(fq(m), 15)).x);
+}
+
+TEST(pedersen_lookup, merkle_damgard_tree_compress)
+{
+    typedef grumpkin::fq fq;
+    typedef grumpkin::fr fr;
+    typedef grumpkin::g1::affine_element affine_element;
+
+    const size_t m = 8;
+    std::vector<size_t> ivs;
+    std::vector<fq> inputs;
+    for (size_t i = 0; i < m; i++) {
+        inputs.push_back(engine.get_random_uint256());
+        ivs.push_back(engine.get_random_uint8());
+    }
+
+    const auto result = crypto::pedersen_commitment::lookup::merkle_damgard_tree_compress(inputs, ivs);
+
+    std::vector<fq> temp;
+    for (size_t i = 0; i < m; i++) {
+        const fq iv_term = (grumpkin::g1::affine_one * fr(ivs[i] + 1)).x;
+        temp.push_back(affine_element(compute_expected(iv_term, 0) + compute_expected(inputs[i], 15)).x);
+    }
+
+    const size_t logm = numeric::get_msb(m);
+    for (size_t j = 1; j <= logm; j++) {
+        const size_t nodes = (1UL << (logm - j));
+        for (size_t i = 0; i < nodes; i++) {
+            temp[i] = affine_element(compute_expected(temp[2 * i], 0) + compute_expected(temp[2 * i + 1], 15)).x;
+        }
+    }
+
+    EXPECT_EQ(affine_element(result).x, affine_element(compute_expected(temp[0], 0) + compute_expected(fq(m), 15)).x);
+}

cpp/src/aztec/ecc/fields/field.hpp

@@ -173,6 +173,11 @@ template <class Params> struct alignas(32) field {
     BBERG_INLINE constexpr field operator-() const noexcept;
     constexpr field operator/(const field& other) const noexcept;
 
+    // prefix increment (++x)
+    BBERG_INLINE constexpr field operator++() noexcept;
+    // postfix increment (x++)
+    BBERG_INLINE constexpr field operator++(int) noexcept;
+
     BBERG_INLINE constexpr field operator*=(const field& other) noexcept;
     BBERG_INLINE constexpr field operator+=(const field& other) noexcept;
     BBERG_INLINE constexpr field operator-=(const field& other) noexcept;