Caphash optimization

libiop/algebra/utils.hpp

@@ -19,6 +19,9 @@ void bitreverse_vector(std::vector<T> &a);
 template<typename T>
 std::vector<T> random_vector(const std::size_t count);
 
+template<typename T, typename U>
+bool compare_first(const std::pair<T, U> &a, const std::pair<T, U> &b);
+
 template<typename T>
 std::vector<T> all_subset_sums(const std::vector<T> &basis, const T& shift = 0)
 #if defined(__clang__)

libiop/algebra/utils.tcc

@@ -168,6 +168,12 @@ std::vector<T> random_vector(const std::size_t count)
     return result;
 }
 
+template<typename T, typename U>
+bool compare_first(const std::pair<T, U> &a, const std::pair<T, U> &b)
+{
+    return a.first < b.first;
+}
+
 template<typename FieldT>
 std::vector<FieldT> random_FieldT_vector(const std::size_t count)
 {

libiop/bcs/bcs_common.hpp

@@ -25,12 +25,14 @@ template<typename FieldT, typename MT_hash_type>
 struct bcs_transformation_parameters {
     std::size_t security_parameter; /* TODO: possibly revisit in the future */
     bcs_hash_type hash_enum;
+    std::size_t cap_size;
 
     pow_parameters pow_params_;
 
     std::shared_ptr<hashchain<FieldT, MT_hash_type>> hashchain_;
     std::shared_ptr<leafhash<FieldT, MT_hash_type>> leafhasher_;
     two_to_one_hash_function<MT_hash_type> compression_hasher;
+    cap_hash_function<MT_hash_type> cap_hasher;
 };
 
 template<typename FieldT, typename MT_hash_type>

libiop/bcs/bcs_common.tcc

@@ -471,9 +471,11 @@ void bcs_protocol<FieldT, MT_root_hash>::seal_interaction_registrations()
                 size,
                 this->parameters_.leafhasher_,
                 this->parameters_.compression_hasher,
+                this->parameters_.cap_hasher,
                 this->digest_len_bytes_,
                 make_zk,
-                this->parameters_.security_parameter);
+                this->parameters_.security_parameter,
+                this->parameters_.cap_size);
             this->Merkle_trees_.emplace_back(MT);
         }
     }
@@ -710,7 +712,7 @@ void print_detailed_transcript_data(
 
     const size_t digest_len_bytes = 2 * (params.security_parameter / 8);
     const size_t field_size = (libff::log_of_field_size_helper<FieldT>(FieldT::zero()) + 7) / 8;
-    std::vector<size_t> two_to_one_hashes_by_round;
+    std::vector<size_t> internal_hash_complexity_by_round;
     std::vector<size_t> leaf_hashes_by_round;
     std::vector<size_t> zk_hashes_by_round;
     std::vector<size_t> IOP_size_by_round;
@@ -724,6 +726,7 @@ void print_detailed_transcript_data(
             MT_size,
             params.leafhasher_,
             params.compression_hasher,
+            params.cap_hasher,
             digest_len_bytes,
             false,
             params.security_parameter);
@@ -741,10 +744,9 @@ void print_detailed_transcript_data(
                 query_positions.emplace_back(MT_position);
             }
         }
-        size_t num_two_to_one_hashes_in_round =
-            MT.count_hashes_to_verify_set_membership_proof(
-            query_positions);
-        two_to_one_hashes_by_round.emplace_back(num_two_to_one_hashes_in_round);
+        size_t internal_hash_complexity_in_round =
+            MT.count_internal_hash_complexity_to_verify_set_membership(query_positions);
+        internal_hash_complexity_by_round.emplace_back(internal_hash_complexity_in_round);
         const size_t num_values_per_leaf = transcript.query_responses_[round][0].size();
         const size_t num_leaves = transcript.query_responses_[round].size();
         leaf_hashes_by_round.emplace_back(num_values_per_leaf * num_leaves);
@@ -788,14 +790,17 @@ void print_detailed_transcript_data(
 
     printf("\n");
     printf("total prover messages size: %lu\n", total_prover_message_size);
-    const size_t total_two_to_one_hashes = std::accumulate(
-        two_to_one_hashes_by_round.begin(), two_to_one_hashes_by_round.end(), 0);
+    const size_t total_internal_hash_complexity = std::accumulate(
+        internal_hash_complexity_by_round.begin(), internal_hash_complexity_by_round.end(), 0);
     const size_t total_leaves_hashed = std::accumulate(
         leaf_hashes_by_round.begin(), leaf_hashes_by_round.end(), 0);
     const size_t total_zk_hashes = std::accumulate(
         zk_hashes_by_round.begin(), zk_hashes_by_round.end(), 0);
-    const size_t total_hashes = total_two_to_one_hashes + total_leaves_hashed + total_zk_hashes;
-    printf("total two to one hashes: %lu\n", total_two_to_one_hashes);
+    /* Since each two-to-one hash is counted as two units, we divide by 2 here to make it consistent.
+       It would be nice to take into account how many leaves are hashed, but we are unfortunately
+       not provided this information. */
+    const size_t total_hashes = total_internal_hash_complexity / 2 + total_leaves_hashed + total_zk_hashes;
+    printf("total internal hash complexity: %lu\n", total_internal_hash_complexity);
     printf("total leaves hashed: %lu\n", total_leaves_hashed);
     printf("total hashes: %lu\n", total_hashes);
     printf("\n");
@@ -808,7 +813,7 @@ void print_detailed_transcript_data(
         printf("MT_depth %lu\n", MT_depths[round]);
         printf("IOP size: %lu bytes\n", IOP_size_by_round[round]);
         printf("BCS size: %lu bytes\n", BCS_size_by_round[round]);
-        printf("number of two to one hashes: %lu\n", two_to_one_hashes_by_round[round]);
+        printf("internal hash complexity: %lu\n", internal_hash_complexity_by_round[round]);
         printf("number of leaves hashed: %lu\n", leaf_hashes_by_round[round]);
         if (make_zk[round])
         {

libiop/bcs/common_bcs_parameters.tcc

@@ -15,10 +15,11 @@ bcs_transformation_parameters<FieldT, MT_root_hash> default_bcs_params(
     params.hash_enum = hash_type;
     /* TODO: Push setting leaf hash into internal BCS code. Currently 2 is fine, as leaf size is internally unused. */
     const size_t leaf_size = 2;
-    params.leafhasher_ = get_leafhash<FieldT, MT_root_hash>(hash_type, security_parameter, leaf_size);
+    params.cap_size = 2;
+    params.leafhasher_ = get_leafhash<MT_root_hash, FieldT>(hash_type, security_parameter, leaf_size);
     params.compression_hasher = get_two_to_one_hash<MT_root_hash, FieldT>(hash_type, security_parameter);
-    params.hashchain_ =
-        get_hashchain<FieldT, MT_root_hash>(hash_type, security_parameter);
+    params.cap_hasher = get_cap_hash<MT_root_hash, FieldT>(hash_type, security_parameter);
+    params.hashchain_ = get_hashchain<FieldT, MT_root_hash>(hash_type, security_parameter);
 
     // Work per hash. Todo generalize this w/ proper explanations of work amounts
     const size_t work_per_hash = (hash_type == 1) ? 1 : 128;

libiop/bcs/hashing/algebraic_sponge.hpp

@@ -99,14 +99,15 @@ class algebraic_hashchain : public hashchain<FieldT, MT_root_type>
     void absorb_internal(const typename libff::enable_if<std::is_same<MT_root_type, FieldT>::value, MT_root_type>::type new_input);
 };
 
+/** The algebraic_vector_hash is used for both the algebraic leaf hash and cap hash. */
 template<typename FieldT>
-class algebraic_leafhash : public leafhash<FieldT, FieldT>
+class algebraic_vector_hash : public leafhash<FieldT, FieldT>
 {
     protected:
     std::shared_ptr<algebraic_sponge<FieldT>> sponge_;
 
     public:
-    algebraic_leafhash(
+    algebraic_vector_hash(
         std::shared_ptr<algebraic_sponge<FieldT>> sponge,
         size_t security_parameter);
     FieldT hash(const std::vector<FieldT> &leaf);
@@ -115,6 +116,9 @@ class algebraic_leafhash : public leafhash<FieldT, FieldT>
         const zk_salt_type &zk_salt);
 };
 
+template<typename FieldT>
+using algebraic_leafhash = algebraic_vector_hash<FieldT>;
+
 template<typename FieldT>
 class algebraic_two_to_one_hash
 {

libiop/bcs/hashing/algebraic_sponge.tcc

@@ -206,7 +206,7 @@ MT_root_type algebraic_hashchain<FieldT, MT_root_type>::squeeze_root_type()
 }
 
 template<typename FieldT>
-algebraic_leafhash<FieldT>::algebraic_leafhash(
+algebraic_vector_hash<FieldT>::algebraic_vector_hash(
     std::shared_ptr<algebraic_sponge<FieldT>> sponge,
     size_t security_parameter) :
     sponge_(sponge->new_sponge())
@@ -218,7 +218,7 @@ algebraic_leafhash<FieldT>::algebraic_leafhash(
 }
 
 template<typename FieldT>
-FieldT algebraic_leafhash<FieldT>::hash(
+FieldT algebraic_vector_hash<FieldT>::hash(
     const std::vector<FieldT> &leaf)
 {
     this->sponge_->absorb(leaf);
@@ -228,7 +228,7 @@ FieldT algebraic_leafhash<FieldT>::hash(
 }
 
 template<typename FieldT>
-FieldT algebraic_leafhash<FieldT>::zk_hash(
+FieldT algebraic_vector_hash<FieldT>::zk_hash(
     const std::vector<FieldT> &leaf,
     const zk_salt_type &zk_salt)
 {

libiop/bcs/hashing/blake2b.cpp

@@ -29,6 +29,8 @@ binary_hash_digest blake2b_two_to_one_hash(const binary_hash_digest &first,
                                            const binary_hash_digest &second,
                                            const std::size_t digest_len_bytes)
 {
+    /* binary_hash_digest is a C++ string pointer so we need to sum them since they are not
+       contiguous in memory. */
     const binary_hash_digest first_plus_second = first + second;
 
     binary_hash_digest result(digest_len_bytes, 'X');
@@ -47,6 +49,33 @@ binary_hash_digest blake2b_two_to_one_hash(const binary_hash_digest &first,
     return result;
 }
 
+binary_hash_digest blake2b_many_to_one_hash(const std::vector<binary_hash_digest> &data,
+                                            const std::size_t digest_len_bytes)
+{
+    /* binary_hash_digest is a C++ string pointer so we need to sum them since they are not
+       contiguous in memory. */
+    binary_hash_digest input = data[0];
+    for (int i = 1; i < data.size(); i++)
+    {
+        input += data[i];
+    }
+
+    binary_hash_digest result(digest_len_bytes, 'X');
+
+    /* see https://download.libsodium.org/doc/hashing/generic_hashing.html */
+    const int status = crypto_generichash_blake2b((unsigned char*)&result[0],
+                                                  digest_len_bytes,
+                                                  (result.empty() ? NULL : (unsigned char*)&input[0]),
+                                                  input.size(),
+                                                  NULL, 0);
+    if (status != 0)
+    {
+        throw std::runtime_error("Got non-zero status from crypto_generichash_blake2b. (Is digest_len_bytes correct?)");
+    }
+
+    return result;
+}
+
 std::size_t blake2b_integer_randomness_extractor(const binary_hash_digest &root,
                                                  const std::size_t index,
                                                  const std::size_t upper_bound)
@@ -73,4 +102,4 @@ std::size_t blake2b_integer_randomness_extractor(const binary_hash_digest &root,
     return result % upper_bound;
 }
 
-}
+} // namespace libiop

libiop/bcs/hashing/blake2b.hpp

@@ -58,6 +58,8 @@ class blake2b_leafhash : public leafhash<FieldT, binary_hash_digest>
         const zk_salt_type &zk_salt);
 };
 
+/* Many-to-one hash which takes in a vector of field elements.
+   Behavior undefined when data is empty. */
 template<typename FieldT>
 binary_hash_digest blake2b_field_element_hash(const std::vector<FieldT> &data,
                                        const std::size_t digest_len_bytes);
@@ -73,11 +75,16 @@ std::size_t blake2b_integer_randomness_extractor(const binary_hash_digest &root,
                                                  const std::size_t upper_bound);
 
 binary_hash_digest blake2b_zk_element_hash(const std::vector<uint8_t> &first,
-                                    const std::size_t digest_len_bytes);
+                                           const std::size_t digest_len_bytes);
 
 binary_hash_digest blake2b_two_to_one_hash(const binary_hash_digest &first,
-                                    const binary_hash_digest &second,
-                                    const std::size_t digest_len_bytes);
+                                           const binary_hash_digest &second,
+                                           const std::size_t digest_len_bytes);
+
+/* Many-to-one hash which takes in a vector of binary_hash_digest.
+   Behavior undefined when data is empty. */
+binary_hash_digest blake2b_many_to_one_hash(const std::vector<binary_hash_digest> &data,
+                                            const std::size_t digest_len_bytes);
 
 } // namespace libiop
 

libiop/bcs/hashing/blake2b.tcc

@@ -139,9 +139,8 @@ binary_hash_digest blake2b_leafhash<FieldT>::zk_hash(
 // don't we need to make them in canonical form first?
 template<typename FieldT>
 binary_hash_digest blake2b_field_element_hash(const std::vector<FieldT> &data,
-                                       const std::size_t digest_len_bytes)
+                                              const std::size_t digest_len_bytes)
 {
-
     binary_hash_digest result(digest_len_bytes, 'X');
 
     /* see https://download.libsodium.org/doc/hashing/generic_hashing.html */
@@ -155,7 +154,6 @@ binary_hash_digest blake2b_field_element_hash(const std::vector<FieldT> &data,
         throw std::runtime_error("Got non-zero status from crypto_generichash_blake2b. (Is digest_len_bytes correct?)");
     }
 
-
     return result;
 }
 
@@ -256,4 +254,4 @@ std::vector<FieldT> blake2b_FieldT_randomness_extractor(const binary_hash_digest
     return result;
 }
 
-}
+} // namespace libiop

libiop/bcs/hashing/dummy_algebraic_hash.hpp

@@ -63,6 +63,9 @@ FieldT dummy_algebraic_two_to_one_hash(
     const FieldT &second,
     const std::size_t digest_len_bytes);
 
+template<typename FieldT>
+FieldT dummy_algebraic_cap_hash(const std::vector<FieldT> &data, const std::size_t digest_len_bytes);
+
 } // namespace libiop
 
 #include "libiop/bcs/hashing/dummy_algebraic_hash.tcc"

libiop/bcs/hashing/dummy_algebraic_hash.tcc

@@ -117,7 +117,7 @@ FieldT dummy_algebraic_leafhash<FieldT>::hash(const std::vector<FieldT> &leaf)
     FieldT sum = FieldT::zero();
     for (size_t i = 0; i < leaf.size(); i++)
     {
-        sum += FieldT(i) * leaf[i];
+        sum += FieldT(i + 1) * leaf[i]; // Add one, otherwise the 0th index is unused.
     }
     return sum;
 }
@@ -147,4 +147,16 @@ FieldT dummy_algebraic_two_to_one_hash(
     return FieldT(2) * first + second;
 }
 
+template<typename FieldT>
+FieldT dummy_algebraic_cap_hash(const std::vector<FieldT> &data, const std::size_t digest_len_bytes)
+{
+    FieldT sum = FieldT::zero();
+    for (size_t i = 0; i < data.size(); i++)
+    {
+        sum += FieldT(i + 1) * data[i]; // Add one, otherwise the 0th index is unused.
+    }
+
+    return sum;
+}
+
 }

libiop/bcs/hashing/hash_enum.hpp

@@ -30,15 +30,18 @@ static const char* bcs_hash_type_names[] = {"", "blake2b", "poseidon with Starkw
 template<typename FieldT, typename MT_root_type>
 std::shared_ptr<hashchain<FieldT, MT_root_type>> get_hashchain(bcs_hash_type hash_type, size_t security_parameter);
 
-template<typename FieldT, typename leaf_hash_type>
+template<typename leaf_hash_type, typename FieldT>
 std::shared_ptr<leafhash<FieldT, leaf_hash_type>> get_leafhash(
-    const bcs_hash_type hash_type, 
-    const size_t security_parameter, 
+    const bcs_hash_type hash_type,
+    const size_t security_parameter,
     const size_t leaf_size);
 
 template<typename hash_type, typename FieldT>
 two_to_one_hash_function<hash_type> get_two_to_one_hash(const bcs_hash_type hash_enum, const size_t security_parameter);
 
+template<typename hash_type, typename FieldT>
+cap_hash_function<hash_type> get_cap_hash(const bcs_hash_type hash_enum, const size_t security_parameter);
+
 }
 #include "libiop/bcs/hashing/hash_enum.tcc"