From a3038f91b99daeb42f96623e8441500211e4af62 Mon Sep 17 00:00:00 2001 From: Alexander Wu Date: Mon, 3 May 2021 00:54:32 -0700 Subject: [PATCH 1/8] Add cap hash function using blake 2b --- libiop/bcs/bcs_common.hpp | 1 + libiop/bcs/bcs_common.tcc | 2 + libiop/bcs/common_bcs_parameters.tcc | 6 +- libiop/bcs/hashing/blake2b.cpp | 2 +- libiop/bcs/hashing/blake2b.hpp | 7 ++ libiop/bcs/hashing/blake2b.tcc | 21 +++-- libiop/bcs/hashing/dummy_algebraic_hash.hpp | 3 + libiop/bcs/hashing/dummy_algebraic_hash.tcc | 11 +++ libiop/bcs/hashing/hash_enum.hpp | 9 +- libiop/bcs/hashing/hash_enum.tcc | 93 +++++++++++++++----- libiop/bcs/hashing/hashing.hpp | 4 + libiop/bcs/merkle_tree.hpp | 10 ++- libiop/bcs/merkle_tree.tcc | 13 ++- libiop/bcs/pow.tcc | 19 ++-- libiop/tests/bcs/test_bcs_transformation.cpp | 4 +- libiop/tests/bcs/test_merkle_tree.cpp | 2 + 16 files changed, 153 insertions(+), 54 deletions(-) diff --git a/libiop/bcs/bcs_common.hpp b/libiop/bcs/bcs_common.hpp index 3ad86dd9..3c690552 100644 --- a/libiop/bcs/bcs_common.hpp +++ b/libiop/bcs/bcs_common.hpp @@ -31,6 +31,7 @@ struct bcs_transformation_parameters { std::shared_ptr> hashchain_; std::shared_ptr> leafhasher_; two_to_one_hash_function compression_hasher; + cap_hash_function cap_hasher; }; template diff --git a/libiop/bcs/bcs_common.tcc b/libiop/bcs/bcs_common.tcc index a17a51d5..0bcd87ed 100644 --- a/libiop/bcs/bcs_common.tcc +++ b/libiop/bcs/bcs_common.tcc @@ -471,6 +471,7 @@ void bcs_protocol::seal_interaction_registrations() size, this->parameters_.leafhasher_, this->parameters_.compression_hasher, + this->parameters_.cap_hasher, this->digest_len_bytes_, make_zk, this->parameters_.security_parameter); @@ -724,6 +725,7 @@ void print_detailed_transcript_data( MT_size, params.leafhasher_, params.compression_hasher, + params.cap_hasher, digest_len_bytes, false, params.security_parameter); diff --git a/libiop/bcs/common_bcs_parameters.tcc b/libiop/bcs/common_bcs_parameters.tcc index 077219cb..27487055 100644 --- a/libiop/bcs/common_bcs_parameters.tcc +++ b/libiop/bcs/common_bcs_parameters.tcc @@ -15,10 +15,10 @@ bcs_transformation_parameters default_bcs_params( params.hash_enum = hash_type; /* TODO: Push setting leaf hash into internal BCS code. Currently 2 is fine, as leaf size is internally unused. */ const size_t leaf_size = 2; - params.leafhasher_ = get_leafhash(hash_type, security_parameter, leaf_size); + params.leafhasher_ = get_leafhash(hash_type, security_parameter, leaf_size); params.compression_hasher = get_two_to_one_hash(hash_type, security_parameter); - params.hashchain_ = - get_hashchain(hash_type, security_parameter); + params.cap_hasher = get_cap_hash(hash_type, security_parameter); + params.hashchain_ = get_hashchain(hash_type, security_parameter); // Work per hash. Todo generalize this w/ proper explanations of work amounts const size_t work_per_hash = (hash_type == 1) ? 1 : 128; diff --git a/libiop/bcs/hashing/blake2b.cpp b/libiop/bcs/hashing/blake2b.cpp index 87eb6caa..8cdee7e6 100644 --- a/libiop/bcs/hashing/blake2b.cpp +++ b/libiop/bcs/hashing/blake2b.cpp @@ -73,4 +73,4 @@ std::size_t blake2b_integer_randomness_extractor(const binary_hash_digest &root, return result % upper_bound; } -} +} // namespace libiop diff --git a/libiop/bcs/hashing/blake2b.hpp b/libiop/bcs/hashing/blake2b.hpp index bdf14bad..2ab5f908 100644 --- a/libiop/bcs/hashing/blake2b.hpp +++ b/libiop/bcs/hashing/blake2b.hpp @@ -58,6 +58,13 @@ class blake2b_leafhash : public leafhash const zk_salt_type &zk_salt); }; +/* Many-to-one hash which takes in a vector. */ +template +binary_hash_digest blake2b_vector_hash(const std::vector &data, + const std::size_t digest_len_bytes); + +/* This is a separate function from blake2b_vector_hash because we might want + to do some special handling on fields. */ template binary_hash_digest blake2b_field_element_hash(const std::vector &data, const std::size_t digest_len_bytes); diff --git a/libiop/bcs/hashing/blake2b.tcc b/libiop/bcs/hashing/blake2b.tcc index 4643fc89..cc17df9c 100644 --- a/libiop/bcs/hashing/blake2b.tcc +++ b/libiop/bcs/hashing/blake2b.tcc @@ -135,10 +135,8 @@ binary_hash_digest blake2b_leafhash::zk_hash( return blake2b_two_to_one_hash(leaf_hash, zk_salt, this->digest_len_bytes_); } -// TODO: Consider how this interacts with field elems being in montgomery form -// don't we need to make them in canonical form first? -template -binary_hash_digest blake2b_field_element_hash(const std::vector &data, +template +binary_hash_digest blake2b_vector_hash(const std::vector &data, const std::size_t digest_len_bytes) { @@ -148,17 +146,24 @@ binary_hash_digest blake2b_field_element_hash(const std::vector &data, const int status = crypto_generichash_blake2b((unsigned char*)&result[0], digest_len_bytes, (result.empty() ? NULL : (unsigned char*)&data[0]), - sizeof(FieldT) * data.size(), + sizeof(hash_type) * data.size(), NULL, 0); if (status != 0) { throw std::runtime_error("Got non-zero status from crypto_generichash_blake2b. (Is digest_len_bytes correct?)"); } - - return result; } +// TODO: Consider how this interacts with field elems being in montgomery form +// don't we need to make them in canonical form first? +template +binary_hash_digest blake2b_field_element_hash(const std::vector &data, + const std::size_t digest_len_bytes) +{ + return blake2b_vector_hash(data, digest_len_bytes); +} + template FieldT blake2b_FieldT_rejection_sample( typename libff::enable_if::value, FieldT>::type _, @@ -256,4 +261,4 @@ std::vector blake2b_FieldT_randomness_extractor(const binary_hash_digest return result; } -} +} // namespace libiop diff --git a/libiop/bcs/hashing/dummy_algebraic_hash.hpp b/libiop/bcs/hashing/dummy_algebraic_hash.hpp index f765ccdf..652ae220 100644 --- a/libiop/bcs/hashing/dummy_algebraic_hash.hpp +++ b/libiop/bcs/hashing/dummy_algebraic_hash.hpp @@ -63,6 +63,9 @@ FieldT dummy_algebraic_two_to_one_hash( const FieldT &second, const std::size_t digest_len_bytes); +template +FieldT dummy_algebraic_cap_hash(const std::vector &data, const std::size_t digest_len_bytes); + } // namespace libiop #include "libiop/bcs/hashing/dummy_algebraic_hash.tcc" diff --git a/libiop/bcs/hashing/dummy_algebraic_hash.tcc b/libiop/bcs/hashing/dummy_algebraic_hash.tcc index e8c0b10b..5c35acb1 100644 --- a/libiop/bcs/hashing/dummy_algebraic_hash.tcc +++ b/libiop/bcs/hashing/dummy_algebraic_hash.tcc @@ -147,4 +147,15 @@ FieldT dummy_algebraic_two_to_one_hash( return FieldT(2) * first + second; } +template +FieldT dummy_algebraic_cap_hash(const std::vector &data, const std::size_t digest_len_bytes) +{ + FieldT sum = FieldT::zero(); + for (size_t i = 0; i < data.size(); i++) + { + sum += FieldT(i) * data[i]; + } + return sum; +} + } diff --git a/libiop/bcs/hashing/hash_enum.hpp b/libiop/bcs/hashing/hash_enum.hpp index f7902991..e42c7eab 100644 --- a/libiop/bcs/hashing/hash_enum.hpp +++ b/libiop/bcs/hashing/hash_enum.hpp @@ -30,15 +30,18 @@ static const char* bcs_hash_type_names[] = {"", "blake2b", "poseidon with Starkw template std::shared_ptr> get_hashchain(bcs_hash_type hash_type, size_t security_parameter); -template +template std::shared_ptr> get_leafhash( - const bcs_hash_type hash_type, - const size_t security_parameter, + const bcs_hash_type hash_type, + const size_t security_parameter, const size_t leaf_size); template two_to_one_hash_function get_two_to_one_hash(const bcs_hash_type hash_enum, const size_t security_parameter); +template +cap_hash_function get_cap_hash(const bcs_hash_type hash_enum, const size_t security_parameter); + } #include "libiop/bcs/hashing/hash_enum.tcc" diff --git a/libiop/bcs/hashing/hash_enum.tcc b/libiop/bcs/hashing/hash_enum.tcc index 8942c756..4a6a0d43 100644 --- a/libiop/bcs/hashing/hash_enum.tcc +++ b/libiop/bcs/hashing/hash_enum.tcc @@ -68,8 +68,23 @@ std::shared_ptr> get_hashchain(bcs_hash_type has return get_hashchain_internal(FieldT::zero(), hash_enum, security_parameter); } +/* Binary_hash_digest leafhash */ +template +std::shared_ptr> get_leafhash_internal( + const typename libff::enable_if::value, FieldT>::type _, + const bcs_hash_type hash_enum, + const size_t security_parameter, + const size_t leaf_size) +{ + if (hash_enum == blake2b_type) + { + return std::make_shared>(security_parameter); + } + throw std::invalid_argument("bcs_hash_type unknown"); +} + /* Algebraic leafhash case */ -template +template std::shared_ptr> get_leafhash_internal( const typename libff::enable_if::value, FieldT>::type _, const bcs_hash_type hash_enum, @@ -93,33 +108,18 @@ std::shared_ptr> get_leafhash_internal( throw std::invalid_argument("bcs_hash_type unknown (algebraic leaf hash)"); } -/* Binary_hash_digest leafhash */ -template -std::shared_ptr> get_leafhash_internal( - const typename libff::enable_if::value, FieldT>::type _, - const bcs_hash_type hash_enum, - const size_t security_parameter, - const size_t leaf_size) -{ - if (hash_enum == blake2b_type) - { - return std::make_shared>(security_parameter); - } - throw std::invalid_argument("bcs_hash_type unknown"); -} - -template +template std::shared_ptr> get_leafhash( const bcs_hash_type hash_enum, const size_t security_parameter, const size_t leaf_size) { - return get_leafhash_internal(FieldT::zero(), hash_enum, security_parameter, leaf_size); + return get_leafhash_internal(FieldT::zero(), hash_enum, security_parameter, leaf_size); } /* binary hash digest 2->1 hash */ template two_to_one_hash_function get_two_to_one_hash_internal( - const typename libff::enable_if::value, FieldT>::type _, - const bcs_hash_type hash_enum, + const typename libff::enable_if::value, FieldT>::type _, + const bcs_hash_type hash_enum, const size_t security_parameter) { if (hash_enum == blake2b_type) @@ -132,8 +132,8 @@ two_to_one_hash_function get_two_to_one_hash_internal( /* algebraic 2->1 hash */ template two_to_one_hash_function get_two_to_one_hash_internal( - const typename libff::enable_if::value, FieldT>::type _, - const bcs_hash_type hash_enum, + const typename libff::enable_if::value, FieldT>::type _, + const bcs_hash_type hash_enum, const size_t security_parameter) { if (hash_enum == starkware_poseidon_type || hash_enum == high_alpha_poseidon_type) @@ -164,4 +164,53 @@ two_to_one_hash_function get_two_to_one_hash(const bcs_hash_type hash return get_two_to_one_hash_internal(FieldT::zero(), hash_enum, security_parameter); } +/* Hash digest 2^n->1 hash. */ +template +cap_hash_function get_cap_hash_internal( + const typename libff::enable_if::value, FieldT>::type _, + const bcs_hash_type hash_enum, + const size_t security_parameter) +{ + if (hash_enum == blake2b_type) + { + return blake2b_vector_hash; + } + throw std::invalid_argument("bcs_hash_type unknown"); +} + +/* Algebraic 2^n->1 hash. */ +template +cap_hash_function get_cap_hash_internal( + const typename libff::enable_if::value, FieldT>::type _, + const bcs_hash_type hash_enum, + const size_t security_parameter) +{ + // if (hash_enum == starkware_poseidon_type || hash_enum == high_alpha_poseidon_type) + // { + // if (security_parameter != 128) + // { + // throw std::invalid_argument("Poseidon only supported for 128 bit soundness."); + // } + // poseidon_params params = get_poseidon_parameters(hash_enum); + // /* security parameter is -1 b/c */ + // std::shared_ptr> permutation = std::make_shared>(params); + // /* We explicitly place this on heap with no destructor, + // as this reference has to live after the function terminates */ + // std::shared_ptr> hash_class = + // std::make_shared>(permutation, security_parameter - 1); + // std::function f = [permutation, hash_class](const FieldT& left, const FieldT& right, const std::size_t unused) -> FieldT + // { + // return hash_class->hash(left, right); + // }; + // return f; + // } + throw std::invalid_argument("bcs_hash_type unknown (algebraic cap hash)"); +} + +template +cap_hash_function get_cap_hash(const bcs_hash_type hash_enum, const size_t security_parameter) +{ + return get_cap_hash_internal(FieldT::zero(), hash_enum, security_parameter); } + +} // namespace libiop diff --git a/libiop/bcs/hashing/hashing.hpp b/libiop/bcs/hashing/hashing.hpp index 29473a1c..a4f7d977 100644 --- a/libiop/bcs/hashing/hashing.hpp +++ b/libiop/bcs/hashing/hashing.hpp @@ -52,6 +52,10 @@ class leafhash template using two_to_one_hash_function = std::function; +/* Function used for cap hash of merkle tree which takes in a vector of size 2^n. */ +template +using cap_hash_function = std::function&, const std::size_t)>; + /* Sizeof algebraic hash */ template size_t get_hash_size(const typename libff::enable_if::value, hash_type>::type h) diff --git a/libiop/bcs/merkle_tree.hpp b/libiop/bcs/merkle_tree.hpp index 522328eb..d48751e5 100644 --- a/libiop/bcs/merkle_tree.hpp +++ b/libiop/bcs/merkle_tree.hpp @@ -54,6 +54,8 @@ class merkle_tree { std::size_t digest_len_bytes_; bool make_zk_; std::size_t num_zk_bytes_; + cap_hash_function cap_hasher_; + std::size_t cap_size_; /* Each element will be hashed (individually) to produce a random hash digest. */ std::vector zk_leaf_randomness_elements_; @@ -62,14 +64,16 @@ class merkle_tree { public: /* Create a merkle tree with the given configuration. If make_zk is true, 2 * security parameter random bytes will be appended to each leaf - before hashing, to prevent a low entropy leaf value from being inferred - from its hash. */ + before hashing, to prevent a low entropy leaf value from being inferred from its hash. + cap_size is the number of children of the root and must be a power of 2. */ merkle_tree(const std::size_t num_leaves, const std::shared_ptr> &leaf_hasher, const two_to_one_hash_function &node_hasher, + const cap_hash_function &cap_hasher, const std::size_t digest_len_bytes, const bool make_zk, - const std::size_t security_parameter); + const std::size_t security_parameter, + const std::size_t cap_size=2); /** This treats each leaf as a column. * e.g. The ith leaf is the vector formed by leaf_contents[j][i] for all j */ diff --git a/libiop/bcs/merkle_tree.tcc b/libiop/bcs/merkle_tree.tcc index a86cf68d..7382b59a 100644 --- a/libiop/bcs/merkle_tree.tcc +++ b/libiop/bcs/merkle_tree.tcc @@ -14,15 +14,19 @@ merkle_tree::merkle_tree( const std::size_t num_leaves, const std::shared_ptr> &leaf_hasher, const two_to_one_hash_function &node_hasher, + const cap_hash_function &cap_hasher, const std::size_t digest_len_bytes, const bool make_zk, - const std::size_t security_parameter) : + const std::size_t security_parameter, + const std::size_t cap_size) : num_leaves_(num_leaves), leaf_hasher_(leaf_hasher), node_hasher_(node_hasher), + cap_hasher_(cap_hasher), digest_len_bytes_(digest_len_bytes), make_zk_(make_zk), - num_zk_bytes_((security_parameter * 2 + 7) / 8) /* = ceil((2 * security_parameter_bits) / 8) */ + num_zk_bytes_((security_parameter * 2 + 7) / 8), /* = ceil((2 * security_parameter_bits) / 8) */ + cap_size_(cap_size) { if (num_leaves < 2 || !libff::is_power_of_2(num_leaves)) { @@ -30,6 +34,11 @@ merkle_tree::merkle_tree( throw std::invalid_argument("Merkle tree size must be a power of two, and at least 2."); } + if (cap_size < 2 || !libff::is_power_of_2(cap_size)) + { + throw std::invalid_argument("Merkle tree cap size must be a power of two, and at least 2."); + } + this->constructed_ = false; } diff --git a/libiop/bcs/pow.tcc b/libiop/bcs/pow.tcc index 184eb873..f361bfee 100644 --- a/libiop/bcs/pow.tcc +++ b/libiop/bcs/pow.tcc @@ -76,7 +76,7 @@ hash_digest_type pow::solve_pow_internal( const typename libff::enable_if::value, hash_digest_type>::type challenge) const { FieldT pow = FieldT::zero(); - while (this->verify_pow(node_hasher, challenge, pow) == false) + while (!this->verify_pow(node_hasher, challenge, pow)) { pow += FieldT::one(); } @@ -93,8 +93,9 @@ hash_digest_type pow::solve_pow_internal( size_t num_words = pow.length() / sizeof(size_t); size_t pow_int = 0; - while (this->verify_pow(node_hasher, challenge, pow) == false) + while (!this->verify_pow(node_hasher, challenge, pow)) { + // printf("Trying %zx\n", pow[(num_words - 1)*sizeof(size_t)]); std::memcpy(&pow[(num_words - 1)*sizeof(size_t)], &pow_int, sizeof(size_t)); pow_int += 1; } @@ -147,15 +148,11 @@ bool pow::verify_pow_internal( size_t least_significant_word; std::memcpy(&least_significant_word, &hash[(num_words - 1)*sizeof(size_t)], sizeof(size_t)); size_t relevant_bits = least_significant_word & ((1 << this->parameters_.pow_bitlen()) - 1); - if (relevant_bits <= this->parameters_.pow_upperbound()) - { - // printf("%d\n", (1 << this->parameters_.pow_bitlen())); - // printf("%zu\n", least_significant_word); - // printf("%\n", relevant_bits); - // print_string_in_hex(hash); - return true; - } - return false; + // printf("upper bound: %zx\n", this->parameters_.pow_upperbound()); + // printf("bit mask: %zx\n", (1 << this->parameters_.pow_bitlen()) - 1); + // printf("least significant word: %zx\n", least_significant_word); + // printf("relevant bits: %zx\n", relevant_bits); + return relevant_bits <= this->parameters_.pow_upperbound(); } } // libiop diff --git a/libiop/tests/bcs/test_bcs_transformation.cpp b/libiop/tests/bcs/test_bcs_transformation.cpp index 3e07fa70..142b98a0 100644 --- a/libiop/tests/bcs/test_bcs_transformation.cpp +++ b/libiop/tests/bcs/test_bcs_transformation.cpp @@ -44,6 +44,7 @@ void set_bcs_parameters_leafhash(bcs_transformation_parameters>(security_parameter); params.compression_hasher = blake2b_two_to_one_hash; + params.cap_hasher = blake2b_vector_hash; } // Algebraic case @@ -52,7 +53,8 @@ template ¶ms) { params.leafhasher_ = std::make_shared>(); - params.compression_hasher = dummy_algebraic_two_to_one_hash; + params.compression_hasher = dummy_algebraic_two_to_one_hash; + params.cap_hasher = dummy_algebraic_cap_hash; } template diff --git a/libiop/tests/bcs/test_merkle_tree.cpp b/libiop/tests/bcs/test_merkle_tree.cpp index 8bafcf92..fc6ba829 100644 --- a/libiop/tests/bcs/test_merkle_tree.cpp +++ b/libiop/tests/bcs/test_merkle_tree.cpp @@ -25,6 +25,7 @@ merkle_tree new_MT( size, std::make_shared>(security_parameter), blake2b_two_to_one_hash, + blake2b_vector_hash, digest_len_bytes, make_zk, security_parameter); @@ -39,6 +40,7 @@ merkle_tree new_MT( size, std::make_shared>(), dummy_algebraic_two_to_one_hash, + dummy_algebraic_cap_hash, digest_len_bytes, make_zk, security_parameter); From e03582d4b234645015a343783262ccde1a2b1a32 Mon Sep 17 00:00:00 2001 From: Alexander Wu Date: Wed, 5 May 2021 01:24:17 -0700 Subject: [PATCH 2/8] Add cap hashing mechanism to merkle tree --- libiop/bcs/hashing/blake2b.cpp | 29 +++ libiop/bcs/hashing/blake2b.hpp | 20 +- libiop/bcs/hashing/blake2b.tcc | 21 +- libiop/bcs/hashing/dummy_algebraic_hash.tcc | 5 +- libiop/bcs/hashing/hash_enum.tcc | 2 +- libiop/bcs/merkle_tree.hpp | 19 ++ libiop/bcs/merkle_tree.tcc | 207 +++++++++++++------ libiop/tests/bcs/test_bcs_transformation.cpp | 2 +- libiop/tests/bcs/test_merkle_tree.cpp | 79 ++++--- 9 files changed, 265 insertions(+), 119 deletions(-) diff --git a/libiop/bcs/hashing/blake2b.cpp b/libiop/bcs/hashing/blake2b.cpp index 8cdee7e6..2006e2b2 100644 --- a/libiop/bcs/hashing/blake2b.cpp +++ b/libiop/bcs/hashing/blake2b.cpp @@ -29,6 +29,8 @@ binary_hash_digest blake2b_two_to_one_hash(const binary_hash_digest &first, const binary_hash_digest &second, const std::size_t digest_len_bytes) { + /* binary_hash_digest is a C++ string pointer so we need to sum them since they are not + contiguous in memory. */ const binary_hash_digest first_plus_second = first + second; binary_hash_digest result(digest_len_bytes, 'X'); @@ -47,6 +49,33 @@ binary_hash_digest blake2b_two_to_one_hash(const binary_hash_digest &first, return result; } +binary_hash_digest blake2b_many_to_one_hash(const std::vector &data, + const std::size_t digest_len_bytes) +{ + /* binary_hash_digest is a C++ string pointer so we need to sum them since they are not + contiguous in memory. */ + binary_hash_digest input = data[0]; + for (int i = 1; i < data.size(); i++) + { + input += data[i]; + } + + binary_hash_digest result(digest_len_bytes, 'X'); + + /* see https://download.libsodium.org/doc/hashing/generic_hashing.html */ + const int status = crypto_generichash_blake2b((unsigned char*)&result[0], + digest_len_bytes, + (result.empty() ? NULL : (unsigned char*)&input[0]), + input.size(), + NULL, 0); + if (status != 0) + { + throw std::runtime_error("Got non-zero status from crypto_generichash_blake2b. (Is digest_len_bytes correct?)"); + } + + return result; +} + std::size_t blake2b_integer_randomness_extractor(const binary_hash_digest &root, const std::size_t index, const std::size_t upper_bound) diff --git a/libiop/bcs/hashing/blake2b.hpp b/libiop/bcs/hashing/blake2b.hpp index 2ab5f908..78a63c44 100644 --- a/libiop/bcs/hashing/blake2b.hpp +++ b/libiop/bcs/hashing/blake2b.hpp @@ -58,13 +58,8 @@ class blake2b_leafhash : public leafhash const zk_salt_type &zk_salt); }; -/* Many-to-one hash which takes in a vector. */ -template -binary_hash_digest blake2b_vector_hash(const std::vector &data, - const std::size_t digest_len_bytes); - -/* This is a separate function from blake2b_vector_hash because we might want - to do some special handling on fields. */ +/* Many-to-one hash which takes in a vector of field elements. + Behavior undefined when data is empty. */ template binary_hash_digest blake2b_field_element_hash(const std::vector &data, const std::size_t digest_len_bytes); @@ -80,11 +75,16 @@ std::size_t blake2b_integer_randomness_extractor(const binary_hash_digest &root, const std::size_t upper_bound); binary_hash_digest blake2b_zk_element_hash(const std::vector &first, - const std::size_t digest_len_bytes); + const std::size_t digest_len_bytes); binary_hash_digest blake2b_two_to_one_hash(const binary_hash_digest &first, - const binary_hash_digest &second, - const std::size_t digest_len_bytes); + const binary_hash_digest &second, + const std::size_t digest_len_bytes); + +/* Many-to-one hash which takes in a vector of binary_hash_digest. + Behavior undefined when data is empty. */ +binary_hash_digest blake2b_many_to_one_hash(const std::vector &data, + const std::size_t digest_len_bytes); } // namespace libiop diff --git a/libiop/bcs/hashing/blake2b.tcc b/libiop/bcs/hashing/blake2b.tcc index cc17df9c..0375aab1 100644 --- a/libiop/bcs/hashing/blake2b.tcc +++ b/libiop/bcs/hashing/blake2b.tcc @@ -135,33 +135,26 @@ binary_hash_digest blake2b_leafhash::zk_hash( return blake2b_two_to_one_hash(leaf_hash, zk_salt, this->digest_len_bytes_); } -template -binary_hash_digest blake2b_vector_hash(const std::vector &data, - const std::size_t digest_len_bytes) +// TODO: Consider how this interacts with field elems being in montgomery form +// don't we need to make them in canonical form first? +template +binary_hash_digest blake2b_field_element_hash(const std::vector &data, + const std::size_t digest_len_bytes) { - binary_hash_digest result(digest_len_bytes, 'X'); /* see https://download.libsodium.org/doc/hashing/generic_hashing.html */ const int status = crypto_generichash_blake2b((unsigned char*)&result[0], digest_len_bytes, (result.empty() ? NULL : (unsigned char*)&data[0]), - sizeof(hash_type) * data.size(), + sizeof(FieldT) * data.size(), NULL, 0); if (status != 0) { throw std::runtime_error("Got non-zero status from crypto_generichash_blake2b. (Is digest_len_bytes correct?)"); } - return result; -} -// TODO: Consider how this interacts with field elems being in montgomery form -// don't we need to make them in canonical form first? -template -binary_hash_digest blake2b_field_element_hash(const std::vector &data, - const std::size_t digest_len_bytes) -{ - return blake2b_vector_hash(data, digest_len_bytes); + return result; } template diff --git a/libiop/bcs/hashing/dummy_algebraic_hash.tcc b/libiop/bcs/hashing/dummy_algebraic_hash.tcc index 5c35acb1..dc6c9337 100644 --- a/libiop/bcs/hashing/dummy_algebraic_hash.tcc +++ b/libiop/bcs/hashing/dummy_algebraic_hash.tcc @@ -117,7 +117,7 @@ FieldT dummy_algebraic_leafhash::hash(const std::vector &leaf) FieldT sum = FieldT::zero(); for (size_t i = 0; i < leaf.size(); i++) { - sum += FieldT(i) * leaf[i]; + sum += FieldT(i + 1) * leaf[i]; // Add one, otherwise the 0th index is unused. } return sum; } @@ -153,8 +153,9 @@ FieldT dummy_algebraic_cap_hash(const std::vector &data, const std::size FieldT sum = FieldT::zero(); for (size_t i = 0; i < data.size(); i++) { - sum += FieldT(i) * data[i]; + sum += FieldT(i + 1) * data[i]; // Add one, otherwise the 0th index is unused. } + return sum; } diff --git a/libiop/bcs/hashing/hash_enum.tcc b/libiop/bcs/hashing/hash_enum.tcc index 4a6a0d43..0de8b720 100644 --- a/libiop/bcs/hashing/hash_enum.tcc +++ b/libiop/bcs/hashing/hash_enum.tcc @@ -173,7 +173,7 @@ cap_hash_function get_cap_hash_internal( { if (hash_enum == blake2b_type) { - return blake2b_vector_hash; + return blake2b_many_to_one_hash; } throw std::invalid_argument("bcs_hash_type unknown"); } diff --git a/libiop/bcs/merkle_tree.hpp b/libiop/bcs/merkle_tree.hpp index d48751e5..8e390559 100644 --- a/libiop/bcs/merkle_tree.hpp +++ b/libiop/bcs/merkle_tree.hpp @@ -46,6 +46,9 @@ template class merkle_tree { protected: bool constructed_; + /* inner_nodes_ is a vector of the (num_leaves - 1) nodes in the tree, with the root at + index 0, left child at 1, right child at 2, etc. If cap_size_ is greater than 2, the + first (log_2(cap_size_) - 1) layers under the root are empty to make the math easier. */ std::vector inner_nodes_; std::size_t num_leaves_; @@ -54,13 +57,29 @@ class merkle_tree { std::size_t digest_len_bytes_; bool make_zk_; std::size_t num_zk_bytes_; + /* The top log_2(cap_size_) layers are hashed with a single computation to improve efficiency. + The root along with its cap_size_ direct children are referred to as the "cap," and the + operation that transforms these children to the root is the cap hash. + See https://github.com/scipr-lab/libiop/issues/41. */ cap_hash_function cap_hasher_; + /* cap_size_ is the number of direct children the root has. It must be a power of 2 and at + least 2. For example if cap_size == 4, the root has 4 children, and in inner_nodes_ the + indices 1 and 2 are unused. */ std::size_t cap_size_; /* Each element will be hashed (individually) to produce a random hash digest. */ std::vector zk_leaf_randomness_elements_; void sample_leaf_randomness(); void compute_inner_nodes(); + + /* Helper functions for dealing with the tree strucutre. Correctness not guaranteed + when out of bounds. */ + std::size_t parent_of(const std::size_t node_index) const; + std::size_t left_child_of(const std::size_t node_index) const; + std::size_t right_child_of(const std::size_t node_index) const; + bool is_in_cap(const std::size_t node_index) const; + std::size_t cap_children_start() const; // Inclusive. + std::size_t cap_children_end() const; // Exclusive. public: /* Create a merkle tree with the given configuration. If make_zk is true, 2 * security parameter random bytes will be appended to each leaf diff --git a/libiop/bcs/merkle_tree.tcc b/libiop/bcs/merkle_tree.tcc index 7382b59a..0c155b77 100644 --- a/libiop/bcs/merkle_tree.tcc +++ b/libiop/bcs/merkle_tree.tcc @@ -9,16 +9,18 @@ namespace libiop { +using std::size_t; + template merkle_tree::merkle_tree( - const std::size_t num_leaves, + const size_t num_leaves, const std::shared_ptr> &leaf_hasher, const two_to_one_hash_function &node_hasher, const cap_hash_function &cap_hasher, - const std::size_t digest_len_bytes, + const size_t digest_len_bytes, const bool make_zk, - const std::size_t security_parameter, - const std::size_t cap_size) : + const size_t security_parameter, + const size_t cap_size) : num_leaves_(num_leaves), leaf_hasher_(leaf_hasher), node_hasher_(node_hasher), @@ -129,7 +131,7 @@ void merkle_tree::construct_with_leaves_serialized_by_ * our slice is of size num_input_oracles * coset_size */ std::vector slice(leaf_contents.size() * coset_serialization_size, FieldT::zero()); - for (std::size_t i = 0; i < this->num_leaves_; ++i) + for (size_t i = 0; i < this->num_leaves_; ++i) { std::vector positions_in_this_slice = leaf_domain.all_positions_in_coset_i(i, coset_serialization_size); @@ -209,32 +211,39 @@ std::vector> merkle_tree::serializ template void merkle_tree::compute_inner_nodes() { - // TODO: Better document this function, its hashing layer by layer. - std::size_t n = (this->num_leaves_ - 1) / 2; + /* n is the first index of the layer we're about to compute. It starts at the bottom layer. + This hack works because num_leaves is the index of the right child of the bottom-left node. */ + size_t n = this->parent_of(this->num_leaves_); while (true) { // TODO: Evaluate how much time is spent in hashing vs memory access. // For better memory efficiency, we could hash sub-tree by sub-tree // in an unrolled recursive fashion. - for (std::size_t j = n; j <= 2*n; ++j) + for (size_t j = n; j <= 2*n; ++j) { // TODO: Can we rely on left and right to be placed sequentially in memory, // for better performance in node hasher? - const hash_digest_type& left = this->inner_nodes_[2*j + 1]; - const hash_digest_type& right = this->inner_nodes_[2*j + 2]; + const hash_digest_type& left = this->inner_nodes_[this->left_child_of(j)]; + const hash_digest_type& right = this->inner_nodes_[this->right_child_of(j)]; const hash_digest_type digest = this->node_hasher_(left, right, this->digest_len_bytes_); this->inner_nodes_[j] = digest; } - if (n > 0) - { - n /= 2; - } - else + if (this->is_in_cap(n)) { + /* We are done with the main portion after the cap layer is filled out. + There is one edge case where the entire tree is the cap, and in that case we + will do some extra work, but it will still correctly compute everything. */ break; } + n /= 2; // Go to the layer obove this one. } + + // Now compute the cap hash. + auto cap_children_start = this->inner_nodes_.begin() + this->cap_children_start(); + auto cap_children_end = this->inner_nodes_.begin() + this->cap_children_end(); + std::vector cap_children(cap_children_start, cap_children_end); + this->inner_nodes_[0] = this->cap_hasher_(cap_children, this->digest_len_bytes_); } template @@ -251,7 +260,7 @@ hash_digest_type merkle_tree::get_root() const template merkle_tree_set_membership_proof merkle_tree::get_set_membership_proof( - const std::vector &positions) const + const std::vector &positions) const { if (!this->constructed_) { @@ -264,12 +273,12 @@ merkle_tree_set_membership_proof return result; } - std::vector S = positions; /* sorted set of positions */ + std::vector S = positions; /* sorted set of positions */ std::sort(S.begin(), S.end()); S.erase(std__unique(S.begin(), S.end()), S.end()); /* remove possible duplicates */ if (std::any_of(S.begin(), S.end(), - [this](const std::size_t pos) { return pos >= this->num_leaves_; })) + [this](const size_t pos) { return pos >= this->num_leaves_; })) { throw std::invalid_argument("All positions must be between 0 and num_leaves-1."); } @@ -289,37 +298,37 @@ merkle_tree_set_membership_proof /* transform leaf positions to indices in this->inner_nodes_ */ for (auto &pos : S) { - pos += (this->num_leaves_ - 1); + pos += this->num_leaves_ - 1; } - while (true) /* for every layer */ + // Each iteration adds the hashes for one layer, up until the layer below the cap. + while (true) { auto it = S.begin(); - if (*it == 0 && it == --S.end()) - { - /* we have arrived at the root */ + if (is_in_cap(*it)) + { // We have arrived at the cap, which will be handled differently. break; } - std::vector new_S; + // new_S contains the hash indices we need in the layer above this one. + std::vector new_S; while (it != S.end()) { - const std::size_t it_pos = *it; + const size_t it_pos = *it; auto next_it = ++it; /* Always process parent. */ - new_S.emplace_back((it_pos - 1)/2); + new_S.emplace_back(this->parent_of(it_pos)); - if ((it_pos & 1) == 0) + if (it_pos % 2 == 0) { - /* We are the right node, so there was no left node - (o.w. would have been processed in b) - below). Insert it as auxiliary */ + /* it_pos is a right node, so there was no left node (otherwise it would have been + processed already). Insert left node as auxiliary */ result.auxiliary_hashes.emplace_back(this->inner_nodes_[it_pos - 1]); } else { - /* We are the left node. Two cases: */ + /* it_pos is a left node. Two cases: */ if (next_it == S.end() || *next_it != it_pos + 1) { /* a) Our right sibling is not in S, so we must @@ -330,8 +339,7 @@ merkle_tree_set_membership_proof { /* b) Our right sibling is in S. So don't need auxiliary and skip over the right sibling. - (Note that only one parent will be processed.) - */ + (Note that only one parent will be processed.) */ ++next_it; } } @@ -341,13 +349,33 @@ merkle_tree_set_membership_proof std::swap(S, new_S); } + // Add the cap, including the root's direct children and the root. + // The only elements should be the cap (not including the root). + assert(S.size() <= this->cap_size_); + auto it = S.begin(); + // Iterate over every direct child of the root, and add the ones not obtainable from positions. + for (size_t j = this->cap_children_start(); j < this->cap_children_end(); j++) + { + // Since S is sorted, we can just compare to the next element of S. + if (j == *it) + { + it++; + } + else + { + result.auxiliary_hashes.emplace_back(this->inner_nodes_[j]); + } + } + return result; } +/* Large portions of this code is duplicated from get_set_membership_proof, but it's just + different enough that I can't extract them into a single function. */ template bool merkle_tree::validate_set_membership_proof( const hash_digest_type &root, - const std::vector &positions, + const std::vector &positions, const std::vector> &leaf_contents, const merkle_tree_set_membership_proof &proof) { @@ -371,7 +399,7 @@ bool merkle_tree::validate_set_membership_proof( auto rand_it = proof.randomness_hashes.begin(); auto aux_it = proof.auxiliary_hashes.begin(); - typedef std::pair pos_and_digest_t; + typedef std::pair pos_and_digest_t; std::vector S; S.reserve(positions.size()); @@ -382,7 +410,9 @@ bool merkle_tree::validate_set_membership_proof( const zk_salt_type zk_salt = *rand_it++; leaf_hashes.emplace_back(this->leaf_hasher_->zk_hash(leaf, zk_salt)); } - } else { + } + else + { for (auto &leaf : leaf_contents) { leaf_hashes.emplace_back(this->leaf_hasher_->hash(leaf)); @@ -393,7 +423,7 @@ bool merkle_tree::validate_set_membership_proof( // with a single transform at the bottom. std::transform(positions.begin(), positions.end(), leaf_hashes.begin(), std::back_inserter(S), - [](const std::size_t pos, const hash_digest_type &hash) { + [](const size_t pos, const hash_digest_type &hash) { return std::make_pair(pos, hash); }); @@ -416,25 +446,26 @@ bool merkle_tree::validate_set_membership_proof( throw std::invalid_argument("All positions must be between 0 and num_leaves-1."); } - /* transform to sorted set of indices */ + /* transform to set of indices */ for (auto &pos : S) { - pos.first += (this->num_leaves_ - 1); + pos.first += this->num_leaves_ - 1; } - while (true) /* for every layer */ + // Each iteration calculates the hashes for one layer, up until the layer below the cap. + while (true) { auto it = S.begin(); - if (it->first == 0 && it == --S.end()) - { - /* we have arrived at the root */ + if (is_in_cap(it->first)) + { // We have arrived at the cap. The cap is hashed differently, so we stop here. break; } - std::vector > new_S; + // new_S contains the indices and hashes we calculate in the layer above this one. + std::vector > new_S; while (it != S.end()) { - const std::size_t it_pos = it->first; + const size_t it_pos = it->first; const hash_digest_type it_hash = it->second; auto next_it = ++it; @@ -442,17 +473,16 @@ bool merkle_tree::validate_set_membership_proof( hash_digest_type left_hash; hash_digest_type right_hash; - if ((it_pos & 1) == 0) + if (it_pos % 2 == 0) { - /* We are the right node, so there was no left node - (o.w. would have been processed in b) - below). Take it from the auxiliary. */ + /* it_pos is a right node, so there was no left node (otherwise it would have been + processed already). Take left node from the auxiliary. */ left_hash = *aux_it++; right_hash = it_hash; } else { - /* We are the left node. Two cases: */ + /* it_pos is a left node. Two cases: */ left_hash = it_hash; if (next_it == S.end() || next_it->first != it_pos + 1) @@ -463,16 +493,14 @@ bool merkle_tree::validate_set_membership_proof( } else { - /* b) Our right sibling is in S. So don't need - auxiliary and skip over the right sibling. - (Note that only one parent will be processed.) - */ + /* b) Our right sibling is in S. So don't need auxiliary and skip over the + right sibling The parent will be obtained) next iteration. */ right_hash = next_it->second; ++next_it; } } - const std::size_t parent_pos = (it_pos - 1)/2; + const size_t parent_pos = this->parent_of(it_pos); const hash_digest_type parent_hash = this->node_hasher_(left_hash, right_hash, this->digest_len_bytes_); new_S.emplace_back(std::make_pair(parent_pos, parent_hash)); @@ -483,17 +511,41 @@ bool merkle_tree::validate_set_membership_proof( std::swap(S, new_S); } + // Add the cap, including the root's direct children and the root. + // The only elements should be the cap (not including the root). + assert(S.size() <= this->cap_size_); + + auto it = S.begin(); + std::vector cap_children; + cap_children.reserve(this->cap_size_); + /* Iterate over every direct child of the root, choosing either the calculated hash or + auxiliary hash. */ + for (size_t j = this->cap_children_start(); j < this->cap_children_end(); j++) + { + // Since S is sorted, we can just compare to the next element of S. + if (it != S.end() && j == it->first) + { + cap_children.emplace_back(it->second); + it++; + } + else + { + cap_children.emplace_back(*aux_it); + aux_it++; + } + } + if (aux_it != proof.auxiliary_hashes.end()) { throw std::logic_error("Validation did not consume the entire proof."); } - return (S.begin()->second == root); + return this->cap_hasher_(cap_children, this->digest_len_bytes_) == root; } template size_t merkle_tree::count_hashes_to_verify_set_membership_proof( - const std::vector &positions) const + const std::vector &positions) const { /** This goes layer by layer, * and counts the number of hashes needed to be computed. @@ -524,13 +576,13 @@ size_t merkle_tree::count_hashes_to_verify_set_members } template -std::size_t merkle_tree::num_leaves() const +size_t merkle_tree::num_leaves() const { return (this->num_leaves_); } template -std::size_t merkle_tree::depth() const +size_t merkle_tree::depth() const { return libff::log2(this->num_leaves_); } @@ -542,10 +594,45 @@ bool merkle_tree::zk() const } template -std::size_t merkle_tree::num_total_bytes() const +size_t merkle_tree::num_total_bytes() const { return (this->digest_len_bytes_ * (2 * this->num_leaves() - 1)); } +template +size_t merkle_tree::parent_of(const std::size_t node_index) const +{ + return (node_index - 1) / 2; +} + +template +size_t merkle_tree::left_child_of(const std::size_t node_index) const +{ + return 2 * node_index + 1; +} + +template +size_t merkle_tree::right_child_of(const std::size_t node_index) const +{ + return 2 * node_index + 2; +} + +template +bool merkle_tree::is_in_cap(const std::size_t node_index) const +{ + return node_index < this->cap_children_end(); +} + +template +size_t merkle_tree::cap_children_start() const +{ + return this->cap_size_ - 1; +} + +template +size_t merkle_tree::cap_children_end() const +{ + return this->cap_size_ * 2 - 1; +} } // libiop diff --git a/libiop/tests/bcs/test_bcs_transformation.cpp b/libiop/tests/bcs/test_bcs_transformation.cpp index 142b98a0..688cfa88 100644 --- a/libiop/tests/bcs/test_bcs_transformation.cpp +++ b/libiop/tests/bcs/test_bcs_transformation.cpp @@ -44,7 +44,7 @@ void set_bcs_parameters_leafhash(bcs_transformation_parameters>(security_parameter); params.compression_hasher = blake2b_two_to_one_hash; - params.cap_hasher = blake2b_vector_hash; + params.cap_hasher = blake2b_many_to_one_hash; } // Algebraic case diff --git a/libiop/tests/bcs/test_merkle_tree.cpp b/libiop/tests/bcs/test_merkle_tree.cpp index fc6ba829..0efbf6da 100644 --- a/libiop/tests/bcs/test_merkle_tree.cpp +++ b/libiop/tests/bcs/test_merkle_tree.cpp @@ -12,29 +12,34 @@ namespace libiop { +using std::size_t; + template< bool B, class T = void > using enable_if_t = typename libff::enable_if::type; + // Binary hash type template::value, int> = 42> merkle_tree new_MT( - const std::size_t size, const std::size_t digest_len_bytes, const bool make_zk, - const std::size_t security_parameter) + const size_t size, const size_t digest_len_bytes, const bool make_zk, + const size_t security_parameter, const size_t cap_size=2) { return merkle_tree( size, std::make_shared>(security_parameter), blake2b_two_to_one_hash, - blake2b_vector_hash, + blake2b_many_to_one_hash, digest_len_bytes, make_zk, - security_parameter); + security_parameter, + cap_size); } + template::value, int> = 42> merkle_tree new_MT( - const std::size_t size, const std::size_t digest_len_bytes, const bool make_zk, - const std::size_t security_parameter) + const size_t size, const size_t digest_len_bytes, const bool make_zk, + const size_t security_parameter, const size_t cap_size=2) { return merkle_tree( size, @@ -43,14 +48,18 @@ merkle_tree new_MT( dummy_algebraic_cap_hash, digest_len_bytes, make_zk, - security_parameter); + security_parameter, + cap_size); } +/** Constructs a merkle tree with leaf size 2. Generates and verifies membership proofs for + * each leaf individually, and makes sure reversing the contents of each leaf causes the + * verification to fail (unless the leaf contents are symmetric). */ template -void run_simple_MT_test(const std::size_t size, const std::size_t digest_len_bytes, const bool make_zk, - const std::size_t security_parameter) { +void run_simple_MT_test(const size_t size, const size_t digest_len_bytes, const bool make_zk, + const size_t security_parameter, const size_t cap_size) { merkle_tree tree = - new_MT(size, digest_len_bytes, make_zk, security_parameter); + new_MT(size, digest_len_bytes, make_zk, security_parameter, cap_size); const std::vector vec1 = random_vector(size); const std::vector vec2 = random_vector(size); @@ -58,7 +67,7 @@ void run_simple_MT_test(const std::size_t size, const std::size_t digest_len_byt const hash_type root = tree.get_root(); - for (std::size_t i = 0; i < size; ++i) + for (size_t i = 0; i < size; ++i) { /* membership proof for the set {i} */ const std::vector set = {i}; @@ -98,30 +107,37 @@ void run_simple_MT_test(const std::size_t size, const std::size_t digest_len_byt TEST(MerkleTreeTest, SimpleTest) { typedef libff::gf64 FieldT; - const std::size_t size = 16; - const std::size_t digest_len_bytes = 256/8; - const std::size_t security_parameter = 128; - run_simple_MT_test(size, digest_len_bytes, false, security_parameter); - run_simple_MT_test(size, digest_len_bytes, false, security_parameter); + const size_t size = 16; + const size_t cap_size = 2; + const size_t digest_len_bytes = 256/8; + const size_t security_parameter = 128; + + run_simple_MT_test(size, digest_len_bytes, false, + security_parameter, cap_size); + run_simple_MT_test(size, digest_len_bytes, false, + security_parameter, cap_size); } TEST(MerkleTreeZKTest, SimpleTest) { typedef libff::gf64 FieldT; - const std::size_t size_small = 16; - const std::size_t size_large = 1ull << 18; /* The goal is to test batch randomness logic */ - const std::size_t digest_len_bytes = 256/8; - const std::size_t security_parameter = 128; - run_simple_MT_test(size_small, digest_len_bytes, true, security_parameter); - run_simple_MT_test(size_large, digest_len_bytes, true, security_parameter); + const size_t size_small = 16; + const size_t size_large = 1ull << 18; /* The goal is to test batch randomness logic */ + const size_t cap_size = 4; + const size_t digest_len_bytes = 256/8; + const size_t security_parameter = 128; + run_simple_MT_test(size_small, digest_len_bytes, true, + security_parameter, cap_size); + run_simple_MT_test(size_large, digest_len_bytes, true, + security_parameter, cap_size); } void run_multi_test(const bool make_zk) { typedef libff::gf64 FieldT; - const std::size_t size = 8; - const std::size_t security_parameter = 128; - const std::size_t digest_len_bytes = 256/8; + const size_t size = 8; + const size_t security_parameter = 128; + const size_t digest_len_bytes = 256/8; const bool algebraic_hash = false; merkle_tree tree = new_MT( @@ -138,17 +154,17 @@ void run_multi_test(const bool make_zk) { const binary_hash_digest root = tree.get_root(); std::vector> leafs; - for (std::size_t i = 0; i < size; ++i) + for (size_t i = 0; i < size; ++i) { std::vector leaf({ vec1[i], vec2[i] }); leafs.emplace_back(leaf); } - for (std::size_t subset = 0; subset < (1ull< subset_elements; + std::vector subset_elements; std::vector> subset_leafs; - for (std::size_t k = 0; k < size; ++k) + for (size_t k = 0; k < size; ++k) { if (subset & (1ull< mp = tree.get_set_membership_proof(subset_elements); + const merkle_tree_set_membership_proof mp = + tree.get_set_membership_proof(subset_elements); const bool is_valid = tree.validate_set_membership_proof(root, subset_elements, @@ -177,7 +194,7 @@ TEST(MerkleTreeZKTest, MultiTest) { run_multi_test(make_zk); } -TEST(MerkleTreeTwoToOneHashTest, SimpleTest) +TEST(MerkleTreeHashCountTest, SimpleTest) { typedef libff::gf64 FieldT; bool make_zk = false; From 87828ca8e3d810a33ffadbf379f80481479ba62f Mon Sep 17 00:00:00 2001 From: Alexander Wu Date: Sat, 8 May 2021 01:42:34 -0700 Subject: [PATCH 3/8] Add more tests for merkle trees and add sorting to validate_set_membership_proof --- libiop/algebra/utils.hpp | 3 + libiop/algebra/utils.tcc | 6 ++ libiop/bcs/merkle_tree.hpp | 2 + libiop/bcs/merkle_tree.tcc | 6 +- libiop/tests/bcs/test_merkle_tree.cpp | 133 ++++++++++++++++++++++---- 5 files changed, 133 insertions(+), 17 deletions(-) diff --git a/libiop/algebra/utils.hpp b/libiop/algebra/utils.hpp index f25ed7ab..9f41b023 100644 --- a/libiop/algebra/utils.hpp +++ b/libiop/algebra/utils.hpp @@ -19,6 +19,9 @@ void bitreverse_vector(std::vector &a); template std::vector random_vector(const std::size_t count); +template +bool compare_first(const std::pair &a, const std::pair &b); + template std::vector all_subset_sums(const std::vector &basis, const T& shift = 0) #if defined(__clang__) diff --git a/libiop/algebra/utils.tcc b/libiop/algebra/utils.tcc index e1adde2c..8ed6778b 100644 --- a/libiop/algebra/utils.tcc +++ b/libiop/algebra/utils.tcc @@ -168,6 +168,12 @@ std::vector random_vector(const std::size_t count) return result; } +template +bool compare_first(const std::pair &a, const std::pair &b) +{ + return a.first < b.first; +} + template std::vector random_FieldT_vector(const std::size_t count) { diff --git a/libiop/bcs/merkle_tree.hpp b/libiop/bcs/merkle_tree.hpp index 8e390559..7dcf54e0 100644 --- a/libiop/bcs/merkle_tree.hpp +++ b/libiop/bcs/merkle_tree.hpp @@ -126,6 +126,8 @@ class merkle_tree { hash_digest_type get_root() const; + /* These two functions do not currently work if the given positions aren't sorted or + have duplicates, AND the tree is set to be zero knowledge. */ merkle_tree_set_membership_proof get_set_membership_proof( const std::vector &positions) const; bool validate_set_membership_proof( diff --git a/libiop/bcs/merkle_tree.tcc b/libiop/bcs/merkle_tree.tcc index 0c155b77..9408b7dc 100644 --- a/libiop/bcs/merkle_tree.tcc +++ b/libiop/bcs/merkle_tree.tcc @@ -407,6 +407,9 @@ bool merkle_tree::validate_set_membership_proof( if (this->make_zk_) { for (auto &leaf : leaf_contents) { + /* FIXME: This code is currently incorrect if the given list of positions is not + sorted or has duplicates. This could be fixed if both positions and leaf_contents + are sorted before the leaf hashes are calculated, which would require refactoring. */ const zk_salt_type zk_salt = *rand_it++; leaf_hashes.emplace_back(this->leaf_hasher_->zk_hash(leaf, zk_salt)); } @@ -427,6 +430,7 @@ bool merkle_tree::validate_set_membership_proof( return std::make_pair(pos, hash); }); + std::sort(S.begin(), S.end(), compare_first); S.erase(std__unique(S.begin(), S.end()), S.end()); /* remove possible duplicates */ if (std__adjacent_find(S.begin(), S.end(), @@ -446,7 +450,7 @@ bool merkle_tree::validate_set_membership_proof( throw std::invalid_argument("All positions must be between 0 and num_leaves-1."); } - /* transform to set of indices */ + /* transform to sorted set of indices */ for (auto &pos : S) { pos.first += this->num_leaves_ - 1; diff --git a/libiop/tests/bcs/test_merkle_tree.cpp b/libiop/tests/bcs/test_merkle_tree.cpp index 0efbf6da..8c731341 100644 --- a/libiop/tests/bcs/test_merkle_tree.cpp +++ b/libiop/tests/bcs/test_merkle_tree.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "libiop/algebra/utils.hpp" @@ -108,14 +109,17 @@ TEST(MerkleTreeTest, SimpleTest) { typedef libff::gf64 FieldT; const size_t size = 16; - const size_t cap_size = 2; + const std::vector cap_sizes = {2, 4, 8, 16}; // Test all possible cap sizes. const size_t digest_len_bytes = 256/8; const size_t security_parameter = 128; - run_simple_MT_test(size, digest_len_bytes, false, - security_parameter, cap_size); - run_simple_MT_test(size, digest_len_bytes, false, - security_parameter, cap_size); + for (size_t cap_size : cap_sizes) + { + run_simple_MT_test(size, digest_len_bytes, false, + security_parameter, cap_size); + run_simple_MT_test(size, digest_len_bytes, false, + security_parameter, cap_size); + } } TEST(MerkleTreeZKTest, SimpleTest) { @@ -132,10 +136,14 @@ TEST(MerkleTreeZKTest, SimpleTest) { security_parameter, cap_size); } -void run_multi_test(const bool make_zk) { +/** Constructs a merkle tree with 8 leaves each of size 2, and cap size 4. Generates and verifies + * membership proofs for every possible subset of leaves. */ +void run_fixed_multi_test(const bool make_zk) { typedef libff::gf64 FieldT; + // The size is fixed because large values would quickly cause the program run out of memory. const size_t size = 8; + const size_t cap_size = 4; const size_t security_parameter = 128; const size_t digest_len_bytes = 256/8; const bool algebraic_hash = false; @@ -144,7 +152,8 @@ void run_multi_test(const bool make_zk) { size, digest_len_bytes, make_zk, - security_parameter); + security_parameter, + cap_size); const std::vector vec1 = random_vector(size); const std::vector vec2 = random_vector(size); @@ -153,23 +162,25 @@ void run_multi_test(const bool make_zk) { const binary_hash_digest root = tree.get_root(); - std::vector> leafs; + std::vector> leaves; for (size_t i = 0; i < size; ++i) { std::vector leaf({ vec1[i], vec2[i] }); - leafs.emplace_back(leaf); + leaves.emplace_back(leaf); } + /* This code generates every possible subset. `subset` is a binary string that encodes for each + element, whether it is in this subset. */ for (size_t subset = 0; subset < (1ull< subset_elements; - std::vector> subset_leafs; + std::vector> subset_leaves; for (size_t k = 0; k < size; ++k) { if (subset & (1ull< tree = new_MT( + size, + digest_len_bytes, + make_zk, + security_parameter, + cap_size); + + const std::vector vec1 = random_vector(size); + const std::vector vec2 = random_vector(size); + + tree.construct({ vec1, vec2 }); + + const binary_hash_digest root = tree.get_root(); + + std::vector> leaves; + leaves.reserve(size); + std::vector shuffled_leaf_indices; + shuffled_leaf_indices.reserve(size); + for (size_t i = 0; i < size; ++i) + { + std::vector leaf({ vec1[i], vec2[i] }); + leaves.emplace_back(leaf); + shuffled_leaf_indices.emplace_back(i); + } + + for (size_t i = 0; i < num_iterations; i++) + { + std::vector subset_elements; + std::vector> subset_leaves; + /* The commented-out code generates subsets that are unsorted and may be repeats. + They are not used because the code currently cannot handle these cases if it is + zero knowledge. */ + // for (size_t j = 0; j < subset_size; j++) + // { + // size_t k = randombytes_uniform(size); + // subset_elements.emplace_back(k); + // subset_leaves.emplace_back(leaves[k]); + // } + + // Generate a random sorted subset of indices at the beginning of shuffled_leaf_indices. + std::shuffle(shuffled_leaf_indices.begin(), shuffled_leaf_indices.end(), + std::default_random_engine(i)); + std::sort(shuffled_leaf_indices.begin(), shuffled_leaf_indices.begin() + subset_size); + for (size_t j = 0; j < subset_size; j++) + { + size_t k = shuffled_leaf_indices[j]; + subset_elements.emplace_back(k); + subset_leaves.emplace_back(leaves[k]); + } + + const merkle_tree_set_membership_proof mp = + tree.get_set_membership_proof(subset_elements); + + const bool is_valid = tree.validate_set_membership_proof(root, + subset_elements, + subset_leaves, mp); EXPECT_TRUE(is_valid); } } -TEST(MerkleTreeTest, MultiTest) { +TEST(MerkleTreeTest, RandomMultiTest) { + const size_t security_parameter = 128; + const size_t digest_len_bytes = 256/8; const bool make_zk = false; - run_multi_test(make_zk); + // Test a small and a large tree. + run_random_multi_test(16, digest_len_bytes, make_zk, security_parameter, 4, 5); + run_random_multi_test(1ull << 16, digest_len_bytes, make_zk, security_parameter, 256, 100); } -TEST(MerkleTreeZKTest, MultiTest) { +TEST(MerkleTreeZKTest, RandomMultiTest) { + const size_t security_parameter = 128; + const size_t digest_len_bytes = 256/8; const bool make_zk = true; - run_multi_test(make_zk); + // Test a small and a large tree. + run_random_multi_test(16, digest_len_bytes, make_zk, security_parameter, 4, 5); + run_random_multi_test(1ull << 16, digest_len_bytes, make_zk, security_parameter, 256, 100); } TEST(MerkleTreeHashCountTest, SimpleTest) From 62cff9494ce73ad50b7ab9a37731747d75834281 Mon Sep 17 00:00:00 2001 From: Alexander Wu Date: Sat, 8 May 2021 18:45:09 -0700 Subject: [PATCH 4/8] Update method that counts the number of hashes in merkle tree verification --- libiop/bcs/bcs_common.tcc | 22 +++++++++++---------- libiop/bcs/merkle_tree.hpp | 6 ++++-- libiop/bcs/merkle_tree.tcc | 13 +++++++------ libiop/tests/bcs/test_merkle_tree.cpp | 28 ++++++++++++++++++--------- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/libiop/bcs/bcs_common.tcc b/libiop/bcs/bcs_common.tcc index 0bcd87ed..74bb7b77 100644 --- a/libiop/bcs/bcs_common.tcc +++ b/libiop/bcs/bcs_common.tcc @@ -711,7 +711,7 @@ void print_detailed_transcript_data( const size_t digest_len_bytes = 2 * (params.security_parameter / 8); const size_t field_size = (libff::log_of_field_size_helper(FieldT::zero()) + 7) / 8; - std::vector two_to_one_hashes_by_round; + std::vector internal_hash_complexity_by_round; std::vector leaf_hashes_by_round; std::vector zk_hashes_by_round; std::vector IOP_size_by_round; @@ -743,10 +743,9 @@ void print_detailed_transcript_data( query_positions.emplace_back(MT_position); } } - size_t num_two_to_one_hashes_in_round = - MT.count_hashes_to_verify_set_membership_proof( - query_positions); - two_to_one_hashes_by_round.emplace_back(num_two_to_one_hashes_in_round); + size_t internal_hash_complexity_in_round = + MT.count_internal_hash_complexity_to_verify_set_membership(query_positions); + internal_hash_complexity_by_round.emplace_back(internal_hash_complexity_in_round); const size_t num_values_per_leaf = transcript.query_responses_[round][0].size(); const size_t num_leaves = transcript.query_responses_[round].size(); leaf_hashes_by_round.emplace_back(num_values_per_leaf * num_leaves); @@ -790,14 +789,17 @@ void print_detailed_transcript_data( printf("\n"); printf("total prover messages size: %lu\n", total_prover_message_size); - const size_t total_two_to_one_hashes = std::accumulate( - two_to_one_hashes_by_round.begin(), two_to_one_hashes_by_round.end(), 0); + const size_t total_internal_hash_complexity = std::accumulate( + internal_hash_complexity_by_round.begin(), internal_hash_complexity_by_round.end(), 0); const size_t total_leaves_hashed = std::accumulate( leaf_hashes_by_round.begin(), leaf_hashes_by_round.end(), 0); const size_t total_zk_hashes = std::accumulate( zk_hashes_by_round.begin(), zk_hashes_by_round.end(), 0); - const size_t total_hashes = total_two_to_one_hashes + total_leaves_hashed + total_zk_hashes; - printf("total two to one hashes: %lu\n", total_two_to_one_hashes); + /* Since each two-to-one hash is counted as two units, we divide by 2 here to make it consistent. + It would be nice to take into account how many leaves are hashed, but we are unfortunately + not provided this information. */ + const size_t total_hashes = total_internal_hash_complexity / 2 + total_leaves_hashed + total_zk_hashes; + printf("total internal hash complexity: %lu\n", total_internal_hash_complexity); printf("total leaves hashed: %lu\n", total_leaves_hashed); printf("total hashes: %lu\n", total_hashes); printf("\n"); @@ -810,7 +812,7 @@ void print_detailed_transcript_data( printf("MT_depth %lu\n", MT_depths[round]); printf("IOP size: %lu bytes\n", IOP_size_by_round[round]); printf("BCS size: %lu bytes\n", BCS_size_by_round[round]); - printf("number of two to one hashes: %lu\n", two_to_one_hashes_by_round[round]); + printf("internal hash complexity: %lu\n", internal_hash_complexity_by_round[round]); printf("number of leaves hashed: %lu\n", leaf_hashes_by_round[round]); if (make_zk[round]) { diff --git a/libiop/bcs/merkle_tree.hpp b/libiop/bcs/merkle_tree.hpp index 7dcf54e0..24470fe1 100644 --- a/libiop/bcs/merkle_tree.hpp +++ b/libiop/bcs/merkle_tree.hpp @@ -136,8 +136,10 @@ class merkle_tree { const std::vector> &leaf_contents, const merkle_tree_set_membership_proof &proof); - /* Returns number of two to one hashes */ - size_t count_hashes_to_verify_set_membership_proof( + /** Returns a number that is proportional to the hashing runtime of verifying a set membership + * proof. Each two-to-one hash is counted as 2 units, and each input of the cap hash is 1 unit. + * Leaf hashes are not counted. */ + size_t count_internal_hash_complexity_to_verify_set_membership( const std::vector &positions) const; std::size_t num_leaves() const; diff --git a/libiop/bcs/merkle_tree.tcc b/libiop/bcs/merkle_tree.tcc index 9408b7dc..5d26d5a3 100644 --- a/libiop/bcs/merkle_tree.tcc +++ b/libiop/bcs/merkle_tree.tcc @@ -548,18 +548,20 @@ bool merkle_tree::validate_set_membership_proof( } template -size_t merkle_tree::count_hashes_to_verify_set_membership_proof( +size_t merkle_tree::count_internal_hash_complexity_to_verify_set_membership( const std::vector &positions) const { /** This goes layer by layer, * and counts the number of hashes needed to be computed. * Essentially when moving up a layer in the verifier, * every unique parent is one hash that has to be computed. */ - size_t num_two_to_one_hashes = 0; + size_t num_two_to_one_hashes = 0; std::vector cur_pos_set = positions; sort(cur_pos_set.begin(), cur_pos_set.end()); assert(cur_pos_set[cur_pos_set.size() - 1] < this->num_leaves()); - for (size_t cur_depth = this->depth(); cur_depth > 0; cur_depth--) + + const size_t cap_depth = libff::log2(this->cap_size_); + for (size_t cur_depth = this->depth(); cur_depth > cap_depth; cur_depth--) { // contains positions in range [0, 2^{cur_depth - 1}) std::vector next_pos_set; @@ -567,8 +569,7 @@ size_t merkle_tree::count_hashes_to_verify_set_members { size_t parent_pos = cur_pos_set[i] / 2; // Check that parent pos isn't already in next pos set - if (next_pos_set.size() == 0 - || next_pos_set[next_pos_set.size() - 1] != parent_pos) + if (next_pos_set.size() == 0 || next_pos_set[next_pos_set.size() - 1] != parent_pos) { next_pos_set.emplace_back(parent_pos); } @@ -576,7 +577,7 @@ size_t merkle_tree::count_hashes_to_verify_set_members num_two_to_one_hashes += next_pos_set.size(); cur_pos_set = next_pos_set; } - return num_two_to_one_hashes; + return 2 * num_two_to_one_hashes + this->cap_size_; } template diff --git a/libiop/tests/bcs/test_merkle_tree.cpp b/libiop/tests/bcs/test_merkle_tree.cpp index 8c731341..9f97db1b 100644 --- a/libiop/tests/bcs/test_merkle_tree.cpp +++ b/libiop/tests/bcs/test_merkle_tree.cpp @@ -295,6 +295,8 @@ TEST(MerkleTreeZKTest, RandomMultiTest) { run_random_multi_test(1ull << 16, digest_len_bytes, make_zk, security_parameter, 256, 100); } +/** Verify that count_internal_hash_complexity_to_verify_set_membership is correct for a fixed tree + * size and query set, for various cap sizes. */ TEST(MerkleTreeHashCountTest, SimpleTest) { typedef libff::gf64 FieldT; @@ -304,16 +306,24 @@ TEST(MerkleTreeHashCountTest, SimpleTest) const size_t hash_length = 32; const bool algebraic_hash = false; - merkle_tree tree = new_MT( - num_leaves, - hash_length, - make_zk, - security_parameter); + const std::vector cap_sizes = {2, 4, 8}; + const std::vector expected_num_hashes = {12, 10, 8}; + + const std::vector positions = {1, 3, 6, 7}; - std::vector positions = {1, 3, 6, 7}; - size_t expected_num_hashes = 6; - size_t actual_num_hashes = tree.count_hashes_to_verify_set_membership_proof(positions); - ASSERT_EQ(expected_num_hashes, actual_num_hashes); + for (size_t i = 0; i < cap_sizes.size(); i++) + { + merkle_tree tree = new_MT( + num_leaves, + hash_length, + make_zk, + security_parameter, + cap_sizes[i]); + + size_t actual_num_hashes = tree.count_internal_hash_complexity_to_verify_set_membership( + positions); + ASSERT_EQ(expected_num_hashes[i], actual_num_hashes); + } } } From 62245b9e6965aad06010be65c5ba76bb927fe3bc Mon Sep 17 00:00:00 2001 From: Alexander Wu Date: Sat, 8 May 2021 19:50:31 -0700 Subject: [PATCH 5/8] Add algebraic cap hash --- libiop/bcs/hashing/algebraic_sponge.hpp | 8 +++-- libiop/bcs/hashing/algebraic_sponge.tcc | 6 ++-- libiop/bcs/hashing/hash_enum.tcc | 45 ++++++++++++------------- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/libiop/bcs/hashing/algebraic_sponge.hpp b/libiop/bcs/hashing/algebraic_sponge.hpp index 5f63408c..89ebc322 100644 --- a/libiop/bcs/hashing/algebraic_sponge.hpp +++ b/libiop/bcs/hashing/algebraic_sponge.hpp @@ -99,14 +99,15 @@ class algebraic_hashchain : public hashchain void absorb_internal(const typename libff::enable_if::value, MT_root_type>::type new_input); }; +/** The algebraic_vector_hash is used for both the algebraic leaf hash and cap hash. */ template -class algebraic_leafhash : public leafhash +class algebraic_vector_hash : public leafhash { protected: std::shared_ptr> sponge_; public: - algebraic_leafhash( + algebraic_vector_hash( std::shared_ptr> sponge, size_t security_parameter); FieldT hash(const std::vector &leaf); @@ -115,6 +116,9 @@ class algebraic_leafhash : public leafhash const zk_salt_type &zk_salt); }; +template +using algebraic_leafhash = algebraic_vector_hash; + template class algebraic_two_to_one_hash { diff --git a/libiop/bcs/hashing/algebraic_sponge.tcc b/libiop/bcs/hashing/algebraic_sponge.tcc index 0733f42b..9d2bb4dd 100644 --- a/libiop/bcs/hashing/algebraic_sponge.tcc +++ b/libiop/bcs/hashing/algebraic_sponge.tcc @@ -206,7 +206,7 @@ MT_root_type algebraic_hashchain::squeeze_root_type() } template -algebraic_leafhash::algebraic_leafhash( +algebraic_vector_hash::algebraic_vector_hash( std::shared_ptr> sponge, size_t security_parameter) : sponge_(sponge->new_sponge()) @@ -218,7 +218,7 @@ algebraic_leafhash::algebraic_leafhash( } template -FieldT algebraic_leafhash::hash( +FieldT algebraic_vector_hash::hash( const std::vector &leaf) { this->sponge_->absorb(leaf); @@ -228,7 +228,7 @@ FieldT algebraic_leafhash::hash( } template -FieldT algebraic_leafhash::zk_hash( +FieldT algebraic_vector_hash::zk_hash( const std::vector &leaf, const zk_salt_type &zk_salt) { diff --git a/libiop/bcs/hashing/hash_enum.tcc b/libiop/bcs/hashing/hash_enum.tcc index 0de8b720..50585707 100644 --- a/libiop/bcs/hashing/hash_enum.tcc +++ b/libiop/bcs/hashing/hash_enum.tcc @@ -100,9 +100,8 @@ std::shared_ptr> get_leafhash_internal( poseidon_params params = get_poseidon_parameters(hash_enum); /* security parameter is -1 b/c */ std::shared_ptr> permutation = std::make_shared>(params); - std::shared_ptr> leafhasher = std::make_shared>( - permutation, - security_parameter - 1); + std::shared_ptr> leafhasher = + std::make_shared>(permutation, security_parameter - 1); return leafhasher; } throw std::invalid_argument("bcs_hash_type unknown (algebraic leaf hash)"); @@ -149,7 +148,7 @@ two_to_one_hash_function get_two_to_one_hash_internal( as this reference has to live after the function terminates */ std::shared_ptr> hash_class = std::make_shared>(permutation, security_parameter - 1); - std::function f = [permutation, hash_class](const FieldT& left, const FieldT& right, const std::size_t unused) -> FieldT + std::function f = [permutation, hash_class](const FieldT& left, const FieldT& right, const std::size_t unused) -> FieldT { return hash_class->hash(left, right); }; @@ -185,25 +184,25 @@ cap_hash_function get_cap_hash_internal( const bcs_hash_type hash_enum, const size_t security_parameter) { - // if (hash_enum == starkware_poseidon_type || hash_enum == high_alpha_poseidon_type) - // { - // if (security_parameter != 128) - // { - // throw std::invalid_argument("Poseidon only supported for 128 bit soundness."); - // } - // poseidon_params params = get_poseidon_parameters(hash_enum); - // /* security parameter is -1 b/c */ - // std::shared_ptr> permutation = std::make_shared>(params); - // /* We explicitly place this on heap with no destructor, - // as this reference has to live after the function terminates */ - // std::shared_ptr> hash_class = - // std::make_shared>(permutation, security_parameter - 1); - // std::function f = [permutation, hash_class](const FieldT& left, const FieldT& right, const std::size_t unused) -> FieldT - // { - // return hash_class->hash(left, right); - // }; - // return f; - // } + if (hash_enum == starkware_poseidon_type || hash_enum == high_alpha_poseidon_type) + { + if (security_parameter != 128) + { + throw std::invalid_argument("Poseidon only supported for 128 bit soundness."); + } + poseidon_params params = get_poseidon_parameters(hash_enum); + /* security parameter is -1 b/c */ + std::shared_ptr> permutation = std::make_shared>(params); + /* We explicitly place this on heap with no destructor, + as this reference has to live after the function terminates */ + std::shared_ptr> hash_class = + std::make_shared>(permutation, security_parameter - 1); + std::function &leaf, const std::size_t)> f = [permutation, hash_class](const std::vector &leaf, const std::size_t unused) -> FieldT + { + return hash_class->hash(leaf); + }; + return f; + } throw std::invalid_argument("bcs_hash_type unknown (algebraic cap hash)"); } From 12259be95cdb3c2b031e6a47b813354f6871b83c Mon Sep 17 00:00:00 2001 From: Alexander Wu Date: Wed, 12 May 2021 21:14:05 -0700 Subject: [PATCH 6/8] Add cap size to BCS parameters --- libiop/bcs/bcs_common.hpp | 1 + libiop/bcs/bcs_common.tcc | 3 ++- libiop/bcs/common_bcs_parameters.tcc | 1 + libiop/tests/bcs/test_bcs_transformation.cpp | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/libiop/bcs/bcs_common.hpp b/libiop/bcs/bcs_common.hpp index 3c690552..8f4a7f59 100644 --- a/libiop/bcs/bcs_common.hpp +++ b/libiop/bcs/bcs_common.hpp @@ -25,6 +25,7 @@ template struct bcs_transformation_parameters { std::size_t security_parameter; /* TODO: possibly revisit in the future */ bcs_hash_type hash_enum; + std::size_t cap_size; pow_parameters pow_params_; diff --git a/libiop/bcs/bcs_common.tcc b/libiop/bcs/bcs_common.tcc index 74bb7b77..b4409a0b 100644 --- a/libiop/bcs/bcs_common.tcc +++ b/libiop/bcs/bcs_common.tcc @@ -474,7 +474,8 @@ void bcs_protocol::seal_interaction_registrations() this->parameters_.cap_hasher, this->digest_len_bytes_, make_zk, - this->parameters_.security_parameter); + this->parameters_.security_parameter, + this->parameters_.cap_size); this->Merkle_trees_.emplace_back(MT); } } diff --git a/libiop/bcs/common_bcs_parameters.tcc b/libiop/bcs/common_bcs_parameters.tcc index 27487055..bef04c76 100644 --- a/libiop/bcs/common_bcs_parameters.tcc +++ b/libiop/bcs/common_bcs_parameters.tcc @@ -15,6 +15,7 @@ bcs_transformation_parameters default_bcs_params( params.hash_enum = hash_type; /* TODO: Push setting leaf hash into internal BCS code. Currently 2 is fine, as leaf size is internally unused. */ const size_t leaf_size = 2; + params.cap_size = 2; params.leafhasher_ = get_leafhash(hash_type, security_parameter, leaf_size); params.compression_hasher = get_two_to_one_hash(hash_type, security_parameter); params.cap_hasher = get_cap_hash(hash_type, security_parameter); diff --git a/libiop/tests/bcs/test_bcs_transformation.cpp b/libiop/tests/bcs/test_bcs_transformation.cpp index 688cfa88..f4320b87 100644 --- a/libiop/tests/bcs/test_bcs_transformation.cpp +++ b/libiop/tests/bcs/test_bcs_transformation.cpp @@ -72,6 +72,7 @@ bcs_transformation_parameters get_bcs_parameters(bool alge bcs_parameters.hash_enum = bcs_hash_type::blake2b_type; } set_bcs_parameters_leafhash(bcs_parameters); + bcs_parameters.cap_size = 2; return bcs_parameters; } From 5b276ee9607991a404b78655777f9eae572b499a Mon Sep 17 00:00:00 2001 From: Alexander Wu Date: Wed, 12 May 2021 22:59:18 -0700 Subject: [PATCH 7/8] Make hash enum function errors more descriptive --- libiop/bcs/hashing/hash_enum.tcc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libiop/bcs/hashing/hash_enum.tcc b/libiop/bcs/hashing/hash_enum.tcc index 50585707..490adaef 100644 --- a/libiop/bcs/hashing/hash_enum.tcc +++ b/libiop/bcs/hashing/hash_enum.tcc @@ -58,7 +58,7 @@ std::shared_ptr> get_hashchain_internal( { return std::make_shared>(security_parameter); } - throw std::invalid_argument("bcs_hash_type unknown"); + throw std::invalid_argument("bcs_hash_type unknown (blake2b hashchain)"); } @@ -80,7 +80,7 @@ std::shared_ptr> get_leafhash_internal( { return std::make_shared>(security_parameter); } - throw std::invalid_argument("bcs_hash_type unknown"); + throw std::invalid_argument("bcs_hash_type unknown (blake2b leaf hash)"); } /* Algebraic leafhash case */ @@ -125,7 +125,7 @@ two_to_one_hash_function get_two_to_one_hash_internal( { return blake2b_two_to_one_hash; } - throw std::invalid_argument("bcs_hash_type unknown"); + throw std::invalid_argument("bcs_hash_type unknown (blake2b two to one hash)"); } /* algebraic 2->1 hash */ @@ -174,7 +174,7 @@ cap_hash_function get_cap_hash_internal( { return blake2b_many_to_one_hash; } - throw std::invalid_argument("bcs_hash_type unknown"); + throw std::invalid_argument("bcs_hash_type unknown (blake2b cap hash)"); } /* Algebraic 2^n->1 hash. */ From e9670120142590c82182100786c978f784172065 Mon Sep 17 00:00:00 2001 From: Alexander Wu Date: Sun, 25 Sep 2022 13:34:11 -0700 Subject: [PATCH 8/8] Fix some comments in merkle_tree, pow, blake2b, and test_merkle_tree --- libiop/bcs/hashing/blake2b.cpp | 2 +- libiop/bcs/merkle_tree.hpp | 83 ++++++++++++++++----------- libiop/bcs/merkle_tree.tcc | 15 +++-- libiop/bcs/pow.tcc | 5 -- libiop/tests/bcs/test_merkle_tree.cpp | 2 +- 5 files changed, 60 insertions(+), 47 deletions(-) diff --git a/libiop/bcs/hashing/blake2b.cpp b/libiop/bcs/hashing/blake2b.cpp index 2006e2b2..5e4d61cd 100644 --- a/libiop/bcs/hashing/blake2b.cpp +++ b/libiop/bcs/hashing/blake2b.cpp @@ -65,7 +65,7 @@ binary_hash_digest blake2b_many_to_one_hash(const std::vector struct merkle_tree_set_membership_proof { std::vector auxiliary_hashes; @@ -46,9 +46,12 @@ template class merkle_tree { protected: bool constructed_; - /* inner_nodes_ is a vector of the (num_leaves - 1) nodes in the tree, with the root at - index 0, left child at 1, right child at 2, etc. If cap_size_ is greater than 2, the - first (log_2(cap_size_) - 1) layers under the root are empty to make the math easier. */ + /** + * inner_nodes_ is a vector of the `2 * num_leaves_ - 1` nodes in the tree, with the root + * at index 0, left child at 1, right child at 2, etc. If cap_size_ is greater than 2, the + * first `log_2(cap_size_) - 1` layers under (and not including) the root are empty to make + * the math easier. + */ std::vector inner_nodes_; std::size_t num_leaves_; @@ -57,17 +60,21 @@ class merkle_tree { std::size_t digest_len_bytes_; bool make_zk_; std::size_t num_zk_bytes_; - /* The top log_2(cap_size_) layers are hashed with a single computation to improve efficiency. - The root along with its cap_size_ direct children are referred to as the "cap," and the - operation that transforms these children to the root is the cap hash. - See https://github.com/scipr-lab/libiop/issues/41. */ + /** + * The top `log_2(cap_size_)` layers are hashed with a single computation to improve efficiency. + * The root along with its cap_size_ direct children are referred to as the "cap," and the + * operation that transforms these children to the root is the cap hash. + * See https://github.com/scipr-lab/libiop/issues/41. + */ cap_hash_function cap_hasher_; - /* cap_size_ is the number of direct children the root has. It must be a power of 2 and at - least 2. For example if cap_size == 4, the root has 4 children, and in inner_nodes_ the - indices 1 and 2 are unused. */ + /** + * cap_size_ is the number of direct children the root has. It must be a power of 2 and at + * least 2. For example if cap_size == 4, the root has 4 children, and in inner_nodes_ the + * indices 1 and 2 are unused. + */ std::size_t cap_size_; - /* Each element will be hashed (individually) to produce a random hash digest. */ + /** Each element will be hashed (individually) to produce a random hash digest. */ std::vector zk_leaf_randomness_elements_; void sample_leaf_randomness(); void compute_inner_nodes(); @@ -81,10 +88,12 @@ class merkle_tree { std::size_t cap_children_start() const; // Inclusive. std::size_t cap_children_end() const; // Exclusive. public: - /* Create a merkle tree with the given configuration. - If make_zk is true, 2 * security parameter random bytes will be appended to each leaf - before hashing, to prevent a low entropy leaf value from being inferred from its hash. - cap_size is the number of children of the root and must be a power of 2. */ + /** + * Create a merkle tree with the given configuration. + * If make_zk is true, 2 * security parameter random bytes will be appended to each leaf + * before hashing, to prevent a low entropy leaf value from being inferred from its hash. + * cap_size is the number of children of the root and must be a power of 2. + */ merkle_tree(const std::size_t num_leaves, const std::shared_ptr> &leaf_hasher, const two_to_one_hash_function &node_hasher, @@ -94,31 +103,35 @@ class merkle_tree { const std::size_t security_parameter, const std::size_t cap_size=2); - /** This treats each leaf as a column. - * e.g. The ith leaf is the vector formed by leaf_contents[j][i] for all j */ + /** + * This treats each leaf as a column. + * e.g. The ith leaf is the vector formed by leaf_contents[j][i] for all j. + */ void construct(const std::vector>> &leaf_contents); // TODO: Remove this overload in favor of only using the former void construct(const std::vector > &leaf_contents); - /** Leaf contents is a table with `r` rows - * (`r` typically being the number of oracles) - * and (MT_num_leaves * coset_serialization_size) columns. - * Each MT leaf is the serialization of a table with `r` rows, - * and coset_serialization_size columns. + /** + * Leaf contents is a table with `r` rows + * (`r` typically being the number of oracles) + * and (MT_num_leaves * coset_serialization_size) columns. + * Each MT leaf is the serialization of a table with `r` rows, + * and coset_serialization_size columns. * - * This is done here rather than the BCS layer to avoid needing to copy the data, - * as this will take a significant amount of memory. + * This is done here rather than the BCS layer to avoid needing to copy the data, + * as this will take a significant amount of memory. */ void construct_with_leaves_serialized_by_cosets( const std::vector>> &leaf_contents, size_t coset_serialization_size); - /** Takes in a set of query positions to input oracles to a domain of size: - * `num_leaves * coset_serialization_size`, - * and the associated evaluations for each query position. + /** + * Takes in a set of query positions to input oracles to a domain of size: + * `num_leaves * coset_serialization_size`, + * and the associated evaluations for each query position. * - * This function then serializes these evaluations into leaf entries. - * The rows of a leaf entry are the same as in the eva - */ + * This function then serializes these evaluations into leaf entries. + * The rows of a leaf entry are the same as in the eva + */ std::vector> serialize_leaf_values_by_coset( const std::vector &query_positions, const std::vector > &query_responses, @@ -136,9 +149,11 @@ class merkle_tree { const std::vector> &leaf_contents, const merkle_tree_set_membership_proof &proof); - /** Returns a number that is proportional to the hashing runtime of verifying a set membership - * proof. Each two-to-one hash is counted as 2 units, and each input of the cap hash is 1 unit. - * Leaf hashes are not counted. */ + /** + * Returns a number that is proportional to the hashing runtime of verifying a set membership + * proof. Each two-to-one hash is counted as 2 units, and each input of the cap hash is 1 unit. + * Leaf hashes are not counted. + */ size_t count_internal_hash_complexity_to_verify_set_membership( const std::vector &positions) const; diff --git a/libiop/bcs/merkle_tree.tcc b/libiop/bcs/merkle_tree.tcc index 5d26d5a3..711068ca 100644 --- a/libiop/bcs/merkle_tree.tcc +++ b/libiop/bcs/merkle_tree.tcc @@ -211,8 +211,9 @@ std::vector> merkle_tree::serializ template void merkle_tree::compute_inner_nodes() { - /* n is the first index of the layer we're about to compute. It starts at the bottom layer. - This hack works because num_leaves is the index of the right child of the bottom-left node. */ + /* n is the first index of the layer we're about to compute. It starts at the bottom-left most + inner node. This hack works because num_leaves is the index of the right child of the + bottom-left inner node. */ size_t n = this->parent_of(this->num_leaves_); while (true) { @@ -349,8 +350,8 @@ merkle_tree_set_membership_proof std::swap(S, new_S); } - // Add the cap, including the root's direct children and the root. - // The only elements should be the cap (not including the root). + // Add the cap, i.e. the root's direct children. + // The only elements should be the cap. assert(S.size() <= this->cap_size_); auto it = S.begin(); // Iterate over every direct child of the root, and add the ones not obtainable from positions. @@ -497,8 +498,10 @@ bool merkle_tree::validate_set_membership_proof( } else { - /* b) Our right sibling is in S. So don't need auxiliary and skip over the - right sibling The parent will be obtained) next iteration. */ + /* b) Our right sibling is in S. So don't need + auxiliary and skip over the right sibling. + (Note that only one parent will be processed.) + */ right_hash = next_it->second; ++next_it; } diff --git a/libiop/bcs/pow.tcc b/libiop/bcs/pow.tcc index f361bfee..83a0fa78 100644 --- a/libiop/bcs/pow.tcc +++ b/libiop/bcs/pow.tcc @@ -95,7 +95,6 @@ hash_digest_type pow::solve_pow_internal( size_t pow_int = 0; while (!this->verify_pow(node_hasher, challenge, pow)) { - // printf("Trying %zx\n", pow[(num_words - 1)*sizeof(size_t)]); std::memcpy(&pow[(num_words - 1)*sizeof(size_t)], &pow_int, sizeof(size_t)); pow_int += 1; } @@ -148,10 +147,6 @@ bool pow::verify_pow_internal( size_t least_significant_word; std::memcpy(&least_significant_word, &hash[(num_words - 1)*sizeof(size_t)], sizeof(size_t)); size_t relevant_bits = least_significant_word & ((1 << this->parameters_.pow_bitlen()) - 1); - // printf("upper bound: %zx\n", this->parameters_.pow_upperbound()); - // printf("bit mask: %zx\n", (1 << this->parameters_.pow_bitlen()) - 1); - // printf("least significant word: %zx\n", least_significant_word); - // printf("relevant bits: %zx\n", relevant_bits); return relevant_bits <= this->parameters_.pow_upperbound(); } diff --git a/libiop/tests/bcs/test_merkle_tree.cpp b/libiop/tests/bcs/test_merkle_tree.cpp index 9f97db1b..b0fdd75b 100644 --- a/libiop/tests/bcs/test_merkle_tree.cpp +++ b/libiop/tests/bcs/test_merkle_tree.cpp @@ -245,7 +245,7 @@ void run_random_multi_test(const size_t size, const size_t digest_len_bytes, con { std::vector subset_elements; std::vector> subset_leaves; - /* The commented-out code generates subsets that are unsorted and may be repeats. + /* TODO: The commented-out code generates subsets that are unsorted and may be repeats. They are not used because the code currently cannot handle these cases if it is zero knowledge. */ // for (size_t j = 0; j < subset_size; j++)