From 159bbf3ca8a7a54d64296bd39e7f85c3532a685d Mon Sep 17 00:00:00 2001 From: renancloudwalk <53792026+renancloudwalk@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:48:47 -0300 Subject: [PATCH] Proto overhaul (#1259) * chore: create a proto based on eth * chore: further optimize proto types * chore: finish proto serialization * chore: adapt tests to the new proto definitions * lint --- build.rs | 5 +- chaos/experiments/leader-election.sh | 2 +- src/eth/block_miner.rs | 1 + .../consensus/append_log_entries_storage.rs | 6 -- src/eth/consensus/mod.rs | 1 + src/eth/consensus/tests/factories.rs | 53 ++++++++-------- src/eth/consensus/tests/test_simple_blocks.rs | 2 +- src/eth/consensus/utils.rs | 5 ++ src/eth/primitives/block_header.rs | 27 ++++----- src/eth/primitives/hash.rs | 4 ++ src/eth/primitives/logs_bloom.rs | 6 ++ src/eth/primitives/transaction_execution.rs | 29 ++++----- static/proto/append_entry.proto | 60 +++++++++---------- 13 files changed, 103 insertions(+), 98 deletions(-) create mode 100644 src/eth/consensus/utils.rs diff --git a/build.rs b/build.rs index bb80e3120..759089e2c 100644 --- a/build.rs +++ b/build.rs @@ -36,7 +36,10 @@ fn print_build_directives() { // Code generation: Proto files // ----------------------------------------------------------------------------- fn generate_proto_structs() { - tonic_build::compile_protos("static/proto/append_entry.proto").unwrap(); + tonic_build::configure() + .protoc_arg("--experimental_allow_proto3_optional") + .compile(&["static/proto/append_entry.proto"], &["static/proto"]) + .unwrap(); } // ----------------------------------------------------------------------------- diff --git a/chaos/experiments/leader-election.sh b/chaos/experiments/leader-election.sh index 84c26a706..c175146ab 100755 --- a/chaos/experiments/leader-election.sh +++ b/chaos/experiments/leader-election.sh @@ -86,7 +86,7 @@ check_leader() { local grpc_address=$1 # Send the gRPC request using grpcurl and capture both stdout and stderr - response=$(grpcurl -import-path static/proto -proto append_entry.proto -plaintext -d '{"term": 0, "prevLogIndex": 0, "prevLogTerm": 0, "leader_id": "leader_id_value", "block_entry": {"number": 1, "hash": "hash_value", "transactions_root": "transactions_root_value", "gas_used": "gas_used_value", "gas_limit": "gas_limit_value", "bloom": "bloom_value", "timestamp": 123456789, "parent_hash": "parent_hash_value", "author": "author_value", "extra_data": "ZXh0cmFfZGF0YV92YWx1ZQ==", "miner": "miner_value", "difficulty": "difficulty_value", "receipts_root": "receipts_root_value", "uncle_hash": "uncle_hash_value", "size": 12345, "state_root": "state_root_value", "total_difficulty": "total_difficulty_value", "nonce": "nonce_value", "transaction_hashes": ["tx_hash1", "tx_hash2"]}}' "$grpc_address" append_entry.AppendEntryService/AppendBlockCommit 2>&1) + response=$(grpcurl -import-path static/proto -proto append_entry.proto -plaintext -d '{"term": 5, "prevLogIndex": 0, "prevLogTerm": 4, "leader_id": "leader_id_value", "block_entry": {"number": 1, "hash": "ZXh0cmFfZGF0YV92YWx1ZQ==", "transactions_root": "ZXh0cmFfZGF0YV92YWx1ZQ==", "gas_used": 999, "gas_limit": 999, "bloom": "ZXh0cmFfZGF0YV92YWx1ZQ==", "timestamp": 123456789, "parent_hash": "ZXh0cmFfZGF0YV92YWx1ZQ==", "author": "ZXh0cmFfZGF0YV92YWx1ZQ==", "extra_data": "ZXh0cmFfZGF0YV92YWx1ZQ==", "miner": "ZXh0cmFfZGF0YV92YWx1ZQ==", "receipts_root": "ZXh0cmFfZGF0YV92YWx1ZQ==", "uncle_hash": "ZXh0cmFfZGF0YV92YWx1ZQ==", "size": 12345, "state_root": "ZXh0cmFfZGF0YV92YWx1ZQ==", "transaction_hashes": []}}' "$grpc_address" append_entry.AppendEntryService/AppendBlockCommit 2>&1) # Check the response for specific strings to determine the node status if [[ "$response" == *"append_transaction_executions called on leader node"* ]]; then diff --git a/src/eth/block_miner.rs b/src/eth/block_miner.rs index 625f83811..6fa6dd5e9 100644 --- a/src/eth/block_miner.rs +++ b/src/eth/block_miner.rs @@ -96,6 +96,7 @@ impl BlockMiner { self.storage.save_execution(tx_execution.clone())?; // decide what to do based on mining mode + // FIXME consensus should be synchronous here and wait for the confirmation from the majority let _ = self.notifier_pending_txs.send(tx_execution); if self.mode.is_automine() { self.mine_local_and_commit()?; diff --git a/src/eth/consensus/append_log_entries_storage.rs b/src/eth/consensus/append_log_entries_storage.rs index 82f2ab923..26ee8430d 100644 --- a/src/eth/consensus/append_log_entries_storage.rs +++ b/src/eth/consensus/append_log_entries_storage.rs @@ -116,14 +116,11 @@ mod tests { assert_eq!(block.hash, expected_block.hash); assert_eq!(block.number, expected_block.number); assert_eq!(block.parent_hash, expected_block.parent_hash); - assert_eq!(block.nonce, expected_block.nonce); assert_eq!(block.uncle_hash, expected_block.uncle_hash); assert_eq!(block.transactions_root, expected_block.transactions_root); assert_eq!(block.state_root, expected_block.state_root); assert_eq!(block.receipts_root, expected_block.receipts_root); assert_eq!(block.miner, expected_block.miner); - assert_eq!(block.difficulty, expected_block.difficulty); - assert_eq!(block.total_difficulty, expected_block.total_difficulty); assert_eq!(block.extra_data, expected_block.extra_data); assert_eq!(block.size, expected_block.size); assert_eq!(block.gas_limit, expected_block.gas_limit); @@ -171,7 +168,6 @@ mod tests { assert_eq!(execution.output, expected_execution.output); assert_eq!(execution.from, expected_execution.from); assert_eq!(execution.to, expected_execution.to); - assert_eq!(execution.block_hash, expected_execution.block_hash); assert_eq!(execution.block_number, expected_execution.block_number); assert_eq!(execution.transaction_index, expected_execution.transaction_index); assert_eq!(execution.logs.len(), expected_execution.logs.len()); @@ -180,8 +176,6 @@ mod tests { assert_eq!(log.topics, expected_log.topics); assert_eq!(log.data, expected_log.data); assert_eq!(log.log_index, expected_log.log_index); - assert_eq!(log.removed, expected_log.removed); - assert_eq!(log.transaction_log_index, expected_log.transaction_log_index); } assert_eq!(execution.gas, expected_execution.gas); assert_eq!(execution.receipt_cumulative_gas_used, expected_execution.receipt_cumulative_gas_used); diff --git a/src/eth/consensus/mod.rs b/src/eth/consensus/mod.rs index b96d157cd..da0f27517 100644 --- a/src/eth/consensus/mod.rs +++ b/src/eth/consensus/mod.rs @@ -5,6 +5,7 @@ mod discovery; pub mod forward_to; #[allow(dead_code)] mod log_entry; +pub mod utils; mod server; diff --git a/src/eth/consensus/tests/factories.rs b/src/eth/consensus/tests/factories.rs index e5a51b093..86a5ce0f4 100644 --- a/src/eth/consensus/tests/factories.rs +++ b/src/eth/consensus/tests/factories.rs @@ -2,8 +2,8 @@ use std::net::Ipv4Addr; use std::net::SocketAddr; use std::sync::Arc; +use ethereum_types::H160; use ethereum_types::H256; -use ethereum_types::U256; use rand::Rng; use tokio::sync::broadcast; use tokio::sync::Mutex; @@ -12,6 +12,7 @@ use crate::eth::consensus::append_entry::AppendBlockCommitResponse; use crate::eth::consensus::append_entry::AppendTransactionExecutionsResponse; use crate::eth::consensus::append_entry::Log; use crate::eth::consensus::append_entry::RequestVoteResponse; +use crate::eth::consensus::append_entry::TransactionExecutionEntry; use crate::eth::consensus::log_entry::LogEntry; use crate::eth::consensus::BlockEntry; use crate::eth::consensus::Consensus; @@ -19,36 +20,32 @@ use crate::eth::consensus::LogEntryData; use crate::eth::consensus::Peer; use crate::eth::consensus::PeerAddress; use crate::eth::consensus::Role; -use crate::eth::consensus::TransactionExecutionEntry; use crate::eth::storage::StratusStorage; -pub fn create_mock_block_entry(transaction_hashes: Vec) -> BlockEntry { +pub fn create_mock_block_entry(transaction_hashes: Vec>) -> BlockEntry { BlockEntry { number: rand::thread_rng().gen(), - hash: H256::random().to_string(), - parent_hash: H256::random().to_string(), - nonce: H256::random().to_string(), - uncle_hash: H256::random().to_string(), - transactions_root: H256::random().to_string(), - state_root: H256::random().to_string(), - receipts_root: H256::random().to_string(), - miner: H256::random().to_string(), - difficulty: U256::from(rand::thread_rng().gen::()).to_string(), - total_difficulty: U256::from(rand::thread_rng().gen::()).to_string(), + hash: H256::random().as_bytes().to_vec(), + parent_hash: H256::random().as_bytes().to_vec(), + uncle_hash: H256::random().as_bytes().to_vec(), + transactions_root: H256::random().as_bytes().to_vec(), + state_root: H256::random().as_bytes().to_vec(), + receipts_root: H256::random().as_bytes().to_vec(), + miner: H160::random().as_bytes().to_vec(), + author: H160::random().as_bytes().to_vec(), extra_data: vec![rand::thread_rng().gen()], size: rand::thread_rng().gen(), - gas_limit: U256::from(rand::thread_rng().gen::()).to_string(), - gas_used: U256::from(rand::thread_rng().gen::()).to_string(), + gas_limit: rand::thread_rng().gen(), + gas_used: rand::thread_rng().gen(), timestamp: rand::thread_rng().gen(), - bloom: H256::random().to_string(), - author: H256::random().to_string(), + bloom: H256::random().as_bytes().to_vec(), transaction_hashes, } } pub fn create_mock_transaction_execution_entry() -> TransactionExecutionEntry { TransactionExecutionEntry { - hash: H256::random().to_string(), + hash: H256::random().as_bytes().to_vec(), nonce: rand::thread_rng().gen(), value: vec![rand::thread_rng().gen()], gas_price: vec![rand::thread_rng().gen()], @@ -56,21 +53,18 @@ pub fn create_mock_transaction_execution_entry() -> TransactionExecutionEntry { v: rand::thread_rng().gen(), r: vec![rand::thread_rng().gen()], s: vec![rand::thread_rng().gen()], - chain_id: rand::thread_rng().gen(), - result: vec![rand::thread_rng().gen()], + chain_id: Some(rand::thread_rng().gen()), + result: "Success".to_string(), output: vec![rand::thread_rng().gen()], - from: H256::random().to_string(), - to: H256::random().to_string(), - block_hash: H256::random().to_string(), + from: H160::random().as_bytes().to_vec(), + to: Some(H160::random().as_bytes().to_vec()), block_number: rand::thread_rng().gen(), transaction_index: rand::thread_rng().gen(), logs: vec![Log { - address: H256::random().to_string(), - topics: vec![H256::random().to_string()], + address: H160::random().as_bytes().to_vec(), + topics: vec![H256::random().as_bytes().to_vec()], data: vec![rand::thread_rng().gen()], log_index: rand::thread_rng().gen(), - removed: rand::thread_rng().gen(), - transaction_log_index: rand::thread_rng().gen(), }], gas: vec![rand::thread_rng().gen()], receipt_cumulative_gas_used: vec![rand::thread_rng().gen()], @@ -79,6 +73,11 @@ pub fn create_mock_transaction_execution_entry() -> TransactionExecutionEntry { receipt_status: rand::thread_rng().gen(), receipt_logs_bloom: vec![rand::thread_rng().gen()], receipt_effective_gas_price: vec![rand::thread_rng().gen()], + tx_type: Some(rand::thread_rng().gen()), + signer: vec![rand::thread_rng().gen()], + gas_limit: vec![rand::thread_rng().gen()], + receipt_applied: rand::thread_rng().gen(), + deployed_contract_address: Some(vec![rand::thread_rng().gen()]), } } diff --git a/src/eth/consensus/tests/test_simple_blocks.rs b/src/eth/consensus/tests/test_simple_blocks.rs index 880943428..35074196e 100644 --- a/src/eth/consensus/tests/test_simple_blocks.rs +++ b/src/eth/consensus/tests/test_simple_blocks.rs @@ -55,7 +55,7 @@ async fn test_append_entries_transaction_executions_and_block() { } // Create and append block with transaction hashes - let transaction_hashes: Vec = all_executions.iter().map(|e| e.hash.clone()).collect(); + let transaction_hashes: Vec> = all_executions.iter().map(|e| e.hash.clone()).collect(); let block_entry = create_mock_block_entry(transaction_hashes.clone()); diff --git a/src/eth/consensus/utils.rs b/src/eth/consensus/utils.rs new file mode 100644 index 000000000..18493546d --- /dev/null +++ b/src/eth/consensus/utils.rs @@ -0,0 +1,5 @@ +pub fn u256_to_bytes(u: ethereum_types::U256) -> Vec { + let mut bytes = [0u8; 32]; + u.to_big_endian(&mut bytes); + bytes.to_vec() +} diff --git a/src/eth/primitives/block_header.rs b/src/eth/primitives/block_header.rs index 3c763f5a2..e0009847f 100644 --- a/src/eth/primitives/block_header.rs +++ b/src/eth/primitives/block_header.rs @@ -73,26 +73,23 @@ impl BlockHeader { } } - pub fn to_append_entry_block_header(&self, transaction_hashes: Vec) -> append_entry::BlockEntry { + pub fn to_append_entry_block_header(&self, transaction_hashes: Vec>) -> append_entry::BlockEntry { append_entry::BlockEntry { number: self.number.into(), - hash: self.hash.to_string(), - transactions_root: self.transactions_root.to_string(), - gas_used: self.gas_used.to_string(), - gas_limit: self.gas_limit.to_string(), - bloom: self.bloom.to_string(), + hash: self.hash.as_fixed_bytes().to_vec(), + transactions_root: self.transactions_root.as_fixed_bytes().to_vec(), + gas_used: self.gas_used.as_u64(), + gas_limit: self.gas_limit.as_u64(), + bloom: self.bloom.as_bytes().to_vec(), timestamp: self.timestamp.as_u64(), - parent_hash: self.parent_hash.to_string(), - author: self.author.to_string(), + parent_hash: self.parent_hash.as_fixed_bytes().to_vec(), + author: self.author.to_fixed_bytes().to_vec(), extra_data: self.extra_data.clone().0, - miner: self.miner.to_string(), - difficulty: self.difficulty.to_string(), - receipts_root: self.receipts_root.to_string(), - uncle_hash: self.uncle_hash.to_string(), + miner: self.miner.to_fixed_bytes().to_vec(), + receipts_root: self.receipts_root.as_fixed_bytes().to_vec(), + uncle_hash: self.uncle_hash.as_fixed_bytes().to_vec(), size: self.size.into(), - state_root: self.state_root.to_string(), - total_difficulty: self.total_difficulty.to_string(), - nonce: self.nonce.to_string(), + state_root: self.state_root.as_fixed_bytes().to_vec(), transaction_hashes, } } diff --git a/src/eth/primitives/hash.rs b/src/eth/primitives/hash.rs index 7fc150e6f..eb03b4252 100644 --- a/src/eth/primitives/hash.rs +++ b/src/eth/primitives/hash.rs @@ -43,6 +43,10 @@ impl Hash { pub fn inner_value(&self) -> H256 { self.0 } + + pub fn as_fixed_bytes(&self) -> &[u8; 32] { + self.0.as_fixed_bytes() + } } impl Display for Hash { diff --git a/src/eth/primitives/logs_bloom.rs b/src/eth/primitives/logs_bloom.rs index 1b9da9d18..0030329be 100644 --- a/src/eth/primitives/logs_bloom.rs +++ b/src/eth/primitives/logs_bloom.rs @@ -9,6 +9,12 @@ use crate::gen_newtype_from; #[serde(transparent)] pub struct LogsBloom(Bloom); +impl LogsBloom { + pub fn as_bytes(&self) -> &[u8] { + self.0.as_bytes() + } +} + impl Deref for LogsBloom { type Target = Bloom; diff --git a/src/eth/primitives/transaction_execution.rs b/src/eth/primitives/transaction_execution.rs index c04dd8c71..872df6072 100644 --- a/src/eth/primitives/transaction_execution.rs +++ b/src/eth/primitives/transaction_execution.rs @@ -1,6 +1,7 @@ use display_json::DebugAsJson; use crate::eth::consensus::append_entry; +use crate::eth::consensus::utils::*; use crate::eth::evm::EvmExecutionResult; use crate::eth::primitives::EvmExecution; use crate::eth::primitives::ExternalReceipt; @@ -66,15 +67,9 @@ impl TransactionExecution { /// TODO: use From or TryFrom trait instead of this function pub fn to_append_entry_transaction(&self) -> append_entry::TransactionExecutionEntry { - fn u256_to_bytes(u: ethereum_types::U256) -> Vec { - let mut bytes = [0u8; 32]; - u.to_big_endian(&mut bytes); - bytes.to_vec() - } - match self { Self::External(ExternalTransactionExecution { tx, receipt, result }) => append_entry::TransactionExecutionEntry { - hash: tx.hash.to_string(), + hash: tx.hash.to_fixed_bytes().to_vec(), nonce: tx.nonce.as_u64(), value: u256_to_bytes(tx.value), gas_price: tx.gas_price.map_or(vec![], u256_to_bytes), @@ -82,24 +77,21 @@ impl TransactionExecution { v: tx.v.as_u64(), r: u256_to_bytes(tx.r), s: u256_to_bytes(tx.s), - chain_id: tx.chain_id.unwrap_or_default().as_u64(), - result: result.execution.output.to_vec(), + chain_id: Some(tx.chain_id.unwrap_or_default().as_u64()), + result: result.execution.result.to_string(), output: result.execution.output.to_vec(), - from: tx.from.to_string(), - to: tx.to.unwrap_or_default().to_string(), - block_hash: receipt.block_hash().to_string(), + from: tx.from.as_bytes().to_vec(), + to: tx.to.map(|to| to.as_bytes().to_vec()), block_number: receipt.block_number().as_u64(), transaction_index: receipt.transaction_index.as_u64(), logs: receipt .logs .iter() .map(|log| append_entry::Log { - address: log.address.to_string(), - topics: log.topics.iter().map(|topic| topic.to_string()).collect(), + address: log.address.as_bytes().to_vec(), + topics: log.topics.iter().map(|topic| topic.as_bytes().to_vec()).collect(), data: log.data.to_vec(), log_index: log.log_index.unwrap_or_default().as_u64(), - transaction_log_index: log.transaction_log_index.unwrap_or_default().as_u64(), - removed: log.removed.unwrap_or(false), }) .collect(), gas: u256_to_bytes(tx.gas), @@ -109,6 +101,11 @@ impl TransactionExecution { receipt_status: receipt.status.unwrap_or_default().as_u32(), receipt_logs_bloom: receipt.logs_bloom.as_bytes().to_vec(), receipt_effective_gas_price: receipt.effective_gas_price.map_or(vec![], u256_to_bytes), + deployed_contract_address: None, + gas_limit: u256_to_bytes(tx.gas), + signer: vec![], + receipt_applied: true, + tx_type: None, }, // TODO: no need to panic here, this could be implemented _ => panic!("Only ExternalTransactionExecution is supported"), diff --git a/static/proto/append_entry.proto b/static/proto/append_entry.proto index 8ec77732f..2e1072e01 100644 --- a/static/proto/append_entry.proto +++ b/static/proto/append_entry.proto @@ -23,16 +23,14 @@ enum StatusCode { } message Log { - string address = 1; - repeated string topics = 2; + bytes address = 1; + repeated bytes topics = 2; bytes data = 3; uint64 log_index = 4; - uint64 transaction_log_index = 5; - bool removed = 6; } message TransactionExecutionEntry { - string hash = 1; + bytes hash = 1; //H256 uint64 nonce = 2; bytes value = 3; bytes gas_price = 4; @@ -40,13 +38,11 @@ message TransactionExecutionEntry { uint64 v = 6; bytes r = 7; bytes s = 8; - uint64 chain_id = 9; - bytes result = 10; + optional uint64 chain_id = 9; + string result = 10; bytes output = 11; - string from = 12; - string to = 13; - string block_hash = 14; - uint64 block_number = 15; + bytes from = 12; + optional bytes to = 13; uint64 transaction_index = 16; repeated Log logs = 17; bytes gas = 18; @@ -56,6 +52,12 @@ message TransactionExecutionEntry { uint32 receipt_status = 22; bytes receipt_logs_bloom = 23; bytes receipt_effective_gas_price = 24; + uint64 block_number = 25; + optional uint64 tx_type = 26; + bytes signer = 27; + bytes gas_limit = 28; + bool receipt_applied = 29; + optional bytes deployed_contract_address = 30; } message AppendTransactionExecutionsRequest { @@ -72,27 +74,23 @@ message AppendTransactionExecutionsResponse { uint64 last_committed_block_number = 3; } -//TODO use eth friendly types message BlockEntry { - uint64 number = 1; - string hash = 2; - string transactions_root = 3; - string gas_used = 4; - string gas_limit = 5; - string bloom = 6; - uint64 timestamp = 7; - string parent_hash = 8; - string author = 9; - bytes extra_data = 10; - string miner = 11; - string difficulty = 12; - string receipts_root = 13; - string uncle_hash = 14; - uint64 size = 15; - string state_root = 16; - string total_difficulty = 17; - string nonce = 18; - repeated string transaction_hashes = 19; + uint64 number = 1; //U64 + bytes hash = 2; //H256 + bytes transactions_root = 3; //H256 + uint64 gas_used = 4; //U64 + uint64 gas_limit = 5; + bytes bloom = 6; //[u8] + uint64 timestamp = 7; //UnixTime + bytes parent_hash = 8; //H256 + bytes author = 9; //H160 + bytes extra_data = 10; //bytes + bytes miner = 11; //H160 + bytes receipts_root = 12; //H256 + bytes uncle_hash = 13; //H256 + uint64 size = 14; + bytes state_root = 15; //H256 + repeated bytes transaction_hashes = 16; //H256 } message AppendBlockCommitRequest {