Proto overhaul (#1259)

* chore: create a proto based on eth

* chore: further optimize proto types

* chore: finish proto serialization

* chore: adapt tests to the new proto definitions

* lint

build.rs

@@ -36,7 +36,10 @@ fn print_build_directives() {
 // Code generation: Proto files
 // -----------------------------------------------------------------------------
 fn generate_proto_structs() {
-    tonic_build::compile_protos("static/proto/append_entry.proto").unwrap();
+    tonic_build::configure()
+        .protoc_arg("--experimental_allow_proto3_optional")
+        .compile(&["static/proto/append_entry.proto"], &["static/proto"])
+        .unwrap();
 }
 
 // -----------------------------------------------------------------------------

chaos/experiments/leader-election.sh

@@ -86,7 +86,7 @@ check_leader() {
     local grpc_address=$1
 
     # Send the gRPC request using grpcurl and capture both stdout and stderr
-    response=$(grpcurl -import-path static/proto -proto append_entry.proto -plaintext -d '{"term": 0, "prevLogIndex": 0, "prevLogTerm": 0, "leader_id": "leader_id_value", "block_entry": {"number": 1, "hash": "hash_value", "transactions_root": "transactions_root_value", "gas_used": "gas_used_value", "gas_limit": "gas_limit_value", "bloom": "bloom_value", "timestamp": 123456789, "parent_hash": "parent_hash_value", "author": "author_value", "extra_data": "ZXh0cmFfZGF0YV92YWx1ZQ==", "miner": "miner_value", "difficulty": "difficulty_value", "receipts_root": "receipts_root_value", "uncle_hash": "uncle_hash_value", "size": 12345, "state_root": "state_root_value", "total_difficulty": "total_difficulty_value", "nonce": "nonce_value", "transaction_hashes": ["tx_hash1", "tx_hash2"]}}' "$grpc_address" append_entry.AppendEntryService/AppendBlockCommit 2>&1)
+    response=$(grpcurl -import-path static/proto -proto append_entry.proto -plaintext -d '{"term": 5, "prevLogIndex": 0, "prevLogTerm": 4, "leader_id": "leader_id_value", "block_entry": {"number": 1, "hash": "ZXh0cmFfZGF0YV92YWx1ZQ==", "transactions_root": "ZXh0cmFfZGF0YV92YWx1ZQ==", "gas_used": 999, "gas_limit": 999, "bloom": "ZXh0cmFfZGF0YV92YWx1ZQ==", "timestamp": 123456789, "parent_hash": "ZXh0cmFfZGF0YV92YWx1ZQ==", "author": "ZXh0cmFfZGF0YV92YWx1ZQ==", "extra_data": "ZXh0cmFfZGF0YV92YWx1ZQ==", "miner": "ZXh0cmFfZGF0YV92YWx1ZQ==", "receipts_root": "ZXh0cmFfZGF0YV92YWx1ZQ==", "uncle_hash": "ZXh0cmFfZGF0YV92YWx1ZQ==", "size": 12345, "state_root": "ZXh0cmFfZGF0YV92YWx1ZQ==", "transaction_hashes": []}}' "$grpc_address" append_entry.AppendEntryService/AppendBlockCommit 2>&1)
 
     # Check the response for specific strings to determine the node status
     if [[ "$response" == *"append_transaction_executions called on leader node"* ]]; then

src/eth/block_miner.rs

@@ -96,6 +96,7 @@ impl BlockMiner {
         self.storage.save_execution(tx_execution.clone())?;
 
         // decide what to do based on mining mode
+        // FIXME consensus should be synchronous here and wait for the confirmation from the majority
         let _ = self.notifier_pending_txs.send(tx_execution);
         if self.mode.is_automine() {
             self.mine_local_and_commit()?;

src/eth/consensus/append_log_entries_storage.rs

@@ -116,14 +116,11 @@ mod tests {
                 assert_eq!(block.hash, expected_block.hash);
                 assert_eq!(block.number, expected_block.number);
                 assert_eq!(block.parent_hash, expected_block.parent_hash);
-                assert_eq!(block.nonce, expected_block.nonce);
                 assert_eq!(block.uncle_hash, expected_block.uncle_hash);
                 assert_eq!(block.transactions_root, expected_block.transactions_root);
                 assert_eq!(block.state_root, expected_block.state_root);
                 assert_eq!(block.receipts_root, expected_block.receipts_root);
                 assert_eq!(block.miner, expected_block.miner);
-                assert_eq!(block.difficulty, expected_block.difficulty);
-                assert_eq!(block.total_difficulty, expected_block.total_difficulty);
                 assert_eq!(block.extra_data, expected_block.extra_data);
                 assert_eq!(block.size, expected_block.size);
                 assert_eq!(block.gas_limit, expected_block.gas_limit);
@@ -171,7 +168,6 @@ mod tests {
                     assert_eq!(execution.output, expected_execution.output);
                     assert_eq!(execution.from, expected_execution.from);
                     assert_eq!(execution.to, expected_execution.to);
-                    assert_eq!(execution.block_hash, expected_execution.block_hash);
                     assert_eq!(execution.block_number, expected_execution.block_number);
                     assert_eq!(execution.transaction_index, expected_execution.transaction_index);
                     assert_eq!(execution.logs.len(), expected_execution.logs.len());
@@ -180,8 +176,6 @@ mod tests {
                         assert_eq!(log.topics, expected_log.topics);
                         assert_eq!(log.data, expected_log.data);
                         assert_eq!(log.log_index, expected_log.log_index);
-                        assert_eq!(log.removed, expected_log.removed);
-                        assert_eq!(log.transaction_log_index, expected_log.transaction_log_index);
                     }
                     assert_eq!(execution.gas, expected_execution.gas);
                     assert_eq!(execution.receipt_cumulative_gas_used, expected_execution.receipt_cumulative_gas_used);

src/eth/consensus/mod.rs

@@ -5,6 +5,7 @@ mod discovery;
 pub mod forward_to;
 #[allow(dead_code)]
 mod log_entry;
+pub mod utils;
 
 mod server;
 

src/eth/consensus/tests/factories.rs

@@ -2,8 +2,8 @@ use std::net::Ipv4Addr;
 use std::net::SocketAddr;
 use std::sync::Arc;
 
+use ethereum_types::H160;
 use ethereum_types::H256;
-use ethereum_types::U256;
 use rand::Rng;
 use tokio::sync::broadcast;
 use tokio::sync::Mutex;
@@ -12,65 +12,59 @@ use crate::eth::consensus::append_entry::AppendBlockCommitResponse;
 use crate::eth::consensus::append_entry::AppendTransactionExecutionsResponse;
 use crate::eth::consensus::append_entry::Log;
 use crate::eth::consensus::append_entry::RequestVoteResponse;
+use crate::eth::consensus::append_entry::TransactionExecutionEntry;
 use crate::eth::consensus::log_entry::LogEntry;
 use crate::eth::consensus::BlockEntry;
 use crate::eth::consensus::Consensus;
 use crate::eth::consensus::LogEntryData;
 use crate::eth::consensus::Peer;
 use crate::eth::consensus::PeerAddress;
 use crate::eth::consensus::Role;
-use crate::eth::consensus::TransactionExecutionEntry;
 use crate::eth::storage::StratusStorage;
 
-pub fn create_mock_block_entry(transaction_hashes: Vec<String>) -> BlockEntry {
+pub fn create_mock_block_entry(transaction_hashes: Vec<Vec<u8>>) -> BlockEntry {
     BlockEntry {
         number: rand::thread_rng().gen(),
-        hash: H256::random().to_string(),
-        parent_hash: H256::random().to_string(),
-        nonce: H256::random().to_string(),
-        uncle_hash: H256::random().to_string(),
-        transactions_root: H256::random().to_string(),
-        state_root: H256::random().to_string(),
-        receipts_root: H256::random().to_string(),
-        miner: H256::random().to_string(),
-        difficulty: U256::from(rand::thread_rng().gen::<u64>()).to_string(),
-        total_difficulty: U256::from(rand::thread_rng().gen::<u64>()).to_string(),
+        hash: H256::random().as_bytes().to_vec(),
+        parent_hash: H256::random().as_bytes().to_vec(),
+        uncle_hash: H256::random().as_bytes().to_vec(),
+        transactions_root: H256::random().as_bytes().to_vec(),
+        state_root: H256::random().as_bytes().to_vec(),
+        receipts_root: H256::random().as_bytes().to_vec(),
+        miner: H160::random().as_bytes().to_vec(),
+        author: H160::random().as_bytes().to_vec(),
         extra_data: vec![rand::thread_rng().gen()],
         size: rand::thread_rng().gen(),
-        gas_limit: U256::from(rand::thread_rng().gen::<u64>()).to_string(),
-        gas_used: U256::from(rand::thread_rng().gen::<u64>()).to_string(),
+        gas_limit: rand::thread_rng().gen(),
+        gas_used: rand::thread_rng().gen(),
         timestamp: rand::thread_rng().gen(),
-        bloom: H256::random().to_string(),
-        author: H256::random().to_string(),
+        bloom: H256::random().as_bytes().to_vec(),
         transaction_hashes,
     }
 }
 
 pub fn create_mock_transaction_execution_entry() -> TransactionExecutionEntry {
     TransactionExecutionEntry {
-        hash: H256::random().to_string(),
+        hash: H256::random().as_bytes().to_vec(),
         nonce: rand::thread_rng().gen(),
         value: vec![rand::thread_rng().gen()],
         gas_price: vec![rand::thread_rng().gen()],
         input: vec![rand::thread_rng().gen()],
         v: rand::thread_rng().gen(),
         r: vec![rand::thread_rng().gen()],
         s: vec![rand::thread_rng().gen()],
-        chain_id: rand::thread_rng().gen(),
-        result: vec![rand::thread_rng().gen()],
+        chain_id: Some(rand::thread_rng().gen()),
+        result: "Success".to_string(),
         output: vec![rand::thread_rng().gen()],
-        from: H256::random().to_string(),
-        to: H256::random().to_string(),
-        block_hash: H256::random().to_string(),
+        from: H160::random().as_bytes().to_vec(),
+        to: Some(H160::random().as_bytes().to_vec()),
         block_number: rand::thread_rng().gen(),
         transaction_index: rand::thread_rng().gen(),
         logs: vec![Log {
-            address: H256::random().to_string(),
-            topics: vec![H256::random().to_string()],
+            address: H160::random().as_bytes().to_vec(),
+            topics: vec![H256::random().as_bytes().to_vec()],
             data: vec![rand::thread_rng().gen()],
             log_index: rand::thread_rng().gen(),
-            removed: rand::thread_rng().gen(),
-            transaction_log_index: rand::thread_rng().gen(),
         }],
         gas: vec![rand::thread_rng().gen()],
         receipt_cumulative_gas_used: vec![rand::thread_rng().gen()],
@@ -79,6 +73,11 @@ pub fn create_mock_transaction_execution_entry() -> TransactionExecutionEntry {
         receipt_status: rand::thread_rng().gen(),
         receipt_logs_bloom: vec![rand::thread_rng().gen()],
         receipt_effective_gas_price: vec![rand::thread_rng().gen()],
+        tx_type: Some(rand::thread_rng().gen()),
+        signer: vec![rand::thread_rng().gen()],
+        gas_limit: vec![rand::thread_rng().gen()],
+        receipt_applied: rand::thread_rng().gen(),
+        deployed_contract_address: Some(vec![rand::thread_rng().gen()]),
     }
 }
 

src/eth/consensus/tests/test_simple_blocks.rs

@@ -55,7 +55,7 @@ async fn test_append_entries_transaction_executions_and_block() {
     }
 
     // Create and append block with transaction hashes
-    let transaction_hashes: Vec<String> = all_executions.iter().map(|e| e.hash.clone()).collect();
+    let transaction_hashes: Vec<Vec<u8>> = all_executions.iter().map(|e| e.hash.clone()).collect();
 
     let block_entry = create_mock_block_entry(transaction_hashes.clone());
 

src/eth/consensus/utils.rs

@@ -0,0 +1,5 @@
+pub fn u256_to_bytes(u: ethereum_types::U256) -> Vec<u8> {
+    let mut bytes = [0u8; 32];
+    u.to_big_endian(&mut bytes);
+    bytes.to_vec()
+}

src/eth/primitives/block_header.rs

@@ -73,26 +73,23 @@ impl BlockHeader {
         }
     }
 
-    pub fn to_append_entry_block_header(&self, transaction_hashes: Vec<String>) -> append_entry::BlockEntry {
+    pub fn to_append_entry_block_header(&self, transaction_hashes: Vec<Vec<u8>>) -> append_entry::BlockEntry {
         append_entry::BlockEntry {
             number: self.number.into(),
-            hash: self.hash.to_string(),
-            transactions_root: self.transactions_root.to_string(),
-            gas_used: self.gas_used.to_string(),
-            gas_limit: self.gas_limit.to_string(),
-            bloom: self.bloom.to_string(),
+            hash: self.hash.as_fixed_bytes().to_vec(),
+            transactions_root: self.transactions_root.as_fixed_bytes().to_vec(),
+            gas_used: self.gas_used.as_u64(),
+            gas_limit: self.gas_limit.as_u64(),
+            bloom: self.bloom.as_bytes().to_vec(),
             timestamp: self.timestamp.as_u64(),
-            parent_hash: self.parent_hash.to_string(),
-            author: self.author.to_string(),
+            parent_hash: self.parent_hash.as_fixed_bytes().to_vec(),
+            author: self.author.to_fixed_bytes().to_vec(),
             extra_data: self.extra_data.clone().0,
-            miner: self.miner.to_string(),
-            difficulty: self.difficulty.to_string(),
-            receipts_root: self.receipts_root.to_string(),
-            uncle_hash: self.uncle_hash.to_string(),
+            miner: self.miner.to_fixed_bytes().to_vec(),
+            receipts_root: self.receipts_root.as_fixed_bytes().to_vec(),
+            uncle_hash: self.uncle_hash.as_fixed_bytes().to_vec(),
             size: self.size.into(),
-            state_root: self.state_root.to_string(),
-            total_difficulty: self.total_difficulty.to_string(),
-            nonce: self.nonce.to_string(),
+            state_root: self.state_root.as_fixed_bytes().to_vec(),
             transaction_hashes,
         }
     }

src/eth/primitives/hash.rs

@@ -43,6 +43,10 @@ impl Hash {
     pub fn inner_value(&self) -> H256 {
         self.0
     }
+
+    pub fn as_fixed_bytes(&self) -> &[u8; 32] {
+        self.0.as_fixed_bytes()
+    }
 }
 
 impl Display for Hash {

src/eth/primitives/logs_bloom.rs

@@ -9,6 +9,12 @@ use crate::gen_newtype_from;
 #[serde(transparent)]
 pub struct LogsBloom(Bloom);
 
+impl LogsBloom {
+    pub fn as_bytes(&self) -> &[u8] {
+        self.0.as_bytes()
+    }
+}
+
 impl Deref for LogsBloom {
     type Target = Bloom;
 

src/eth/primitives/transaction_execution.rs

@@ -1,6 +1,7 @@
 use display_json::DebugAsJson;
 
 use crate::eth::consensus::append_entry;
+use crate::eth::consensus::utils::*;
 use crate::eth::evm::EvmExecutionResult;
 use crate::eth::primitives::EvmExecution;
 use crate::eth::primitives::ExternalReceipt;
@@ -66,40 +67,31 @@ impl TransactionExecution {
 
     /// TODO: use From or TryFrom trait instead of this function
     pub fn to_append_entry_transaction(&self) -> append_entry::TransactionExecutionEntry {
-        fn u256_to_bytes(u: ethereum_types::U256) -> Vec<u8> {
-            let mut bytes = [0u8; 32];
-            u.to_big_endian(&mut bytes);
-            bytes.to_vec()
-        }
-
         match self {
             Self::External(ExternalTransactionExecution { tx, receipt, result }) => append_entry::TransactionExecutionEntry {
-                hash: tx.hash.to_string(),
+                hash: tx.hash.to_fixed_bytes().to_vec(),
                 nonce: tx.nonce.as_u64(),
                 value: u256_to_bytes(tx.value),
                 gas_price: tx.gas_price.map_or(vec![], u256_to_bytes),
                 input: tx.input.to_vec(),
                 v: tx.v.as_u64(),
                 r: u256_to_bytes(tx.r),
                 s: u256_to_bytes(tx.s),
-                chain_id: tx.chain_id.unwrap_or_default().as_u64(),
-                result: result.execution.output.to_vec(),
+                chain_id: Some(tx.chain_id.unwrap_or_default().as_u64()),
+                result: result.execution.result.to_string(),
                 output: result.execution.output.to_vec(),
-                from: tx.from.to_string(),
-                to: tx.to.unwrap_or_default().to_string(),
-                block_hash: receipt.block_hash().to_string(),
+                from: tx.from.as_bytes().to_vec(),
+                to: tx.to.map(|to| to.as_bytes().to_vec()),
                 block_number: receipt.block_number().as_u64(),
                 transaction_index: receipt.transaction_index.as_u64(),
                 logs: receipt
                     .logs
                     .iter()
                     .map(|log| append_entry::Log {
-                        address: log.address.to_string(),
-                        topics: log.topics.iter().map(|topic| topic.to_string()).collect(),
+                        address: log.address.as_bytes().to_vec(),
+                        topics: log.topics.iter().map(|topic| topic.as_bytes().to_vec()).collect(),
                         data: log.data.to_vec(),
                         log_index: log.log_index.unwrap_or_default().as_u64(),
-                        transaction_log_index: log.transaction_log_index.unwrap_or_default().as_u64(),
-                        removed: log.removed.unwrap_or(false),
                     })
                     .collect(),
                 gas: u256_to_bytes(tx.gas),
@@ -109,6 +101,11 @@ impl TransactionExecution {
                 receipt_status: receipt.status.unwrap_or_default().as_u32(),
                 receipt_logs_bloom: receipt.logs_bloom.as_bytes().to_vec(),
                 receipt_effective_gas_price: receipt.effective_gas_price.map_or(vec![], u256_to_bytes),
+                deployed_contract_address: None,
+                gas_limit: u256_to_bytes(tx.gas),
+                signer: vec![],
+                receipt_applied: true,
+                tx_type: None,
             },
             // TODO: no need to panic here, this could be implemented
             _ => panic!("Only ExternalTransactionExecution is supported"),