feat: add blocks table to csv exporter (#396)

* implements display to MinerNonce * feat: add blocks to CsvExporter * add blocks to justfile * flush pending data --------- Co-authored-by: Renato Dinhani <[email protected]>
cloudwalk · Mar 19, 2024 · 2d1eb52 · 2d1eb52
1 parent bbfda20
commit 2d1eb52
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 4 deletions.
diff --git a/justfile b/justfile
@@ -94,17 +94,19 @@ db-load-csv:
     echo "truncate historical_nonces;"   >> data/psql.txt
     echo "truncate historical_balances;" >> data/psql.txt
     echo "truncate historical_slots;"    >> data/psql.txt
+    echo "truncate blocks;"              >> data/psql.txt
     echo "truncate transactions;"        >> data/psql.txt
-    echo "truncate logs;"                >> data/psql.txt
-    echo "truncate topics" >> data/psql.txt
+    echo "truncate logs;"                >> data/psql.txt    
+    echo "truncate topics"               >> data/psql.txt
 
     ls -tr1 data/historical_nonces-*.csv   | xargs -I{} printf "\\\\copy historical_nonces   from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
     ls -tr1 data/historical_balances-*.csv | xargs -I{} printf "\\\\copy historical_balances from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
     ls -tr1 data/historical_slots-*.csv    | xargs -I{} printf "\\\\copy historical_slots    from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
+    ls -tr1 data/blocks-*.csv              | xargs -I{} printf "\\\\copy blocks              from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
     ls -tr1 data/transactions-*.csv        | xargs -I{} printf "\\\\copy transactions        from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
-    ls -tr1 data/logs-*.csv                | xargs -I{} printf "\\\\copy logs                from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
     ls -tr1 data/topics-*.csv              | xargs -I{} printf "\\\\copy topics              from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
-
+    ls -tr1 data/logs-*.csv                | xargs -I{} printf "\\\\copy logs                from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
+
     cat data/psql.txt | pgcli -h localhost -u postgres -d stratus --less-chatty
 
 # ------------------------------------------------------------------------------

diff --git a/src/eth/primitives/miner_nonce.rs b/src/eth/primitives/miner_nonce.rs
@@ -1,3 +1,5 @@
+use std::fmt::Display;
+
 use ethereum_types::H64;
 use fake::Dummy;
 use fake::Faker;
@@ -27,6 +29,12 @@ impl Dummy<Faker> for MinerNonce {
     }
 }
 
+impl Display for MinerNonce {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
 // -----------------------------------------------------------------------------
 // Conversions: Other -> Self
 // -----------------------------------------------------------------------------

diff --git a/src/eth/storage/csv/csv_exporter.rs b/src/eth/storage/csv/csv_exporter.rs
@@ -9,6 +9,7 @@ use itertools::Itertools;
 
 use crate::eth::primitives::Account;
 use crate::eth::primitives::Block;
+use crate::eth::primitives::BlockHeader;
 use crate::eth::primitives::BlockNumber;
 use crate::eth::primitives::ExecutionAccountChanges;
 use crate::eth::primitives::LogMined;
@@ -70,6 +71,32 @@ const TRANSACTIONS_HEADERS: [&str; 20] = [
     "updated_at",
 ];
 
+const BLOCKS_FILE: &str = "data/blocks";
+
+const BLOCKS_HEADERS: [&str; 21] = [
+    "id",
+    "number",
+    "hash",
+    "transactions_root",
+    "gas_limit",
+    "gas_used",
+    "logs_bloom",
+    "timestamp_in_secs",
+    "parent_hash",
+    "author",
+    "extra_data",
+    "miner",
+    "difficulty",
+    "receipts_root",
+    "uncle_hash",
+    "size",
+    "state_root",
+    "total_difficulty",
+    "nonce",
+    "created_at",
+    "updated_at",
+];
+
 const LOGS_FILE: &str = "data/logs";
 
 const LOGS_HEADERS: [&str; 10] = [
@@ -124,6 +151,9 @@ pub struct CsvExporter {
     transactions_csv: csv::Writer<File>,
     transactions_id: LastId,
 
+    blocks_csv: csv::Writer<File>,
+    blocks_id: LastId,
+
     logs_csv: csv::Writer<File>,
     logs_id: LastId,
 
@@ -153,6 +183,9 @@ impl CsvExporter {
             transactions_csv: csv_writer(TRANSACTIONS_FILE, number, &TRANSACTIONS_HEADERS)?,
             transactions_id: LastId::new(TRANSACTIONS_FILE)?,
 
+            blocks_csv: csv_writer(BLOCKS_FILE, number, &BLOCKS_HEADERS)?,
+            blocks_id: LastId::new(BLOCKS_FILE)?,
+
             logs_csv: csv_writer(LOGS_FILE, number, &LOGS_HEADERS)?,
             logs_id: LastId::new(LOGS_FILE)?,
 
@@ -190,6 +223,7 @@ impl CsvExporter {
         for block in blocks {
             self.export_account_changes(block.compact_account_changes(), block.number())?;
             self.export_transactions(block.transactions)?;
+            self.export_blocks(block.header)?;
         }
 
         // flush pending data
@@ -205,6 +239,9 @@ impl CsvExporter {
         self.transactions_csv.flush()?;
         self.transactions_id.save()?;
 
+        self.blocks_csv.flush()?;
+        self.blocks_id.save()?;
+
         self.logs_csv.flush()?;
         self.logs_id.save()?;
 
@@ -271,6 +308,39 @@ impl CsvExporter {
         Ok(())
     }
 
+    fn export_blocks(&mut self, block: BlockHeader) -> anyhow::Result<()> {
+        self.blocks_id.value += 1;
+
+        let now = now();
+        let record = [
+            self.blocks_id.value.to_string(),    // id
+            block.number.to_string(),            // number
+            block.hash.to_string(),              // hash
+            block.transactions_root.to_string(), // transactions_root
+            block.gas_limit.to_string(),         // gas_limit
+            block.gas_used.to_string(),          // gas_used
+            block.bloom.to_string(),             // logs_bloom
+            block.timestamp.to_string(),         // timestamp_in_secs
+            block.parent_hash.to_string(),       // parent_hash
+            block.author.to_string(),            // author
+            block.extra_data.to_string(),        // extra_data
+            block.miner.to_string(),             // miner
+            block.difficulty.to_string(),        // difficulty
+            block.receipts_root.to_string(),     // receipts_root
+            block.uncle_hash.to_string(),        // uncle_hash
+            block.size.to_string(),              // size
+            block.state_root.to_string(),        // state_root
+            block.total_difficulty.to_string(),  // total_difficulty
+            block.nonce.to_string(),             // nonce
+            now.clone(),                         // created_at
+            now,                                 // updated_at
+        ];
+
+        self.blocks_csv.write_record(record).context("failed to write csv block")?;
+
+        Ok(())
+    }
+
     fn export_account_changes(&mut self, changes: Vec<ExecutionAccountChanges>, block_number: &BlockNumber) -> anyhow::Result<()> {
         for change in changes {
             // historical_nonces