From 2d1eb523e93740f179bace0c3dbddc78f5e2ab2a Mon Sep 17 00:00:00 2001 From: Bruno Antonieto <102366754+brunoantonieto-cw@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:32:44 -0300 Subject: [PATCH] feat: add blocks table to csv exporter (#396) * implements display to MinerNonce * feat: add blocks to CsvExporter * add blocks to justfile * flush pending data --------- Co-authored-by: Renato Dinhani <101204870+dinhani-cw@users.noreply.github.com> --- justfile | 10 +++-- src/eth/primitives/miner_nonce.rs | 8 ++++ src/eth/storage/csv/csv_exporter.rs | 70 +++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/justfile b/justfile index 2f1399391..b1bde878b 100644 --- a/justfile +++ b/justfile @@ -94,17 +94,19 @@ db-load-csv: echo "truncate historical_nonces;" >> data/psql.txt echo "truncate historical_balances;" >> data/psql.txt echo "truncate historical_slots;" >> data/psql.txt + echo "truncate blocks;" >> data/psql.txt echo "truncate transactions;" >> data/psql.txt - echo "truncate logs;" >> data/psql.txt - echo "truncate topics" >> data/psql.txt + echo "truncate logs;" >> data/psql.txt + echo "truncate topics" >> data/psql.txt ls -tr1 data/historical_nonces-*.csv | xargs -I{} printf "\\\\copy historical_nonces from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt ls -tr1 data/historical_balances-*.csv | xargs -I{} printf "\\\\copy historical_balances from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt ls -tr1 data/historical_slots-*.csv | xargs -I{} printf "\\\\copy historical_slots from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt + ls -tr1 data/blocks-*.csv | xargs -I{} printf "\\\\copy blocks from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt ls -tr1 data/transactions-*.csv | xargs -I{} printf "\\\\copy transactions from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt - ls -tr1 data/logs-*.csv | xargs -I{} printf "\\\\copy logs from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt ls -tr1 data/topics-*.csv | xargs -I{} printf "\\\\copy topics from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt - + ls -tr1 data/logs-*.csv | xargs -I{} printf "\\\\copy logs from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt + cat data/psql.txt | pgcli -h localhost -u postgres -d stratus --less-chatty # ------------------------------------------------------------------------------ diff --git a/src/eth/primitives/miner_nonce.rs b/src/eth/primitives/miner_nonce.rs index f289edea9..b2d193708 100644 --- a/src/eth/primitives/miner_nonce.rs +++ b/src/eth/primitives/miner_nonce.rs @@ -1,3 +1,5 @@ +use std::fmt::Display; + use ethereum_types::H64; use fake::Dummy; use fake::Faker; @@ -27,6 +29,12 @@ impl Dummy for MinerNonce { } } +impl Display for MinerNonce { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + // ----------------------------------------------------------------------------- // Conversions: Other -> Self // ----------------------------------------------------------------------------- diff --git a/src/eth/storage/csv/csv_exporter.rs b/src/eth/storage/csv/csv_exporter.rs index 57922f48e..7c163aa28 100644 --- a/src/eth/storage/csv/csv_exporter.rs +++ b/src/eth/storage/csv/csv_exporter.rs @@ -9,6 +9,7 @@ use itertools::Itertools; use crate::eth::primitives::Account; use crate::eth::primitives::Block; +use crate::eth::primitives::BlockHeader; use crate::eth::primitives::BlockNumber; use crate::eth::primitives::ExecutionAccountChanges; use crate::eth::primitives::LogMined; @@ -70,6 +71,32 @@ const TRANSACTIONS_HEADERS: [&str; 20] = [ "updated_at", ]; +const BLOCKS_FILE: &str = "data/blocks"; + +const BLOCKS_HEADERS: [&str; 21] = [ + "id", + "number", + "hash", + "transactions_root", + "gas_limit", + "gas_used", + "logs_bloom", + "timestamp_in_secs", + "parent_hash", + "author", + "extra_data", + "miner", + "difficulty", + "receipts_root", + "uncle_hash", + "size", + "state_root", + "total_difficulty", + "nonce", + "created_at", + "updated_at", +]; + const LOGS_FILE: &str = "data/logs"; const LOGS_HEADERS: [&str; 10] = [ @@ -124,6 +151,9 @@ pub struct CsvExporter { transactions_csv: csv::Writer, transactions_id: LastId, + blocks_csv: csv::Writer, + blocks_id: LastId, + logs_csv: csv::Writer, logs_id: LastId, @@ -153,6 +183,9 @@ impl CsvExporter { transactions_csv: csv_writer(TRANSACTIONS_FILE, number, &TRANSACTIONS_HEADERS)?, transactions_id: LastId::new(TRANSACTIONS_FILE)?, + blocks_csv: csv_writer(BLOCKS_FILE, number, &BLOCKS_HEADERS)?, + blocks_id: LastId::new(BLOCKS_FILE)?, + logs_csv: csv_writer(LOGS_FILE, number, &LOGS_HEADERS)?, logs_id: LastId::new(LOGS_FILE)?, @@ -190,6 +223,7 @@ impl CsvExporter { for block in blocks { self.export_account_changes(block.compact_account_changes(), block.number())?; self.export_transactions(block.transactions)?; + self.export_blocks(block.header)?; } // flush pending data @@ -205,6 +239,9 @@ impl CsvExporter { self.transactions_csv.flush()?; self.transactions_id.save()?; + self.blocks_csv.flush()?; + self.blocks_id.save()?; + self.logs_csv.flush()?; self.logs_id.save()?; @@ -271,6 +308,39 @@ impl CsvExporter { Ok(()) } + fn export_blocks(&mut self, block: BlockHeader) -> anyhow::Result<()> { + self.blocks_id.value += 1; + + let now = now(); + let record = [ + self.blocks_id.value.to_string(), // id + block.number.to_string(), // number + block.hash.to_string(), // hash + block.transactions_root.to_string(), // transactions_root + block.gas_limit.to_string(), // gas_limit + block.gas_used.to_string(), // gas_used + block.bloom.to_string(), // logs_bloom + block.timestamp.to_string(), // timestamp_in_secs + block.parent_hash.to_string(), // parent_hash + block.author.to_string(), // author + block.extra_data.to_string(), // extra_data + block.miner.to_string(), // miner + block.difficulty.to_string(), // difficulty + block.receipts_root.to_string(), // receipts_root + block.uncle_hash.to_string(), // uncle_hash + block.size.to_string(), // size + block.state_root.to_string(), // state_root + block.total_difficulty.to_string(), // total_difficulty + block.nonce.to_string(), // nonce + now.clone(), // created_at + now, // updated_at + ]; + + self.blocks_csv.write_record(record).context("failed to write csv block")?; + + Ok(()) + } + fn export_account_changes(&mut self, changes: Vec, block_number: &BlockNumber) -> anyhow::Result<()> { for change in changes { // historical_nonces