From fdd0cd06de0a23e43e00954856f044fdde4bf429 Mon Sep 17 00:00:00 2001 From: Bruno Antonieto <102366754+brunoantonieto-cw@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:52:56 -0300 Subject: [PATCH] feat: add historical slots table to CsvExporter (#394) * feat: add historical slots table to CsvExporter * export historical_slots * fix historical_slots * fix: adjust headers --------- Co-authored-by: Renato Dinhani --- justfile | 17 +++++++++----- src/eth/storage/csv/csv_exporter.rs | 35 +++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/justfile b/justfile index 2ab9df242..c23875e6f 100644 --- a/justfile +++ b/justfile @@ -90,10 +90,17 @@ alias sqlx := db-compile # Database: Load CSV data produced by importer-offline db-load-csv: echo "" > data/psql.txt - echo "truncate transactions;" >> data/psql.txt - echo "truncate logs;" >> data/psql.txt - ls -tr1 data/transactions-*.csv | xargs -I{} printf "\\\\copy transactions from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt - ls -tr1 data/logs-*.csv | xargs -I{} printf "\\\\copy logs from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt + + echo "truncate historical_balances;" >> data/psql.txt + echo "truncate historical_slots;" >> data/psql.txt + echo "truncate transactions;" >> data/psql.txt + echo "truncate logs;" >> data/psql.txt + + ls -tr1 data/historical_balances-*.csv | xargs -I{} printf "\\\\copy historical_balances from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt + ls -tr1 data/historical_slots-*.csv | xargs -I{} printf "\\\\copy historical_slots from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt + ls -tr1 data/transactions-*.csv | xargs -I{} printf "\\\\copy transactions from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt + ls -tr1 data/logs-*.csv | xargs -I{} printf "\\\\copy logs from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt + cat data/psql.txt | pgcli -h localhost -u postgres -d stratus --less-chatty # ------------------------------------------------------------------------------ @@ -384,7 +391,7 @@ contracts-test-stratus-postgres: # Contracts: run contract integration tests contracts-test-int: - #!/bin/bash + #!/bin/bash cd e2e-contracts && ./flatten-contracts.sh [ -d integration ] && cd integration [ ! -f hardhat.config.ts ] && { cp ../../e2e/hardhat.config.ts .; } diff --git a/src/eth/storage/csv/csv_exporter.rs b/src/eth/storage/csv/csv_exporter.rs index 8186d9d41..516400e59 100644 --- a/src/eth/storage/csv/csv_exporter.rs +++ b/src/eth/storage/csv/csv_exporter.rs @@ -35,6 +35,10 @@ const ACCOUNTS_HEADERS: [&str; 10] = [ const HISTORICAL_BALANCES_FILE: &str = "data/historical_balances"; +const HISTORICAL_SLOTS_FILE: &str = "data/historical_slots"; + +const HISTORICAL_SLOTS_HEADERS: [&str; 7] = ["id", "idx", "value", "block_number", "account_address", "created_at", "updated_at"]; + const HISTORICAL_BALANCES_HEADERS: [&str; 6] = ["id", "address", "balance", "block_number", "created_at", "updated_at"]; const TRANSACTIONS_FILE: &str = "data/transactions"; @@ -89,6 +93,9 @@ pub struct CsvExporter { accounts_csv: csv::Writer, accounts_id: LastId, + historical_slots_csv: csv::Writer, + historical_slots_id: LastId, + historical_balances_csv: csv::Writer, historical_balances_id: LastId, @@ -109,6 +116,9 @@ impl CsvExporter { accounts_csv: csv_writer(ACCOUNTS_FILE, BlockNumber::ZERO, &ACCOUNTS_HEADERS)?, accounts_id: LastId::new_zero(ACCOUNTS_FILE), + historical_slots_csv: csv_writer(HISTORICAL_SLOTS_FILE, number, &HISTORICAL_SLOTS_HEADERS)?, + historical_slots_id: LastId::new(HISTORICAL_SLOTS_FILE)?, + historical_balances_csv: csv_writer(HISTORICAL_BALANCES_FILE, number, &HISTORICAL_BALANCES_HEADERS)?, historical_balances_id: LastId::new(HISTORICAL_BALANCES_FILE)?, @@ -152,6 +162,9 @@ impl CsvExporter { } // flush pending data + self.historical_slots_csv.flush()?; + self.historical_slots_id.save()?; + self.historical_balances_csv.flush()?; self.historical_balances_id.save()?; @@ -223,9 +236,12 @@ impl CsvExporter { fn export_account_changes(&mut self, changes: Vec, block_number: &BlockNumber) -> anyhow::Result<()> { for change in changes { + // historical_nonces if let Some(_nonce) = change.nonce.take_modified() { // todo: export historical nonce } + + // historical_balances if let Some(balance) = change.balance.take_modified() { let now = now(); self.historical_balances_id.value += 1; @@ -241,8 +257,23 @@ impl CsvExporter { .write_record(row) .context("failed to write csv historical balances")?; } - if let Some(_bytecode) = change.bytecode.take_modified() { - // todo: export historical bytecode + + // historical_slots + for slot in change.slots.into_values() { + if let Some(slot) = slot.take_modified() { + let now = now(); + self.historical_slots_id.value += 1; + let row = [ + self.historical_slots_id.value.to_string(), // id + slot.index.to_string(), // idx + slot.value.to_string(), // value + block_number.to_string(), // block_number + change.address.to_string(), // account_address + now.clone(), // updated_at + now, // created_at + ]; + self.historical_slots_csv.write_record(row).context("failed to write csv historical slots")?; + } } } Ok(())