feat: add historical slots table to CsvExporter (#394)

* feat: add historical slots table to CsvExporter * export historical_slots * fix historical_slots * fix: adjust headers --------- Co-authored-by: Renato Dinhani <[email protected]>
cloudwalk · Mar 19, 2024 · fdd0cd0 · fdd0cd0
1 parent 894c401
commit fdd0cd0
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 7 deletions.
diff --git a/justfile b/justfile
@@ -90,10 +90,17 @@ alias sqlx := db-compile
 # Database: Load CSV data produced by importer-offline
 db-load-csv:
     echo "" > data/psql.txt
-    echo "truncate transactions;" >> data/psql.txt
-    echo "truncate logs;" >> data/psql.txt
-    ls -tr1 data/transactions-*.csv | xargs -I{} printf "\\\\copy transactions from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
-    ls -tr1 data/logs-*.csv         | xargs -I{} printf "\\\\copy logs         from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
+
+    echo "truncate historical_balances;" >> data/psql.txt
+    echo "truncate historical_slots;"    >> data/psql.txt
+    echo "truncate transactions;"        >> data/psql.txt
+    echo "truncate logs;"                >> data/psql.txt
+
+    ls -tr1 data/historical_balances-*.csv | xargs -I{} printf "\\\\copy historical_balances from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
+    ls -tr1 data/historical_slots-*.csv    | xargs -I{} printf "\\\\copy historical_slots    from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
+    ls -tr1 data/transactions-*.csv        | xargs -I{} printf "\\\\copy transactions        from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
+    ls -tr1 data/logs-*.csv                | xargs -I{} printf "\\\\copy logs                from '$(pwd)/%s' delimiter E'\\\\t' csv header;\n" "{}" >> data/psql.txt
+
     cat data/psql.txt | pgcli -h localhost -u postgres -d stratus --less-chatty
 
 # ------------------------------------------------------------------------------
@@ -384,7 +391,7 @@ contracts-test-stratus-postgres:
 
 # Contracts: run contract integration tests
 contracts-test-int:
-    #!/bin/bash 
+    #!/bin/bash
     cd e2e-contracts && ./flatten-contracts.sh
     [ -d integration ] && cd integration
     [ ! -f hardhat.config.ts ] && { cp ../../e2e/hardhat.config.ts .; }

diff --git a/src/eth/storage/csv/csv_exporter.rs b/src/eth/storage/csv/csv_exporter.rs
@@ -35,6 +35,10 @@ const ACCOUNTS_HEADERS: [&str; 10] = [
 
 const HISTORICAL_BALANCES_FILE: &str = "data/historical_balances";
 
+const HISTORICAL_SLOTS_FILE: &str = "data/historical_slots";
+
+const HISTORICAL_SLOTS_HEADERS: [&str; 7] = ["id", "idx", "value", "block_number", "account_address", "created_at", "updated_at"];
+
 const HISTORICAL_BALANCES_HEADERS: [&str; 6] = ["id", "address", "balance", "block_number", "created_at", "updated_at"];
 
 const TRANSACTIONS_FILE: &str = "data/transactions";
@@ -89,6 +93,9 @@ pub struct CsvExporter {
     accounts_csv: csv::Writer<File>,
     accounts_id: LastId,
 
+    historical_slots_csv: csv::Writer<File>,
+    historical_slots_id: LastId,
+
     historical_balances_csv: csv::Writer<File>,
     historical_balances_id: LastId,
 
@@ -109,6 +116,9 @@ impl CsvExporter {
             accounts_csv: csv_writer(ACCOUNTS_FILE, BlockNumber::ZERO, &ACCOUNTS_HEADERS)?,
             accounts_id: LastId::new_zero(ACCOUNTS_FILE),
 
+            historical_slots_csv: csv_writer(HISTORICAL_SLOTS_FILE, number, &HISTORICAL_SLOTS_HEADERS)?,
+            historical_slots_id: LastId::new(HISTORICAL_SLOTS_FILE)?,
+
             historical_balances_csv: csv_writer(HISTORICAL_BALANCES_FILE, number, &HISTORICAL_BALANCES_HEADERS)?,
             historical_balances_id: LastId::new(HISTORICAL_BALANCES_FILE)?,
 
@@ -152,6 +162,9 @@ impl CsvExporter {
         }
 
         // flush pending data
+        self.historical_slots_csv.flush()?;
+        self.historical_slots_id.save()?;
+
         self.historical_balances_csv.flush()?;
         self.historical_balances_id.save()?;
 
@@ -223,9 +236,12 @@ impl CsvExporter {
 
     fn export_account_changes(&mut self, changes: Vec<ExecutionAccountChanges>, block_number: &BlockNumber) -> anyhow::Result<()> {
         for change in changes {
+            // historical_nonces
             if let Some(_nonce) = change.nonce.take_modified() {
                 // todo: export historical nonce
             }
+
+            // historical_balances
             if let Some(balance) = change.balance.take_modified() {
                 let now = now();
                 self.historical_balances_id.value += 1;
@@ -241,8 +257,23 @@ impl CsvExporter {
                     .write_record(row)
                     .context("failed to write csv historical balances")?;
             }
-            if let Some(_bytecode) = change.bytecode.take_modified() {
-                // todo: export historical bytecode
+
+            // historical_slots
+            for slot in change.slots.into_values() {
+                if let Some(slot) = slot.take_modified() {
+                    let now = now();
+                    self.historical_slots_id.value += 1;
+                    let row = [
+                        self.historical_slots_id.value.to_string(), // id
+                        slot.index.to_string(),                     // idx
+                        slot.value.to_string(),                     // value
+                        block_number.to_string(),                   // block_number
+                        change.address.to_string(),                 // account_address
+                        now.clone(),                                // updated_at
+                        now,                                        // created_at
+                    ];
+                    self.historical_slots_csv.write_record(row).context("failed to write csv historical slots")?;
+                }
             }
         }
         Ok(())