From f570a67ef3988306dde4e32f81db07349466502f Mon Sep 17 00:00:00 2001 From: Daniel Freire Date: Fri, 13 Dec 2024 11:46:09 -0300 Subject: [PATCH] chore: remove redis permanent storage --- README.md | 1 - src/eth/storage/permanent/mod.rs | 14 - src/eth/storage/permanent/redis.rs | 402 ----------------------------- 3 files changed, 417 deletions(-) delete mode 100644 src/eth/storage/permanent/redis.rs diff --git a/README.md b/README.md index 77ef35624..3760bb5aa 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ Welcome to Stratus, the cutting-edge EVM executor and JSON-RPC server with custo ## Current storage implementations - In Memory -- Redis - RocksDB (default) ## Performance Landscape diff --git a/src/eth/storage/permanent/mod.rs b/src/eth/storage/permanent/mod.rs index d33c91cc9..9ad3a5dd0 100644 --- a/src/eth/storage/permanent/mod.rs +++ b/src/eth/storage/permanent/mod.rs @@ -1,10 +1,8 @@ pub use self::inmemory::InMemoryPermanentStorage; -pub use self::redis::RedisPermanentStorage; pub use self::rocks::RocksPermanentStorage; pub use self::rocks::RocksStorageState; mod inmemory; -mod redis; pub mod rocks; use std::str::FromStr; @@ -27,7 +25,6 @@ use crate::eth::primitives::Slot; use crate::eth::primitives::SlotIndex; use crate::eth::primitives::TransactionMined; use crate::ext::parse_duration; -use crate::log_and_err; /// Permanent (committed) storage operations. pub trait PermanentStorage: Send + Sync + 'static { @@ -121,9 +118,6 @@ pub enum PermanentStorageKind { #[serde(rename = "inmemory")] InMemory, - #[serde(rename = "redis")] - Redis, - #[serde(rename = "rocks")] Rocks, } @@ -136,13 +130,6 @@ impl PermanentStorageConfig { let perm: Box = match self.perm_storage_kind { PermanentStorageKind::InMemory => Box::::default(), - PermanentStorageKind::Redis => { - let Some(url) = self.perm_storage_url.as_deref() else { - return log_and_err!("redis connection url not provided when it was expected to be present"); - }; - Box::new(RedisPermanentStorage::new(url)?) - } - PermanentStorageKind::Rocks => Box::new(RocksPermanentStorage::new( self.rocks_path_prefix.clone(), self.rocks_shutdown_timeout, @@ -160,7 +147,6 @@ impl FromStr for PermanentStorageKind { fn from_str(s: &str) -> anyhow::Result { match s { "inmemory" => Ok(Self::InMemory), - "redis" => Ok(Self::Redis), "rocks" => Ok(Self::Rocks), s => Err(anyhow!("unknown permanent storage: {}", s)), } diff --git a/src/eth/storage/permanent/redis.rs b/src/eth/storage/permanent/redis.rs deleted file mode 100644 index 8b6df0f4f..000000000 --- a/src/eth/storage/permanent/redis.rs +++ /dev/null @@ -1,402 +0,0 @@ -use itertools::Itertools; -use redis::Client as RedisClient; -use redis::Commands; -use redis::Connection as RedisConnection; -use redis::RedisResult; - -use crate::eth::primitives::Account; -use crate::eth::primitives::Address; -use crate::eth::primitives::Block; -use crate::eth::primitives::BlockFilter; -use crate::eth::primitives::BlockNumber; -use crate::eth::primitives::Hash; -use crate::eth::primitives::LogFilter; -use crate::eth::primitives::LogMined; -use crate::eth::primitives::PointInTime; -use crate::eth::primitives::Slot; -use crate::eth::primitives::SlotIndex; -use crate::eth::primitives::TransactionMined; -use crate::eth::storage::PermanentStorage; -use crate::ext::from_json_str; -use crate::ext::to_json_object; -use crate::ext::to_json_string; -use crate::ext::to_json_value; -use crate::log_and_err; - -type RedisVecOptString = RedisResult>>; -type RedisVecString = RedisResult>; -type RedisOptString = RedisResult>; -type RedisOptUsize = RedisResult>; -type RedisVoid = RedisResult<()>; - -pub struct RedisPermanentStorage { - client: redis::Client, -} - -impl RedisPermanentStorage { - pub fn new(url: &str) -> anyhow::Result { - let client = match RedisClient::open(url) { - Ok(client) => client, - Err(e) => return log_and_err!(reason = e, "failed to create redis client"), - }; - Ok(Self { client }) - } - - fn conn(&self) -> anyhow::Result { - match self.client.get_connection() { - Ok(conn) => Ok(conn), - Err(e) => log_and_err!(reason = e, "failed to get redis connection"), - } - } -} - -impl PermanentStorage for RedisPermanentStorage { - fn set_mined_block_number(&self, number: BlockNumber) -> anyhow::Result<()> { - // execute command - let mut conn = self.conn()?; - let set: RedisVoid = conn.set("number::mined", number.to_string()); - - // parse - match set { - Ok(_) => Ok(()), - Err(e) => log_and_err!(reason = e, "failed to write mined number to redis"), - } - } - - fn read_mined_block_number(&self) -> anyhow::Result { - // execute command - let mut conn = self.conn()?; - let value: RedisOptUsize = conn.get("number::mined"); - - // parse - match value { - Ok(Some(value)) => Ok(value.into()), - Ok(None) => Ok(BlockNumber::ZERO), - Err(e) => log_and_err!(reason = e, "failed to read miner block number from redis"), - } - } - - fn save_block(&self, block: Block) -> anyhow::Result<()> { - // generate block keys - let key_block_number = key_block_by_number(block.number()); - let key_block_hash = key_block_by_hash(block.hash()); - - // generate values - let block_json = to_json_string(&block); - - // blocks - let mut mset_values = vec![ - (key_block_number, block_json.clone()), - (key_block_hash, block_json.clone()), - ("block::latest".to_owned(), block_json), - ]; - let mut zadd_values = vec![]; - - // transactions - for tx in &block.transactions { - let tx_key = key_tx(tx.input.hash); - let tx_value = to_json_string(&tx); - mset_values.push((tx_key, tx_value)); - } - - // changes - for changes in block.compact_account_changes() { - // account - if changes.is_account_modified() { - let mut account = Account { - address: changes.address, - ..Account::default() - }; - if let Some(nonce) = changes.nonce.take() { - account.nonce = nonce; - } - if let Some(balance) = changes.balance.take() { - account.balance = balance; - } - if let Some(bytecode) = changes.bytecode.take() { - account.bytecode = bytecode; - } - - // add block number to force slot modification - let mut account_value = to_json_object(&account); - account_value.insert("block".to_owned(), to_json_value(block.number())); - let account_value = to_json_string(&account_value); - - mset_values.push((key_account(account.address), account_value.clone())); - zadd_values.push((key_account_history(account.address), account_value, block.number().as_u64())); - } - - // slots - for slot in changes.slots.into_values() { - if let Some(slot) = slot.take() { - // add block number to force slot modification - let mut slot_value = to_json_value(slot); - slot_value.as_object_mut().unwrap().insert("block".to_owned(), to_json_value(block.number())); - let slot_value = to_json_string(&slot_value); - - mset_values.push((key_slot(changes.address, slot.index), slot_value.clone())); - zadd_values.push((key_slot_history(changes.address, slot.index), slot_value, block.number().as_u64())); - } - } - } - - // execute mset command - let mut conn = self.conn()?; - let set: RedisVoid = conn.mset(&mset_values); - if let Err(e) = set { - return log_and_err!(reason = e, "failed to write block mset to redis"); - } - - // execute zadd commands - for (key, value, score) in zadd_values { - let mut cmd = redis::cmd("ZADD"); - cmd.arg(key).arg("NX").arg(score).arg(value); - - let zadd: RedisVoid = cmd.exec(&mut conn); - if let Err(e) = zadd { - return log_and_err!(reason = e, "failed to write block zadd to redis"); - } - } - - Ok(()) - } - - fn read_block(&self, block_filter: BlockFilter) -> anyhow::Result> { - // prepare keys - let block_key = match block_filter { - BlockFilter::Latest | BlockFilter::Pending => "block::latest".to_owned(), - BlockFilter::Earliest => "block::earliest".to_owned(), - BlockFilter::Hash(hash) => key_block_by_hash(hash), - BlockFilter::Number(number) => key_block_by_number(number), - }; - - // execute command - let mut conn = self.conn()?; - let redis_block: RedisOptString = conn.get(block_key); - - // parse - match redis_block { - Ok(Some(json)) => Ok(from_json_str(&json)), - Ok(None) => Ok(None), - Err(e) => log_and_err!(reason = e, "failed to read block from redis"), - } - } - - fn read_transaction(&self, hash: Hash) -> anyhow::Result> { - // prepare keys - let tx_key = key_tx(hash); - - // execute command - let mut conn = self.conn()?; - let redis_transaction: RedisOptString = conn.get(tx_key); - - // parse - match redis_transaction { - Ok(Some(json)) => Ok(from_json_str(&json)), - Ok(None) => Ok(None), - Err(e) => log_and_err!(reason = e, "failed to read transaction from redis"), - } - } - - fn read_logs(&self, filter: &LogFilter) -> anyhow::Result> { - // prepare keys - let from_block = filter.from_block.as_u64(); - let to_block = match filter.to_block { - Some(number) => number.as_u64(), - None => self.read_mined_block_number()?.as_u64(), - }; - let block_keys = (from_block..=to_block).map(key_block_by_number).collect_vec(); - - // exit if no keys - if block_keys.is_empty() { - return Ok(vec![]); - } - - // execute command - let mut conn = self.conn()?; - let blocks: RedisVecOptString = conn.mget(block_keys); - - // parse - let blocks: Vec = match blocks { - Ok(vec_json) => vec_json.into_iter().flatten().map(|json| from_json_str(&json)).collect_vec(), - Err(e) => return log_and_err!(reason = e, "failed to read logs from redis"), - }; - - // filter - let logs = blocks - .into_iter() - .flat_map(|b| b.transactions) - .flat_map(|t| t.logs) - .filter(|log| filter.matches(log)) - .collect_vec(); - - Ok(logs) - } - - fn save_accounts(&self, accounts: Vec) -> anyhow::Result<()> { - // exit if no accounts - if accounts.is_empty() { - return Ok(()); - } - - // prepare values - let redis_accounts = accounts - .into_iter() - .map(|acc| { - let account_key = key_account(acc.address); - let account_value = to_json_string(&acc); - (account_key, account_value) - }) - .collect_vec(); - - // execute command - let mut conn = self.conn()?; - let set: RedisVoid = conn.mset(&redis_accounts); - - // parse - match set { - Ok(_) => Ok(()), - Err(e) => log_and_err!(reason = e, "failed to write accounts to redis"), - } - } - - fn read_account(&self, address: Address, point_in_time: PointInTime) -> anyhow::Result> { - let mut conn = self.conn()?; - match point_in_time { - PointInTime::Mined | PointInTime::Pending => { - // prepare key - let account_key = key_account(address); - - // execute - let redis_account: RedisOptString = conn.get(account_key); - - // parse - match redis_account { - Ok(Some(json)) => Ok(Some(from_json_str(&json))), - Ok(None) => Ok(None), - Err(e) => log_and_err!(reason = e, "failed to read account from redis current value"), - } - } - PointInTime::MinedPast(number) => { - // prepare key - let account_key = key_account_history(address); - - // execute - let mut cmd = redis::cmd("ZRANGE"); - cmd.arg(account_key) - .arg(number.as_u64()) - .arg(0) - .arg("BYSCORE") - .arg("REV") - .arg("LIMIT") - .arg(0) - .arg(1); - let redis_account: RedisVecString = cmd.query(&mut conn); - - // parse - match redis_account { - Ok(vec_json) => match vec_json.first() { - Some(json) => Ok(Some(from_json_str(json))), - None => Ok(None), - }, - Err(e) => log_and_err!(reason = e, "failed to read account from redis historical value"), - } - } - } - } - - fn read_slot(&self, address: Address, index: SlotIndex, point_in_time: PointInTime) -> anyhow::Result> { - // execute command and parse - let mut conn = self.conn()?; - match point_in_time { - PointInTime::Mined | PointInTime::Pending => { - // prepare key - let slot_key = key_slot(address, index); - - // execute - let redis_slot: RedisOptString = conn.get(slot_key); - - // parse - match redis_slot { - Ok(Some(json)) => Ok(Some(from_json_str(&json))), - Ok(None) => Ok(None), - Err(e) => log_and_err!(reason = e, "failed to read slot from redis current value"), - } - } - PointInTime::MinedPast(number) => { - // prepare key - let slot_key = key_slot_history(address, index); - - // execute - let mut cmd = redis::cmd("ZRANGE"); - cmd.arg(slot_key) - .arg(number.as_u64()) - .arg(0) - .arg("BYSCORE") - .arg("REV") - .arg("LIMIT") - .arg(0) - .arg(1); - let redis_account: RedisVecString = cmd.query(&mut conn); - - // parse - match redis_account { - Ok(vec_json) => match vec_json.first() { - Some(json) => Ok(Some(from_json_str(json))), - None => Ok(None), - }, - Err(e) => log_and_err!(reason = e, "failed to read account from redis historical value"), - } - } - } - } - - #[cfg(feature = "dev")] - fn reset(&self) -> anyhow::Result<()> { - let mut conn = self.conn()?; - let flush: RedisVoid = redis::cmd("FLUSHDB").exec(&mut conn); - match flush { - Ok(_) => Ok(()), - Err(e) => log_and_err!(reason = e, "failed to clear all redis keys"), - } - } -} - -// ----------------------------------------------------------------------------- -// Keys helpers -// ----------------------------------------------------------------------------- - -/// Generates a key for accessing a block by number. -fn key_block_by_number(number: impl Into) -> String { - format!("block::number::{}", number.into()) -} - -/// Generates a key for accessing a block by hash. -fn key_block_by_hash(hash: Hash) -> String { - format!("block::hash::{}", hash) -} - -/// Generates a key for accessing an account. -fn key_account(address: Address) -> String { - format!("account::{}", address) -} - -/// Generates a key for accessing an account history. -fn key_account_history(address: Address) -> String { - format!("account_history::{}", address) -} - -/// Generates a key for accessing a slot. -fn key_slot(address: Address, index: SlotIndex) -> String { - format!("slot::{}::{}", address, index) -} - -/// Generates a key for accessing a slot history. -fn key_slot_history(address: Address, index: SlotIndex) -> String { - format!("slot_history::{}::{}", address, index) -} - -/// Generates a key for accessing a transaction. -fn key_tx(hash: Hash) -> String { - format!("tx::{}", hash) -}