From 5408f3f8709128951855b8a00172bd5fde78cb31 Mon Sep 17 00:00:00 2001 From: Jonathan <94441036+zeapoz@users.noreply.github.com> Date: Thu, 16 Nov 2023 11:36:21 +0100 Subject: [PATCH 1/4] feat: initial export snapshot skeleton (#40) * feat: initial export snapshot template * doc: clarify miniblocks --- Cargo.lock | 1 + Cargo.toml | 1 + src/cli.rs | 8 ++ src/main.rs | 25 ++++ src/processor/mod.rs | 3 +- src/processor/snapshot/mod.rs | 238 ++++++++++++++++++++++++++++++++ src/processor/snapshot/types.rs | 74 ++++++++++ 7 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 src/processor/snapshot/mod.rs create mode 100644 src/processor/snapshot/types.rs diff --git a/Cargo.lock b/Cargo.lock index 1ec3b40..4158af8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4631,6 +4631,7 @@ version = "0.1.0" dependencies = [ "async-trait", "blake2 0.10.6", + "chrono", "clap 4.4.7", "ethers", "eyre", diff --git a/Cargo.toml b/Cargo.toml index 2be4481..d4ed50c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = ["state-reconstruct-fetcher"] [dependencies] async-trait = "0.1.74" blake2 = "0.10.6" +chrono = "0.4.31" clap = { version = "4.4.7", features = ["derive", "env"] } ethers = "1.0.2" eyre = "0.6.8" diff --git a/src/cli.rs b/src/cli.rs index a049726..0935bb3 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -72,6 +72,14 @@ pub enum Command { #[arg(short, long, env = "ZK_SYNC_DB_PATH")] db_path: Option, }, + + /// Testing. + ExportSnapshot { + #[command(flatten)] + l1_fetcher_options: L1FetcherOptions, + /// The path of the file to export the snapshot to. + file: Option, + }, } #[derive(Parser)] diff --git a/src/main.rs b/src/main.rs index 024152e..c0eae2b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,7 @@ use std::{ use clap::Parser; use cli::{Cli, Command, ReconstructSource}; use eyre::Result; +use processor::snapshot::SnapshotExporter; use state_reconstruct_fetcher::{ constants::storage, l1_fetcher::{L1Fetcher, L1FetcherOptions}, @@ -52,6 +53,7 @@ fn start_logger(default_level: LevelFilter) { } #[tokio::main] +#[allow(clippy::too_many_lines)] async fn main() -> Result<()> { start_logger(LevelFilter::INFO); @@ -150,6 +152,29 @@ async fn main() -> Result<()> { println!("{result}"); } } + Command::ExportSnapshot { + l1_fetcher_options, + file, + } => { + let fetcher_options = L1FetcherOptions { + http_url: l1_fetcher_options.http_url, + start_block: l1_fetcher_options.start_block, + block_step: l1_fetcher_options.block_step, + block_count: l1_fetcher_options.block_count, + disable_polling: l1_fetcher_options.disable_polling, + }; + + let fetcher = L1Fetcher::new(fetcher_options, None)?; + let processor = SnapshotExporter::new(file); + + let (tx, rx) = mpsc::channel::(5); + let processor_handle = tokio::spawn(async move { + processor.run(rx).await; + }); + + fetcher.run(tx).await?; + processor_handle.await?; + } } Ok(()) diff --git a/src/processor/mod.rs b/src/processor/mod.rs index 316d05d..e88ffae 100644 --- a/src/processor/mod.rs +++ b/src/processor/mod.rs @@ -3,9 +3,10 @@ use state_reconstruct_fetcher::types::CommitBlockInfoV1; use tokio::sync::mpsc; pub mod json; +pub mod snapshot; pub mod tree; #[async_trait] pub trait Processor { - async fn run(self, rx: mpsc::Receiver); + async fn run(self, mut rx: mpsc::Receiver); } diff --git a/src/processor/snapshot/mod.rs b/src/processor/snapshot/mod.rs new file mode 100644 index 0000000..03f3a53 --- /dev/null +++ b/src/processor/snapshot/mod.rs @@ -0,0 +1,238 @@ +use std::{collections::HashMap, fmt, fs, path::PathBuf, str::FromStr}; + +mod types; + +use async_trait::async_trait; +use blake2::{Blake2s256, Digest}; +use ethers::types::{Address, H256, U256, U64}; +use eyre::Result; +use indexmap::IndexSet; +use state_reconstruct_fetcher::{ + constants::{ethereum, storage}, + types::CommitBlockInfoV1, +}; +use tokio::sync::mpsc; + +use self::types::{SnapshotStorageLog, StorageKey, StorageValue}; +use super::Processor; +use crate::processor::snapshot::types::MiniblockNumber; + +// NOTE: What file extension to use? +const DEFAULT_EXPORT_PATH: &str = "snapshot_export"; + +pub struct SnapshotExporter { + storage_log_entries: HashMap, + index_to_key_map: IndexSet, + path: PathBuf, +} + +impl SnapshotExporter { + pub fn new(path: Option) -> Self { + let path = match path { + Some(p) => PathBuf::from(p), + None => PathBuf::from(DEFAULT_EXPORT_PATH), + }; + + let mut index_to_key_map = IndexSet::new(); + let mut storage_log_entries = HashMap::new(); + + reconstruct_genesis_state( + &mut storage_log_entries, + &mut index_to_key_map, + storage::INITAL_STATE_PATH, + ) + .unwrap(); + + Self { + storage_log_entries, + index_to_key_map, + path, + } + } +} + +impl fmt::Display for SnapshotExporter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut s = String::new(); + + for entry in &self.storage_log_entries { + s.push_str(&entry.1.to_string()); + s.push('\n'); + } + + write!(f, "{s}") + } +} + +#[async_trait] +impl Processor for SnapshotExporter { + async fn run(mut self, mut rx: mpsc::Receiver) { + // TODO: Send from fetcher. + let l1_block_number = U64::from(0); + + while let Some(block) = rx.recv().await { + // Initial calldata. + for (key, value) in &block.initial_storage_changes { + let key = U256::from_little_endian(key); + let value = H256::from(value); + self.index_to_key_map.insert(key); + + let log = self + .storage_log_entries + .entry(key) + .or_insert(SnapshotStorageLog { + key, + value: StorageValue::default(), + // NOTE: This isn't stored in L1, can we procure it some other way? + miniblock_number_of_initial_write: U64::from(0), + l1_batch_number_of_initial_write: l1_block_number, + enumeration_index: 0, + }); + log.value = value; + } + + // Repeated calldata. + for (index, value) in &block.repeated_storage_changes { + let index = usize::try_from(*index).expect("truncation failed"); + // Index is 1-based so we subtract 1. + let key = *self.index_to_key_map.get_index(index - 1).unwrap(); + let value = H256::from(value); + + self.storage_log_entries + .entry(key) + .and_modify(|log| log.value = value); + } + + // TODO: We need to index these by hash. + // Factory dependencies. + // for dep in &block.factory_deps {} + } + + fs::write(&self.path, self.to_string()).expect("failed to export snapshot"); + tracing::info!("Successfully exported snapshot to {}", self.path.display()); + } +} + +// TODO: Can this be made somewhat generic? +/// Attempts to reconstruct the genesis state from a CSV file. +fn reconstruct_genesis_state( + storage_log_entries: &mut HashMap, + index_to_key: &mut IndexSet, + path: &str, +) -> Result<()> { + fn cleanup_encoding(input: &'_ str) -> &'_ str { + input + .strip_prefix("E'\\\\x") + .unwrap() + .strip_suffix('\'') + .unwrap() + } + + let mut block_batched_accesses = vec![]; + + let input = fs::read_to_string(path)?; + for line in input.lines() { + let mut separated = line.split(','); + let _derived_key = separated.next().unwrap(); + let address = separated.next().unwrap(); + let key = separated.next().unwrap(); + let value = separated.next().unwrap(); + let op_number: u32 = separated.next().unwrap().parse()?; + let _ = separated.next().unwrap(); + let miniblock_number: u32 = separated.next().unwrap().parse()?; + + if miniblock_number != 0 { + break; + } + + let address = Address::from_str(cleanup_encoding(address))?; + let key = U256::from_str_radix(cleanup_encoding(key), 16)?; + let value = U256::from_str_radix(cleanup_encoding(value), 16)?; + + let record = (address, key, value, op_number, miniblock_number); + block_batched_accesses.push(record); + } + + // Sort in block block. + block_batched_accesses.sort_by(|a, b| match a.0.cmp(&b.0) { + std::cmp::Ordering::Equal => match a.1.cmp(&b.1) { + std::cmp::Ordering::Equal => match a.3.cmp(&b.3) { + std::cmp::Ordering::Equal => { + panic!("must be unique") + } + a => a, + }, + a => a, + }, + a => a, + }); + + let mut key_set = std::collections::HashSet::new(); + + // Batch. + for el in &block_batched_accesses { + let derived_key = derive_final_address_for_params(&el.0, &el.1); + key_set.insert(derived_key); + } + + let mut batched = vec![]; + let mut it = block_batched_accesses.into_iter(); + let mut previous = it.next().unwrap(); + for el in it { + if el.0 != previous.0 || el.1 != previous.1 { + batched.push((previous.0, previous.1, previous.2, previous.4)); + } + + previous = el; + } + + // Finalize. + batched.push((previous.0, previous.1, previous.2, previous.4)); + + tracing::trace!("Have {} unique keys in the tree", key_set.len()); + + for (address, key, value, miniblock_number) in batched { + let derived_key = derive_final_address_for_params(&address, &key); + // TODO: what to do here? + // let version = tree.latest_version().unwrap_or_default(); + // let _leaf = tree.read_leaves(version, &[key]); + + // let existing_value = U256::from_big_endian(existing_leaf.leaf.value()); + // if existing_value == value { + // // we downgrade to read + // // println!("Downgrading to read") + // } else { + // we write + let mut tmp = [0u8; 32]; + value.to_big_endian(&mut tmp); + + let key = U256::from_little_endian(&derived_key); + let value = H256::from(tmp); + + let log = storage_log_entries + .entry(key) + .or_insert(SnapshotStorageLog { + key, + value: StorageValue::default(), + miniblock_number_of_initial_write: MiniblockNumber::from(miniblock_number), + l1_batch_number_of_initial_write: U64::from(ethereum::GENESIS_BLOCK), + enumeration_index: 0, + }); + + log.value = value; + index_to_key.insert(key); + } + + Ok(()) +} + +fn derive_final_address_for_params(address: &Address, key: &U256) -> [u8; 32] { + let mut buffer = [0u8; 64]; + buffer[12..32].copy_from_slice(&address.0); + key.to_big_endian(&mut buffer[32..64]); + + let mut result = [0u8; 32]; + result.copy_from_slice(Blake2s256::digest(buffer).as_slice()); + + result +} diff --git a/src/processor/snapshot/types.rs b/src/processor/snapshot/types.rs new file mode 100644 index 0000000..bda619d --- /dev/null +++ b/src/processor/snapshot/types.rs @@ -0,0 +1,74 @@ +// FIXME: +#![allow(dead_code)] +use std::fmt; + +use chrono::{offset::Utc, DateTime}; +use ethers::types::{H256, U256, U64}; + +pub type L1BatchNumber = U64; +pub type MiniblockNumber = U64; + +pub type StorageKey = U256; +pub type StorageValue = H256; + +#[derive(Default, Debug)] +pub struct SnapshotHeader { + pub l1_batch_number: L1BatchNumber, + pub miniblock_number: MiniblockNumber, + /// Chunk metadata ordered by chunk_id + pub chunks: Vec, + // TODO: + // pub last_l1_batch_with_metadata: L1BatchWithMetadata, + pub generated_at: DateTime, +} + +#[derive(Default, Debug)] +pub struct SnapshotChunkMetadata { + pub key: SnapshotStorageKey, + /// Can be either a gs or filesystem path + pub filepath: String, +} + +#[derive(Default, Debug)] +pub struct SnapshotStorageKey { + pub l1_batch_number: L1BatchNumber, + /// Chunks with smaller id's must contain storage_logs with smaller hashed_keys + pub chunk_id: u64, +} + +#[derive(Default, Debug)] +pub struct SnapshotChunk { + // Sorted by hashed_keys interpreted as little-endian numbers + pub storage_logs: Vec, + pub factory_deps: Vec, +} + +// "most recent" for each key together with info when the key was first used +#[derive(Default, Debug)] +pub struct SnapshotStorageLog { + pub key: StorageKey, + pub value: StorageValue, + pub miniblock_number_of_initial_write: MiniblockNumber, + pub l1_batch_number_of_initial_write: L1BatchNumber, + pub enumeration_index: u64, +} + +impl fmt::Display for SnapshotStorageLog { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{},{},{},{},{}", + self.key, + hex::encode(self.value), + self.miniblock_number_of_initial_write, + self.l1_batch_number_of_initial_write, + self.enumeration_index + ) + } +} + +#[derive(Default, Debug)] +pub struct SnapshotFactoryDependency { + pub bytecode_hash: H256, + pub bytecode: Vec, +} From 48c70295d46cf0a3da194813dece0f8776bf688a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuomas=20M=C3=A4kinen?= <1947505+tuommaki@users.noreply.github.com> Date: Thu, 16 Nov 2023 16:16:56 +0200 Subject: [PATCH 2/4] Bundle L1 block# in `CommitBlockInfoV1` (#41) When exporting snapshot files from L1, each snapshot needs information where each storage log entry originated from, i.e. the L1 block number for given storage log entry. It's added to `CommitBlockInfoV1` as an optional field which is not serialized into JSON. --- state-reconstruct-fetcher/src/l1_fetcher.rs | 19 +++++++++++++------ state-reconstruct-fetcher/src/types.rs | 4 ++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/state-reconstruct-fetcher/src/l1_fetcher.rs b/state-reconstruct-fetcher/src/l1_fetcher.rs index 304f5da..7ca3293 100644 --- a/state-reconstruct-fetcher/src/l1_fetcher.rs +++ b/state-reconstruct-fetcher/src/l1_fetcher.rs @@ -295,7 +295,7 @@ impl L1Fetcher { fn spawn_tx_handler( &self, mut hash_rx: mpsc::Receiver, - calldata_tx: mpsc::Sender, + l1_tx_tx: mpsc::Sender, mut last_block: u64, ) -> tokio::task::JoinHandle<()> { let metrics = self.metrics.clone(); @@ -336,7 +336,7 @@ impl L1Fetcher { } } - calldata_tx.send(tx.input).await.unwrap(); + l1_tx_tx.send(tx).await.unwrap(); } } }) @@ -344,7 +344,7 @@ impl L1Fetcher { fn spawn_parsing_handler( &self, - mut calldata_rx: mpsc::Receiver, + mut l1_tx_rx: mpsc::Receiver, sink: mpsc::Sender, ) -> Result> { let metrics = self.metrics.clone(); @@ -352,8 +352,9 @@ impl L1Fetcher { Ok(tokio::spawn({ async move { - while let Some(calldata) = calldata_rx.recv().await { - let blocks = match parse_calldata(&function, &calldata) { + while let Some(tx) = l1_tx_rx.recv().await { + let block_number = tx.block_number.map(|v| v.as_u64()); + let blocks = match parse_calldata(block_number, &function, &tx.input) { Ok(blks) => blks, Err(e) => { tracing::error!("failed to parse calldata: {e}"); @@ -391,6 +392,7 @@ impl L1Fetcher { } pub fn parse_calldata( + l1_block_number: Option, commit_blocks_fn: &Function, calldata: &[u8], ) -> Result> { @@ -435,7 +437,12 @@ pub fn parse_calldata( // TODO: What to do here? // assert_eq!(previous_enumeration_index, tree.next_enumeration_index()); - parse_commit_block_info(&new_blocks_data) + // Supplement every CommitBlockInfoV1 element with L1 block number information. + parse_commit_block_info(&new_blocks_data).map(|mut vec| { + vec.iter_mut() + .for_each(|e| e.l1_block_number = l1_block_number); + vec + }) } fn parse_commit_block_info(data: &abi::Token) -> Result> { diff --git a/state-reconstruct-fetcher/src/types.rs b/state-reconstruct-fetcher/src/types.rs index af679e2..fed6721 100644 --- a/state-reconstruct-fetcher/src/types.rs +++ b/state-reconstruct-fetcher/src/types.rs @@ -27,6 +27,9 @@ pub enum ParseError { /// Data needed to commit new block #[derive(Debug, Serialize, Deserialize)] pub struct CommitBlockInfoV1 { + /// L1 block number. + #[serde(skip_serializing)] + pub l1_block_number: Option, /// L2 block number. pub block_number: u64, /// Unix timestamp denoting the start of the block execution. @@ -98,6 +101,7 @@ impl TryFrom<&abi::Token> for CommitBlockInfoV1 { ); let mut blk = CommitBlockInfoV1 { + l1_block_number: None, block_number: new_l2_block_number.as_u64(), timestamp: timestamp.as_u64(), index_repeated_storage_changes: new_enumeration_index, From 39b5bf124d24b46eeafb5b104574e268daddff73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuomas=20M=C3=A4kinen?= <1947505+tuommaki@users.noreply.github.com> Date: Fri, 17 Nov 2023 08:49:08 +0200 Subject: [PATCH 3/4] Hash & gather factory deps to snapshot (#42) Factory deps (smart contracts in zkSync) are stored separately in snapshots, but they must be gathered together with storage log entries when fetching the state from L1. This change will do preliminary work required for writing out the separate factory dep snapshot file. --- Cargo.lock | 98 +++++++++++++++++++++--------- Cargo.toml | 1 + src/processor/snapshot/bytecode.rs | 26 ++++++++ src/processor/snapshot/mod.rs | 13 +++- 4 files changed, 107 insertions(+), 31 deletions(-) create mode 100644 src/processor/snapshot/bytecode.rs diff --git a/Cargo.lock b/Cargo.lock index 4158af8..724c247 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -517,6 +517,14 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "blake2" +version = "0.10.6" +source = "git+https://github.com/RustCrypto/hashes.git?rev=1f727ce37ff40fa0cce84eb8543a45bdd3ca4a4e#1f727ce37ff40fa0cce84eb8543a45bdd3ca4a4e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "blake2-rfc_bellman_edition" version = "0.0.1" @@ -875,7 +883,7 @@ dependencies = [ "k256", "lazy_static", "serde", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror", ] @@ -892,7 +900,7 @@ dependencies = [ "hmac 0.12.1", "pbkdf2 0.11.0", "rand 0.8.5", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror", ] @@ -905,15 +913,15 @@ dependencies = [ "base58check", "base64 0.12.3", "bech32", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "digest 0.10.7", "generic-array 0.14.7", "hex", "ripemd", "serde", "serde_derive", - "sha2 0.10.6", - "sha3 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror", ] @@ -1441,8 +1449,8 @@ dependencies = [ "scrypt 0.10.0", "serde", "serde_json", - "sha2 0.10.6", - "sha3 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror", "uuid 0.8.2", ] @@ -1459,7 +1467,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sha3 0.10.6", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror", "uint", ] @@ -1731,7 +1739,7 @@ dependencies = [ "ethers-core", "hex", "rand 0.8.5", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror", ] @@ -2542,8 +2550,8 @@ dependencies = [ "cfg-if 1.0.0", "ecdsa", "elliptic-curve", - "sha2 0.10.6", - "sha3 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3286,7 +3294,7 @@ dependencies = [ "digest 0.10.7", "hmac 0.12.1", "password-hash 0.4.2", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3343,7 +3351,7 @@ checksum = "1df74e9e7ec4053ceb980e7c0c8bd3594e977fde1af91daba9c928e8e8c6708d" dependencies = [ "once_cell", "pest", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3899,7 +3907,7 @@ source = "git+https://github.com/matter-labs/rescue-poseidon#d059b5042df5ed80e15 dependencies = [ "addchain", "arrayvec 0.7.4", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder", "franklin-crypto", "num-bigint 0.3.3", @@ -4163,7 +4171,7 @@ dependencies = [ "hmac 0.12.1", "pbkdf2 0.11.0", "salsa20 0.10.2", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4506,6 +4514,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.10.6" +source = "git+https://github.com/RustCrypto/hashes.git?rev=1731ced4a116d61ba9dc6ee6d0f38fb8102e357a#1731ced4a116d61ba9dc6ee6d0f38fb8102e357a" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.7", +] + [[package]] name = "sha3" version = "0.9.1" @@ -4528,6 +4546,15 @@ dependencies = [ "keccak", ] +[[package]] +name = "sha3" +version = "0.10.6" +source = "git+https://github.com/RustCrypto/hashes.git?rev=7a187e934c1f6c68e4b4e5cf37541b7a0d64d303#7a187e934c1f6c68e4b4e5cf37541b7a0d64d303" +dependencies = [ + "digest 0.10.7", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -4630,7 +4657,7 @@ name = "state-reconstruct" version = "0.1.0" dependencies = [ "async-trait", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "chrono", "clap 4.4.7", "ethers", @@ -4643,6 +4670,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "zkevm_opcode_defs 1.3.2 (git+https://github.com/matter-labs/era-zkevm_opcode_defs.git)", "zksync_merkle_tree", ] @@ -4784,11 +4812,11 @@ dependencies = [ "rand 0.4.6", "rescue_poseidon", "serde", - "sha2 0.10.6", - "sha3 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec", "zk_evm", - "zkevm_opcode_defs", + "zkevm_opcode_defs 1.3.2 (git+https://github.com/matter-labs/era-zkevm_opcode_defs.git?branch=v1.3.2)", ] [[package]] @@ -5665,7 +5693,7 @@ dependencies = [ "serde_json", "static_assertions", "zk_evm_abstractions", - "zkevm_opcode_defs", + "zkevm_opcode_defs 1.3.2 (git+https://github.com/matter-labs/era-zkevm_opcode_defs.git?branch=v1.3.2)", ] [[package]] @@ -5676,7 +5704,7 @@ dependencies = [ "anyhow", "serde", "static_assertions", - "zkevm_opcode_defs", + "zkevm_opcode_defs 1.3.2 (git+https://github.com/matter-labs/era-zkevm_opcode_defs.git?branch=v1.3.2)", ] [[package]] @@ -5691,11 +5719,11 @@ dependencies = [ "nom", "num-bigint 0.4.4", "num-traits", - "sha3 0.10.6", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec", "structopt", "thiserror", - "zkevm_opcode_defs", + "zkevm_opcode_defs 1.3.2 (git+https://github.com/matter-labs/era-zkevm_opcode_defs.git?branch=v1.3.2)", ] [[package]] @@ -5704,12 +5732,26 @@ version = "1.3.2" source = "git+https://github.com/matter-labs/era-zkevm_opcode_defs.git?branch=v1.3.2#c7ab62f4c60b27dfc690c3ab3efb5fff1ded1a25" dependencies = [ "bitflags 2.4.0", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types 0.14.1", + "k256", + "lazy_static", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zkevm_opcode_defs" +version = "1.3.2" +source = "git+https://github.com/matter-labs/era-zkevm_opcode_defs.git#dffacadeccdfdbff4bc124d44c595c4a6eae5013" +dependencies = [ + "bitflags 2.4.0", + "blake2 0.10.6 (git+https://github.com/RustCrypto/hashes.git?rev=1f727ce37ff40fa0cce84eb8543a45bdd3ca4a4e)", "ethereum-types 0.14.1", "k256", "lazy_static", - "sha2 0.10.6", - "sha3 0.10.6", + "sha2 0.10.6 (git+https://github.com/RustCrypto/hashes.git?rev=1731ced4a116d61ba9dc6ee6d0f38fb8102e357a)", + "sha3 0.10.6 (git+https://github.com/RustCrypto/hashes.git?rev=7a187e934c1f6c68e4b4e5cf37541b7a0d64d303)", ] [[package]] @@ -5788,7 +5830,7 @@ version = "0.1.0" source = "git+https://github.com/matter-labs/zksync-era.git#7e231887d3f0585dcccae730bc6b223dd96bd668" dependencies = [ "base64 0.13.1", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "hex", "once_cell", "serde", @@ -5840,7 +5882,7 @@ name = "zksync_types" version = "0.1.0" source = "git+https://github.com/matter-labs/zksync-era.git#7e231887d3f0585dcccae730bc6b223dd96bd668" dependencies = [ - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "chrono", "codegen 0.1.0", "ethereum-types 0.12.1", diff --git a/Cargo.toml b/Cargo.toml index d4ed50c..a1453fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,3 +23,4 @@ tokio = { version = "1.33.0", features = ["macros"] } tracing = "0.1.40" tracing-subscriber = "0.3.17" zksync_merkle_tree = { git = "https://github.com/matter-labs/zksync-era.git" } +zkevm_opcode_defs = { git = "https://github.com/matter-labs/era-zkevm_opcode_defs.git" } diff --git a/src/processor/snapshot/bytecode.rs b/src/processor/snapshot/bytecode.rs new file mode 100644 index 0000000..5919299 --- /dev/null +++ b/src/processor/snapshot/bytecode.rs @@ -0,0 +1,26 @@ +use ethers::types::H256; + +// These are copied from zkSync-era project to avoid pulling in full dependency. +pub fn bytes_to_chunks(bytes: &[u8]) -> Vec<[u8; 32]> { + assert!( + bytes.len() % 32 == 0, + "Bytes must be divisible by 32 to split into chunks" + ); + + bytes + .chunks(32) + .map(|el| { + let mut chunk = [0u8; 32]; + chunk.copy_from_slice(el); + chunk + }) + .collect() +} + +pub fn hash_bytecode(code: &[u8]) -> H256 { + let chunked_code = bytes_to_chunks(code); + let hash = + zkevm_opcode_defs::utils::bytecode_to_code_hash(&chunked_code).expect("Invalid bytecode"); + + H256(hash) +} diff --git a/src/processor/snapshot/mod.rs b/src/processor/snapshot/mod.rs index 03f3a53..42bc044 100644 --- a/src/processor/snapshot/mod.rs +++ b/src/processor/snapshot/mod.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, fmt, fs, path::PathBuf, str::FromStr}; +mod bytecode; mod types; use async_trait::async_trait; @@ -13,7 +14,7 @@ use state_reconstruct_fetcher::{ }; use tokio::sync::mpsc; -use self::types::{SnapshotStorageLog, StorageKey, StorageValue}; +use self::types::{SnapshotFactoryDependency, SnapshotStorageLog, StorageKey, StorageValue}; use super::Processor; use crate::processor::snapshot::types::MiniblockNumber; @@ -22,6 +23,7 @@ const DEFAULT_EXPORT_PATH: &str = "snapshot_export"; pub struct SnapshotExporter { storage_log_entries: HashMap, + factory_deps: Vec, index_to_key_map: IndexSet, path: PathBuf, } @@ -45,6 +47,7 @@ impl SnapshotExporter { Self { storage_log_entries, + factory_deps: vec![], index_to_key_map, path, } @@ -103,9 +106,13 @@ impl Processor for SnapshotExporter { .and_modify(|log| log.value = value); } - // TODO: We need to index these by hash. // Factory dependencies. - // for dep in &block.factory_deps {} + for dep in block.factory_deps { + self.factory_deps.push(SnapshotFactoryDependency { + bytecode_hash: bytecode::hash_bytecode(&dep), + bytecode: dep, + }); + } } fs::write(&self.path, self.to_string()).expect("failed to export snapshot"); From a52a91da249a3f230970325ae23d5a582f3b6e3c Mon Sep 17 00:00:00 2001 From: Jonathan <94441036+zeapoz@users.noreply.github.com> Date: Fri, 17 Nov 2023 07:49:55 +0100 Subject: [PATCH 4/4] ci: adjust checks (#44) --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b35f848..e77534a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,8 +18,8 @@ jobs: checks: [ fmt --all --check, check --all-targets --all-features, - clippy --all-targets --all-features, - test + clippy --all-targets --all-features -- -D warnings, + test --all-targets --all-features ] steps: - uses: actions/checkout@v4