From 5985f47027eead009b35dc217021cd80f7b44c26 Mon Sep 17 00:00:00 2001 From: zeapoz Date: Tue, 6 Aug 2024 14:25:07 +0200 Subject: [PATCH 1/3] feat: support legacy storage logs when importing snapshots --- Cargo.lock | 1072 ++++++++++++++++- Cargo.toml | 7 +- src/main.rs | 13 +- src/processor/snapshot/importer.rs | 9 +- src/processor/snapshot/mod.rs | 14 +- src/processor/tree/tree_wrapper.rs | 13 +- state-reconstruct-storage/Cargo.toml | 1 + state-reconstruct-storage/src/types.rs | 56 +- state-reconstruct-utils/Cargo.toml | 16 + .../src}/json.rs | 0 .../src/lib.rs | 13 + 11 files changed, 1111 insertions(+), 103 deletions(-) create mode 100644 state-reconstruct-utils/Cargo.toml rename {src/util => state-reconstruct-utils/src}/json.rs (100%) rename src/util/mod.rs => state-reconstruct-utils/src/lib.rs (58%) diff --git a/Cargo.lock b/Cargo.lock index 58fbcda..a66db15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,6 +146,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "ascii-canvas" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" +dependencies = [ + "term", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -292,6 +301,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dabbe35f96fb9507f7330793dc490461b2962659ac5d427181e451a623751d1" +[[package]] +name = "bech32" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" + [[package]] name = "beef" version = "0.5.2" @@ -363,6 +378,15 @@ dependencies = [ "syn 2.0.53", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + [[package]] name = "bit-vec" version = "0.6.3" @@ -520,6 +544,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "sha2 0.10.8", + "tinyvec", +] + [[package]] name = "bumpalo" version = "3.15.4" @@ -553,6 +587,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + [[package]] name = "bzip2-sys" version = "0.1.11+1.0.8" @@ -596,6 +640,20 @@ dependencies = [ "thiserror", ] +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.0.90" @@ -734,8 +792,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "634c509653de24b439672164bbf56f5f582a2ab0e313d3b0f6af0b7345cf2560" dependencies = [ "bincode", - "bs58", - "coins-core", + "bs58 0.4.0", + "coins-core 0.7.0", "digest 0.10.7", "getrandom", "hmac", @@ -746,6 +804,22 @@ dependencies = [ "thiserror", ] +[[package]] +name = "coins-bip32" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6be4a5df2098cd811f3194f64ddb96c267606bffd9689ac7b0160097b01ad3" +dependencies = [ + "bs58 0.5.1", + "coins-core 0.8.7", + "digest 0.10.7", + "hmac", + "k256 0.13.3", + "serde", + "sha2 0.10.8", + "thiserror", +] + [[package]] name = "coins-bip39" version = "0.7.0" @@ -753,11 +827,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a11892bcac83b4c6e95ab84b5b06c76d9d70ad73548dd07418269c5c7977171" dependencies = [ "bitvec 0.17.4", - "coins-bip32", + "coins-bip32 0.7.0", "getrandom", "hex", "hmac", - "pbkdf2", + "pbkdf2 0.11.0", + "rand 0.8.5", + "sha2 0.10.8", + "thiserror", +] + +[[package]] +name = "coins-bip39" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db8fba409ce3dc04f7d804074039eb68b960b0829161f8e06c95fea3f122528" +dependencies = [ + "bitvec 1.0.1", + "coins-bip32 0.8.7", + "hmac", + "once_cell", + "pbkdf2 0.12.2", "rand 0.8.5", "sha2 0.10.8", "thiserror", @@ -771,7 +861,7 @@ checksum = "c94090a6663f224feae66ab01e41a2555a8296ee07b5f20dab8888bdefc9f617" dependencies = [ "base58check", "base64 0.12.3", - "bech32", + "bech32 0.7.3", "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "digest 0.10.7", "generic-array 0.14.7", @@ -784,6 +874,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "coins-core" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5286a0843c21f8367f7be734f89df9b822e0321d8bcce8d6e735aadff7d74979" +dependencies = [ + "base64 0.21.7", + "bech32 0.9.1", + "bs58 0.5.1", + "digest 0.10.7", + "generic-array 0.14.7", + "hex", + "ripemd", + "serde", + "serde_derive", + "sha2 0.10.8", + "sha3 0.10.8", + "thiserror", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -795,6 +905,19 @@ name = "compile-fmt" version = "0.1.0" source = "git+https://github.com/slowli/compile-fmt.git?rev=c6a41c846c9a6f70cdba4b44c9f3922242ffcf12#c6a41c846c9a6f70cdba4b44c9f3922242ffcf12" +[[package]] +name = "const-hex" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8a24a26d37e1ffd45343323dc9fe6654ceea44c12f2fcb3d7ac29e610bc6" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "hex", + "proptest", + "serde", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1099,6 +1222,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "data-encoding" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + [[package]] name = "debugid" version = "0.8.0" @@ -1194,6 +1323,48 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dtoa" version = "1.0.9" @@ -1286,6 +1457,15 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "ena" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5" +dependencies = [ + "log", +] + [[package]] name = "encoding_rs" version = "0.8.33" @@ -1295,6 +1475,24 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "enr" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3d8dc56e02f954cac8eb489772c552c473346fc34f67412bb6244fd647f7e4" +dependencies = [ + "base64 0.21.7", + "bytes", + "hex", + "k256 0.13.3", + "log", + "rand 0.8.5", + "rlp", + "serde", + "sha3 0.10.8", + "zeroize", +] + [[package]] name = "envy" version = "0.4.2" @@ -1331,7 +1529,7 @@ dependencies = [ "digest 0.10.7", "hex", "hmac", - "pbkdf2", + "pbkdf2 0.11.0", "rand 0.8.5", "scrypt", "serde", @@ -1396,13 +1594,29 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11f26f9d8d80da18ca72aca51804c65eb2153093af3bec74fd5ce32aa0c1f665" dependencies = [ - "ethers-addressbook", - "ethers-contract", - "ethers-core", - "ethers-etherscan", - "ethers-middleware", - "ethers-providers", - "ethers-signers", + "ethers-addressbook 1.0.2", + "ethers-contract 1.0.2", + "ethers-core 1.0.2", + "ethers-etherscan 1.0.2", + "ethers-middleware 1.0.2", + "ethers-providers 1.0.2", + "ethers-signers 1.0.2", +] + +[[package]] +name = "ethers" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816841ea989f0c69e459af1cf23a6b0033b19a55424a1ea3a30099becdb8dec0" +dependencies = [ + "ethers-addressbook 2.0.14", + "ethers-contract 2.0.14", + "ethers-core 2.0.14", + "ethers-etherscan 2.0.14", + "ethers-middleware 2.0.14", + "ethers-providers 2.0.14", + "ethers-signers 2.0.14", + "ethers-solc", ] [[package]] @@ -1411,7 +1625,19 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe4be54dd2260945d784e06ccdeb5ad573e8f1541838cee13a1ab885485eaa0b" dependencies = [ - "ethers-core", + "ethers-core 1.0.2", + "once_cell", + "serde", + "serde_json", +] + +[[package]] +name = "ethers-addressbook" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5495afd16b4faa556c3bba1f21b98b4983e53c1755022377051a975c3b021759" +dependencies = [ + "ethers-core 2.0.14", "once_cell", "serde", "serde_json", @@ -1423,10 +1649,10 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9c3c3e119a89f0a9a1e539e7faecea815f74ddcf7c90d0b00d1f524db2fdc9c" dependencies = [ - "ethers-contract-abigen", - "ethers-contract-derive", - "ethers-core", - "ethers-providers", + "ethers-contract-abigen 1.0.2", + "ethers-contract-derive 1.0.2", + "ethers-core 1.0.2", + "ethers-providers 1.0.2", "futures-util", "hex", "once_cell", @@ -1436,6 +1662,25 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ethers-contract" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fceafa3578c836eeb874af87abacfb041f92b4da0a78a5edd042564b8ecdaaa" +dependencies = [ + "const-hex", + "ethers-contract-abigen 2.0.14", + "ethers-contract-derive 2.0.14", + "ethers-core 2.0.14", + "ethers-providers 2.0.14", + "futures-util", + "once_cell", + "pin-project", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "ethers-contract-abigen" version = "1.0.2" @@ -1445,7 +1690,7 @@ dependencies = [ "Inflector", "cfg-if 1.0.0", "dunce", - "ethers-core", + "ethers-core 1.0.2", "eyre", "getrandom", "hex", @@ -1456,19 +1701,43 @@ dependencies = [ "serde", "serde_json", "syn 1.0.109", - "toml", + "toml 0.5.11", "url", "walkdir", ] +[[package]] +name = "ethers-contract-abigen" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04ba01fbc2331a38c429eb95d4a570166781f14290ef9fdb144278a90b5a739b" +dependencies = [ + "Inflector", + "const-hex", + "dunce", + "ethers-core 2.0.14", + "ethers-etherscan 2.0.14", + "eyre", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "reqwest", + "serde", + "serde_json", + "syn 2.0.53", + "toml 0.8.2", + "walkdir", +] + [[package]] name = "ethers-contract-derive" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f192e8e4cf2b038318aae01e94e7644e0659a76219e94bcd3203df744341d61f" dependencies = [ - "ethers-contract-abigen", - "ethers-core", + "ethers-contract-abigen 1.0.2", + "ethers-core 1.0.2", "hex", "proc-macro2", "quote", @@ -1476,6 +1745,22 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ethers-contract-derive" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87689dcabc0051cde10caaade298f9e9093d65f6125c14575db3fd8c669a168f" +dependencies = [ + "Inflector", + "const-hex", + "ethers-contract-abigen 2.0.14", + "ethers-core 2.0.14", + "proc-macro2", + "quote", + "serde_json", + "syn 2.0.53", +] + [[package]] name = "ethers-core" version = "1.0.2" @@ -1484,7 +1769,7 @@ checksum = "ade3e9c97727343984e1ceada4fdab11142d2ee3472d2c67027d56b1251d4f15" dependencies = [ "arrayvec 0.7.4", "bytes", - "cargo_metadata", + "cargo_metadata 0.15.4", "chrono", "convert_case 0.6.0", "elliptic-curve 0.12.3", @@ -1500,20 +1785,50 @@ dependencies = [ "rlp-derive", "serde", "serde_json", - "strum", + "strum 0.24.1", "syn 1.0.109", "thiserror", "tiny-keccak 2.0.2", "unicode-xid", ] +[[package]] +name = "ethers-core" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d80cc6ad30b14a48ab786523af33b37f28a8623fc06afd55324816ef18fb1f" +dependencies = [ + "arrayvec 0.7.4", + "bytes", + "cargo_metadata 0.18.1", + "chrono", + "const-hex", + "elliptic-curve 0.13.8", + "ethabi", + "generic-array 0.14.7", + "k256 0.13.3", + "num_enum 0.7.3", + "once_cell", + "open-fastrlp", + "rand 0.8.5", + "rlp", + "serde", + "serde_json", + "strum 0.26.3", + "syn 2.0.53", + "tempfile", + "thiserror", + "tiny-keccak 2.0.2", + "unicode-xid", +] + [[package]] name = "ethers-etherscan" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9713f525348e5dde025d09b0a4217429f8074e8ff22c886263cc191e87d8216" dependencies = [ - "ethers-core", + "ethers-core 1.0.2", "getrandom", "reqwest", "semver", @@ -1524,6 +1839,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "ethers-etherscan" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79e5973c26d4baf0ce55520bd732314328cabe53193286671b47144145b9649" +dependencies = [ + "chrono", + "ethers-core 2.0.14", + "reqwest", + "semver", + "serde", + "serde_json", + "thiserror", + "tracing", +] + [[package]] name = "ethers-middleware" version = "1.0.2" @@ -1532,11 +1863,38 @@ checksum = "e71df7391b0a9a51208ffb5c7f2d068900e99d6b3128d3a4849d138f194778b7" dependencies = [ "async-trait", "auto_impl 0.5.0", - "ethers-contract", - "ethers-core", - "ethers-etherscan", - "ethers-providers", - "ethers-signers", + "ethers-contract 1.0.2", + "ethers-core 1.0.2", + "ethers-etherscan 1.0.2", + "ethers-providers 1.0.2", + "ethers-signers 1.0.2", + "futures-locks", + "futures-util", + "instant", + "reqwest", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "tracing-futures", + "url", +] + +[[package]] +name = "ethers-middleware" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f9fdf09aec667c099909d91908d5eaf9be1bd0e2500ba4172c1d28bfaa43de" +dependencies = [ + "async-trait", + "auto_impl 1.2.0", + "ethers-contract 2.0.14", + "ethers-core 2.0.14", + "ethers-etherscan 2.0.14", + "ethers-providers 2.0.14", + "ethers-signers 2.0.14", + "futures-channel", "futures-locks", "futures-util", "instant", @@ -1559,7 +1917,7 @@ dependencies = [ "async-trait", "auto_impl 1.2.0", "base64 0.13.1", - "ethers-core", + "ethers-core 1.0.2", "futures-core", "futures-timer", "futures-util", @@ -1585,6 +1943,43 @@ dependencies = [ "ws_stream_wasm", ] +[[package]] +name = "ethers-providers" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6434c9a33891f1effc9c75472e12666db2fa5a0fec4b29af6221680a6fe83ab2" +dependencies = [ + "async-trait", + "auto_impl 1.2.0", + "base64 0.21.7", + "bytes", + "const-hex", + "enr", + "ethers-core 2.0.14", + "futures-core", + "futures-timer", + "futures-util", + "hashers", + "http", + "instant", + "jsonwebtoken 8.3.0", + "once_cell", + "pin-project", + "reqwest", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-tungstenite", + "tracing", + "tracing-futures", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "ws_stream_wasm", +] + [[package]] name = "ethers-signers" version = "1.0.2" @@ -1592,17 +1987,68 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f41ced186867f64773db2e55ffdd92959e094072a1d09a5e5e831d443204f98" dependencies = [ "async-trait", - "coins-bip32", - "coins-bip39", + "coins-bip32 0.7.0", + "coins-bip39 0.7.0", "elliptic-curve 0.12.3", "eth-keystore", - "ethers-core", + "ethers-core 1.0.2", "hex", "rand 0.8.5", "sha2 0.10.8", "thiserror", ] +[[package]] +name = "ethers-signers" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "228875491c782ad851773b652dd8ecac62cda8571d3bc32a5853644dd26766c2" +dependencies = [ + "async-trait", + "coins-bip32 0.8.7", + "coins-bip39 0.8.7", + "const-hex", + "elliptic-curve 0.13.8", + "eth-keystore", + "ethers-core 2.0.14", + "rand 0.8.5", + "sha2 0.10.8", + "thiserror", + "tracing", +] + +[[package]] +name = "ethers-solc" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66244a771d9163282646dbeffe0e6eca4dda4146b6498644e678ac6089b11edd" +dependencies = [ + "cfg-if 1.0.0", + "const-hex", + "dirs", + "dunce", + "ethers-core 2.0.14", + "glob", + "home", + "md-5", + "num_cpus", + "once_cell", + "path-slash", + "rayon", + "regex", + "semver", + "serde", + "serde_json", + "solang-parser", + "svm-rs", + "thiserror", + "tiny-keccak 2.0.2", + "tokio", + "tracing", + "walkdir", + "yansi", +] + [[package]] name = "eyre" version = "0.6.12" @@ -1749,6 +2195,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -1848,6 +2304,10 @@ name = "futures-timer" version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +dependencies = [ + "gloo-timers", + "send_wrapper 0.4.0", +] [[package]] name = "futures-util" @@ -1921,6 +2381,18 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "google-cloud-auth" version = "0.13.1" @@ -1932,7 +2404,7 @@ dependencies = [ "google-cloud-metadata", "google-cloud-token", "home", - "jsonwebtoken", + "jsonwebtoken 9.2.0", "reqwest", "serde", "serde_json", @@ -1974,7 +2446,7 @@ dependencies = [ "pkcs8 0.10.2", "regex", "reqwest", - "ring", + "ring 0.17.8", "serde", "serde_json", "sha2 0.10.8", @@ -2400,6 +2872,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "jsonwebtoken" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +dependencies = [ + "base64 0.21.7", + "pem 1.1.1", + "ring 0.16.20", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "jsonwebtoken" version = "9.2.0" @@ -2408,8 +2894,8 @@ checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" dependencies = [ "base64 0.21.7", "js-sys", - "pem", - "ring", + "pem 3.0.3", + "ring 0.17.8", "serde", "serde_json", "simple_asn1", @@ -2451,6 +2937,36 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "lalrpop" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cb077ad656299f160924eb2912aa147d7339ea7d69e1b5517326fdcec3c1ca" +dependencies = [ + "ascii-canvas", + "bit-set", + "ena", + "itertools 0.11.0", + "lalrpop-util", + "petgraph", + "regex", + "regex-syntax 0.8.2", + "string_cache", + "term", + "tiny-keccak 2.0.2", + "unicode-xid", + "walkdir", +] + +[[package]] +name = "lalrpop-util" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" +dependencies = [ + "regex-automata 0.4.6", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -2491,6 +3007,16 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.4.2", + "libc", +] + [[package]] name = "librocksdb-sys" version = "0.11.0+8.1.1" @@ -2623,6 +3149,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if 1.0.0", + "digest 0.10.7", +] + [[package]] name = "memchr" version = "2.7.1" @@ -2749,6 +3285,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nom" version = "7.1.3" @@ -2880,7 +3422,16 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a015b430d3c108a207fd776d2e2196aaf8b1cf8cf93253e3a097ff3085076a1" dependencies = [ - "num_enum_derive", + "num_enum_derive 0.6.1", +] + +[[package]] +name = "num_enum" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +dependencies = [ + "num_enum_derive 0.7.3", ] [[package]] @@ -2895,6 +3446,18 @@ dependencies = [ "syn 2.0.53", ] +[[package]] +name = "num_enum_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +dependencies = [ + "proc-macro-crate 2.0.2", + "proc-macro2", + "quote", + "syn 2.0.53", +] + [[package]] name = "object" version = "0.32.2" @@ -2991,6 +3554,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-float" version = "2.10.1" @@ -3137,16 +3706,32 @@ dependencies = [ "subtle", ] +[[package]] +name = "path-slash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" + [[package]] name = "pbkdf2" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest 0.10.7", + "hmac", + "password-hash", + "sha2 0.10.8", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ "digest 0.10.7", "hmac", - "password-hash", - "sha2 0.10.8", ] [[package]] @@ -3155,6 +3740,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + [[package]] name = "pem" version = "3.0.3" @@ -3200,6 +3794,57 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared 0.11.2", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator", + "phf_shared 0.11.2", + "proc-macro2", + "quote", + "syn 2.0.53", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -3276,6 +3921,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "prettyplease" version = "0.2.16" @@ -3376,6 +4027,22 @@ dependencies = [ "syn 2.0.53", ] +[[package]] +name = "proptest" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +dependencies = [ + "bitflags 2.4.2", + "lazy_static", + "num-traits", + "rand 0.8.5", + "rand_chacha", + "rand_xorshift", + "regex-syntax 0.8.2", + "unarray", +] + [[package]] name = "prost" version = "0.12.4" @@ -3560,6 +4227,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rayon" version = "1.9.0" @@ -3607,6 +4283,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.10.4" @@ -3719,6 +4406,21 @@ dependencies = [ "subtle", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + [[package]] name = "ring" version = "0.17.8" @@ -3729,8 +4431,8 @@ dependencies = [ "cfg-if 1.0.0", "getrandom", "libc", - "spin", - "untrusted", + "spin 0.9.8", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -3750,6 +4452,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" dependencies = [ "bytes", + "rlp-derive", "rustc-hex", ] @@ -3821,7 +4524,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring", + "ring 0.17.8", "rustls-webpki", "sct", ] @@ -3841,8 +4544,8 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -3921,7 +4624,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f9e24d2b632954ded8ab2ef9fea0a0c769ea56ea98bddbafbad22caeeadf45d" dependencies = [ "hmac", - "pbkdf2", + "pbkdf2 0.11.0", "salsa20", "sha2 0.10.8", ] @@ -3932,8 +4635,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -4014,6 +4717,12 @@ dependencies = [ "serde", ] +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + [[package]] name = "send_wrapper" version = "0.6.0" @@ -4136,9 +4845,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.197" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] @@ -4165,9 +4874,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", @@ -4176,11 +4885,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -4206,6 +4916,15 @@ dependencies = [ "syn 2.0.53", ] +[[package]] +name = "serde_spanned" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4373,6 +5092,12 @@ dependencies = [ "time", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -4401,6 +5126,26 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "solang-parser" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c425ce1c59f4b154717592f0bdf4715c3a1d55058883622d3157e1f0908a5b26" +dependencies = [ + "itertools 0.11.0", + "lalrpop", + "lalrpop-util", + "phf", + "thiserror", + "unicode-xid", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.8" @@ -4442,7 +5187,7 @@ dependencies = [ "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "chrono", "clap", - "ethers", + "ethers 1.0.2", "eyre", "hex", "indexmap", @@ -4453,6 +5198,7 @@ dependencies = [ "serde_json", "state-reconstruct-fetcher", "state-reconstruct-storage", + "state-reconstruct-utils", "thiserror", "tikv-jemallocator", "tokio", @@ -4469,7 +5215,7 @@ dependencies = [ "bincode", "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "chrono", - "ethers", + "ethers 1.0.2", "eyre", "hex", "indexmap", @@ -4494,7 +5240,7 @@ dependencies = [ "bincode", "bytes", "chrono", - "ethers", + "ethers 1.0.2", "eyre", "flate2", "prost", @@ -4502,16 +5248,42 @@ dependencies = [ "rocksdb", "serde", "serde_repr", + "state-reconstruct-utils", "thiserror", "zkevm_opcode_defs 1.3.2 (git+https://github.com/matter-labs/era-zkevm_opcode_defs.git)", ] +[[package]] +name = "state-reconstruct-utils" +version = "0.1.0" +dependencies = [ + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "ethers 2.0.14", + "primitive-types", + "serde", + "serde_json", + "zksync_storage", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "string_cache" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot 0.12.1", + "phf_shared 0.10.0", + "precomputed-hash", +] + [[package]] name = "strsim" version = "0.10.0" @@ -4530,7 +5302,16 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ - "strum_macros", + "strum_macros 0.24.3", +] + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", ] [[package]] @@ -4546,12 +5327,45 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.53", +] + [[package]] name = "subtle" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "svm-rs" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11297baafe5fa0c99d5722458eac6a5e25c01eb1b8e5cd137f54079093daa7a4" +dependencies = [ + "dirs", + "fs2", + "hex", + "once_cell", + "reqwest", + "semver", + "serde", + "serde_json", + "sha2 0.10.8", + "thiserror", + "url", + "zip", +] + [[package]] name = "syn" version = "1.0.109" @@ -4619,6 +5433,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.58" @@ -4783,6 +5608,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" +dependencies = [ + "futures-util", + "log", + "rustls", + "tokio", + "tokio-rustls", + "tungstenite", + "webpki-roots", +] + [[package]] name = "tokio-util" version = "0.7.10" @@ -4806,11 +5646,26 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.20.2", +] + [[package]] name = "toml_datetime" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -4830,6 +5685,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" dependencies = [ "indexmap", + "serde", + "serde_spanned", "toml_datetime", "winnow", ] @@ -4931,6 +5788,26 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.5", + "rustls", + "sha1", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.17.0" @@ -4958,6 +5835,12 @@ dependencies = [ "libc", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.7.0" @@ -5016,6 +5899,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -5053,6 +5942,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8parse" version = "0.2.1" @@ -5500,7 +6395,7 @@ dependencies = [ "log", "pharos", "rustc_version", - "send_wrapper", + "send_wrapper 0.6.0", "thiserror", "wasm-bindgen", "wasm-bindgen-futures", @@ -5516,6 +6411,12 @@ dependencies = [ "tap", ] +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zerocopy" version = "0.7.32" @@ -5542,6 +6443,26 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils 0.8.19", + "flate2", + "hmac", + "pbkdf2 0.11.0", + "sha1", + "time", + "zstd", +] + [[package]] name = "zk_evm" version = "1.3.3" @@ -5578,7 +6499,7 @@ version = "0.1.0" source = "git+https://github.com/matter-labs/era-zk_evm_abstractions.git#32dd320953841aa78579d9da08abbc70bcaed175" dependencies = [ "anyhow", - "num_enum", + "num_enum 0.6.1", "serde", "static_assertions", "zkevm_opcode_defs 1.3.2 (git+https://github.com/matter-labs/era-zkevm_opcode_defs.git?branch=v1.3.2)", @@ -5590,7 +6511,7 @@ version = "1.4.1" source = "git+https://github.com/matter-labs/era-zk_evm_abstractions.git?branch=v1.4.1#0aac08c3b097ee8147e748475117ac46bddcdcef" dependencies = [ "anyhow", - "num_enum", + "num_enum 0.6.1", "serde", "static_assertions", "zkevm_opcode_defs 1.4.1", @@ -5819,7 +6740,7 @@ dependencies = [ "circuit_sequencer_api", "serde", "serde_with", - "strum", + "strum 0.24.1", "zksync_object_store", "zksync_types", ] @@ -5857,7 +6778,7 @@ dependencies = [ "hex", "itertools 0.10.5", "num", - "num_enum", + "num_enum 0.6.1", "once_cell", "prost", "rlp", @@ -5865,7 +6786,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "strum", + "strum 0.24.1", "thiserror", "zksync_basic_types", "zksync_config", @@ -5899,6 +6820,25 @@ dependencies = [ "zksync_basic_types", ] +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.9+zstd.1.5.5" diff --git a/Cargo.toml b/Cargo.toml index f2287dd..cd164c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,11 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] -members = ["state-reconstruct-fetcher", "state-reconstruct-storage"] +members = [ + "state-reconstruct-fetcher", + "state-reconstruct-storage", + "state-reconstruct-utils", +] [dependencies] async-trait = "0.1.74" @@ -24,6 +28,7 @@ serde = { version = "1.0.189", features = ["derive"] } serde_json = { version = "1.0.107", features = ["std"] } state-reconstruct-fetcher = { path = "./state-reconstruct-fetcher" } state-reconstruct-storage = { path = "./state-reconstruct-storage" } +state-reconstruct-utils = { path = "./state-reconstruct-utils" } thiserror = "1.0.50" tikv-jemallocator = "0.5" tokio = { version = "1.33.0", features = ["macros"] } diff --git a/src/main.rs b/src/main.rs index 0648b4a..36a2d04 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,6 @@ mod cli; mod processor; -mod util; use std::{ env, @@ -23,17 +22,15 @@ use state_reconstruct_fetcher::{ l1_fetcher::{L1Fetcher, L1FetcherOptions}, types::CommitBlock, }; +use state_reconstruct_utils::json; use tikv_jemallocator::Jemalloc; use tokio::sync::mpsc; use tracing_subscriber::{filter::LevelFilter, EnvFilter}; -use crate::{ - processor::{ - json::JsonSerializationProcessor, - tree::{query_tree::QueryTree, TreeProcessor}, - Processor, - }, - util::json, +use crate::processor::{ + json::JsonSerializationProcessor, + tree::{query_tree::QueryTree, TreeProcessor}, + Processor, }; #[global_allocator] diff --git a/src/processor/snapshot/importer.rs b/src/processor/snapshot/importer.rs index 9a39457..b78af79 100644 --- a/src/processor/snapshot/importer.rs +++ b/src/processor/snapshot/importer.rs @@ -7,8 +7,9 @@ use ethers::types::U64; use eyre::Result; use regex::{Captures, Regex}; use state_reconstruct_storage::types::{ - Proto, SnapshotFactoryDependencies, SnapshotHeader, SnapshotStorageLogsChunk, + LegacyProto, Proto, SnapshotFactoryDependencies, SnapshotHeader, SnapshotStorageLogsChunk, SnapshotStorageLogsChunkMetadata, + SnapshotVersion::{Version0, Version1}, }; use tokio::sync::mpsc::{self, Sender}; @@ -87,7 +88,11 @@ impl SnapshotImporter { let total_chunks = filepaths.len(); for (i, path) in filepaths.into_iter().enumerate() { let bytes = fs::read(path)?; - let storage_logs_chunk = SnapshotStorageLogsChunk::decode(&bytes)?; + + let storage_logs_chunk = match header.version { + Version0 => SnapshotStorageLogsChunk::decode_legacy(&bytes)?, + Version1 => SnapshotStorageLogsChunk::decode(&bytes)?, + }; tracing::info!("Read chunk {}/{}, processing...", i + 1, total_chunks); tx.send(storage_logs_chunk).await?; } diff --git a/src/processor/snapshot/mod.rs b/src/processor/snapshot/mod.rs index eda7a65..bdd989a 100644 --- a/src/processor/snapshot/mod.rs +++ b/src/processor/snapshot/mod.rs @@ -6,7 +6,6 @@ pub mod exporter; pub mod importer; use async_trait::async_trait; -use blake2::{Blake2s256, Digest}; use ethers::types::{Address, H256, U256, U64}; use eyre::Result; use state_reconstruct_fetcher::{ @@ -17,10 +16,10 @@ use state_reconstruct_storage::{ bytecode, types::{SnapshotFactoryDependency, SnapshotStorageLog}, }; +use state_reconstruct_utils::{derive_final_address_for_params, h256_to_u256, unpack_block_info}; use tokio::sync::mpsc; use super::Processor; -use crate::util::{h256_to_u256, unpack_block_info}; pub const DEFAULT_DB_PATH: &str = "snapshot_db"; pub const SNAPSHOT_HEADER_FILE_NAME: &str = "snapshot-header.json"; @@ -235,17 +234,6 @@ fn reconstruct_genesis_state(database: &mut SnapshotDatabase, path: &str) -> Res Ok(()) } -fn derive_final_address_for_params(address: &Address, key: &U256) -> [u8; 32] { - let mut buffer = [0u8; 64]; - buffer[12..32].copy_from_slice(&address.0); - key.to_big_endian(&mut buffer[32..64]); - - let mut result = [0u8; 32]; - result.copy_from_slice(Blake2s256::digest(buffer).as_slice()); - - result -} - #[cfg(test)] mod tests { use std::fs; diff --git a/src/processor/tree/tree_wrapper.rs b/src/processor/tree/tree_wrapper.rs index 3577d0c..502038d 100644 --- a/src/processor/tree/tree_wrapper.rs +++ b/src/processor/tree/tree_wrapper.rs @@ -1,6 +1,5 @@ use std::{collections::HashMap, fs, num::NonZeroU32, path::Path, str::FromStr, sync::Arc}; -use blake2::{Blake2s256, Digest}; use ethers::types::{Address, H256, U256, U64}; use eyre::Result; use state_reconstruct_fetcher::{ @@ -10,6 +9,7 @@ use state_reconstruct_fetcher::{ use state_reconstruct_storage::{ reconstruction::ReconstructionDatabase, types::SnapshotStorageLogsChunk, PackingType, }; +use state_reconstruct_utils::derive_final_address_for_params; use thiserror::Error; use tokio::sync::{ mpsc::{self, Receiver}, @@ -326,14 +326,3 @@ fn reconstruct_genesis_state( Ok(()) } - -fn derive_final_address_for_params(address: &Address, key: &U256) -> [u8; 32] { - let mut buffer = [0u8; 64]; - buffer[12..32].copy_from_slice(&address.0); - key.to_big_endian(&mut buffer[32..64]); - - let mut result = [0u8; 32]; - result.copy_from_slice(Blake2s256::digest(buffer).as_slice()); - - result -} diff --git a/state-reconstruct-storage/Cargo.toml b/state-reconstruct-storage/Cargo.toml index f2db894..648d414 100644 --- a/state-reconstruct-storage/Cargo.toml +++ b/state-reconstruct-storage/Cargo.toml @@ -18,6 +18,7 @@ prost = "0.12.4" rocksdb = "0.21.0" thiserror = "1.0.50" zkevm_opcode_defs = { git = "https://github.com/matter-labs/era-zkevm_opcode_defs.git" } +state-reconstruct-utils = { path = "../state-reconstruct-utils" } [build-dependencies] prost-build = "0.12.4" diff --git a/state-reconstruct-storage/src/types.rs b/state-reconstruct-storage/src/types.rs index e837539..e220e6d 100644 --- a/state-reconstruct-storage/src/types.rs +++ b/state-reconstruct-storage/src/types.rs @@ -4,12 +4,13 @@ use std::{ }; use bytes::BytesMut; -use ethers::types::{H256, U256, U64}; +use ethers::types::{Address, H256, U256, U64}; use eyre::Result; use flate2::{read::GzDecoder, write::GzEncoder, Compression}; use prost::Message; use serde::{Deserialize, Serialize}; use serde_repr::{Deserialize_repr, Serialize_repr}; +use state_reconstruct_utils::derive_final_address_for_params; use super::bytecode; @@ -73,6 +74,27 @@ pub trait Proto { } } +pub trait LegacyProto { + type ProtoStruct: Message + Default; + + fn from_legacy_proto(proto: Self::ProtoStruct) -> Result + where + Self: Sized; + + /// Decode a slice of gzip-compressed bytes into [`Self`]. + fn decode_legacy(bytes: &[u8]) -> Result + where + Self: Sized, + { + let mut decoder = GzDecoder::new(bytes); + let mut decompressed_bytes = Vec::new(); + decoder.read_to_end(&mut decompressed_bytes)?; + + let proto = Self::ProtoStruct::decode(&decompressed_bytes[..])?; + Self::from_legacy_proto(proto) + } +} + /// Version of snapshot influencing the format of data stored in GCS. #[derive(Clone, Default, Debug, Serialize_repr, Deserialize_repr)] #[repr(u16)] @@ -140,6 +162,20 @@ impl Proto for SnapshotStorageLogsChunk { } } +impl LegacyProto for SnapshotStorageLogsChunk { + type ProtoStruct = protobuf::SnapshotStorageLogsChunk; + + fn from_legacy_proto(proto: Self::ProtoStruct) -> Result { + Ok(Self { + storage_logs: proto + .storage_logs + .into_iter() + .map(SnapshotStorageLog::from_legacy_proto) + .collect::>>()?, + }) + } +} + // "most recent" for each key together with info when the key was first used #[derive(Default, Debug, Serialize, Deserialize)] pub struct SnapshotStorageLog { @@ -177,6 +213,24 @@ impl Proto for SnapshotStorageLog { } } +impl LegacyProto for SnapshotStorageLog { + type ProtoStruct = protobuf::SnapshotStorageLog; + + fn from_legacy_proto(proto: Self::ProtoStruct) -> Result { + let address_bytes: [u8; 20] = proto.account_address().try_into()?; + let address = Address::from(address_bytes); + let storage_key = StorageKey::from(proto.storage_key()); + let hashed_key = StorageKey::from(derive_final_address_for_params(&address, &storage_key)); + let value_bytes: [u8; 32] = proto.storage_value().try_into()?; + Ok(Self { + key: hashed_key, + value: StorageValue::from(&value_bytes), + l1_batch_number_of_initial_write: proto.l1_batch_number_of_initial_write().into(), + enumeration_index: proto.enumeration_index(), + }) + } +} + #[derive(Default, Debug, Serialize, Deserialize)] pub struct SnapshotFactoryDependencies { pub factory_deps: Vec, diff --git a/state-reconstruct-utils/Cargo.toml b/state-reconstruct-utils/Cargo.toml new file mode 100644 index 0000000..28909c6 --- /dev/null +++ b/state-reconstruct-utils/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "state-reconstruct-utils" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +blake2 = "0.10.6" +ethers = "2.0.14" +primitive-types = "0.12.2" +serde = "1.0.204" +serde_json = "1.0.122" +zksync_storage = { git = "https://github.com/matter-labs/zksync-era.git" } + +[build-dependencies] diff --git a/src/util/json.rs b/state-reconstruct-utils/src/json.rs similarity index 100% rename from src/util/json.rs rename to state-reconstruct-utils/src/json.rs diff --git a/src/util/mod.rs b/state-reconstruct-utils/src/lib.rs similarity index 58% rename from src/util/mod.rs rename to state-reconstruct-utils/src/lib.rs index 94bb554..3e5a91d 100644 --- a/src/util/mod.rs +++ b/state-reconstruct-utils/src/lib.rs @@ -1,3 +1,5 @@ +use blake2::{Blake2s256, Digest}; +use ethers::types::Address; use primitive_types::{H256, U256}; pub mod json; @@ -14,3 +16,14 @@ pub fn unpack_block_info(info: U256) -> (u64, u64) { let block_timestamp = (info % SYSTEM_BLOCK_INFO_BLOCK_NUMBER_MULTIPLIER).as_u64(); (block_number, block_timestamp) } + +pub fn derive_final_address_for_params(address: &Address, key: &U256) -> [u8; 32] { + let mut buffer = [0u8; 64]; + buffer[12..32].copy_from_slice(&address.0); + key.to_big_endian(&mut buffer[32..64]); + + let mut result = [0u8; 32]; + result.copy_from_slice(Blake2s256::digest(buffer).as_slice()); + + result +} From 72a79e8affc72198920828f4e89c92272ea392c6 Mon Sep 17 00:00:00 2001 From: zeapoz Date: Wed, 7 Aug 2024 13:54:58 +0200 Subject: [PATCH 2/3] chore: endianness --- state-reconstruct-storage/src/types.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/state-reconstruct-storage/src/types.rs b/state-reconstruct-storage/src/types.rs index e220e6d..4a149fb 100644 --- a/state-reconstruct-storage/src/types.rs +++ b/state-reconstruct-storage/src/types.rs @@ -205,7 +205,7 @@ impl Proto for SnapshotStorageLog { fn from_proto(proto: Self::ProtoStruct) -> Result { let value_bytes: [u8; 32] = proto.storage_value().try_into()?; Ok(Self { - key: U256::from_big_endian(proto.hashed_key()), + key: StorageKey::from_big_endian(proto.hashed_key()), value: StorageValue::from(&value_bytes), l1_batch_number_of_initial_write: proto.l1_batch_number_of_initial_write().into(), enumeration_index: proto.enumeration_index(), @@ -219,8 +219,11 @@ impl LegacyProto for SnapshotStorageLog { fn from_legacy_proto(proto: Self::ProtoStruct) -> Result { let address_bytes: [u8; 20] = proto.account_address().try_into()?; let address = Address::from(address_bytes); - let storage_key = StorageKey::from(proto.storage_key()); - let hashed_key = StorageKey::from(derive_final_address_for_params(&address, &storage_key)); + let storage_key = StorageKey::from_big_endian(proto.storage_key()); + let hashed_key = StorageKey::from_little_endian(&derive_final_address_for_params( + &address, + &storage_key, + )); let value_bytes: [u8; 32] = proto.storage_value().try_into()?; Ok(Self { key: hashed_key, From 2e25b52a3f23e01b0a2fe9f88012524302dcc279 Mon Sep 17 00:00:00 2001 From: zeapoz Date: Thu, 15 Aug 2024 12:50:51 +0200 Subject: [PATCH 3/3] chore: update snapshot header file name --- src/processor/snapshot/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processor/snapshot/mod.rs b/src/processor/snapshot/mod.rs index bdd989a..154e2f3 100644 --- a/src/processor/snapshot/mod.rs +++ b/src/processor/snapshot/mod.rs @@ -22,7 +22,7 @@ use tokio::sync::mpsc; use super::Processor; pub const DEFAULT_DB_PATH: &str = "snapshot_db"; -pub const SNAPSHOT_HEADER_FILE_NAME: &str = "snapshot-header.json"; +pub const SNAPSHOT_HEADER_FILE_NAME: &str = "snapshot_header.json"; pub const SNAPSHOT_FACTORY_DEPS_FILE_NAME_SUFFIX: &str = "factory_deps.proto.gzip"; pub struct SnapshotBuilder {