Altair devnets

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "1.4.0"
+version = "1.5.0-rc.0"
 authors = ["Paul Hauner <[email protected]>", "Age Manning <[email protected]"]
 edition = "2018"
 

beacon_node/eth2_libp2p/Cargo.toml

@@ -42,13 +42,14 @@ regex = "1.3.9"
 strum = { version = "0.20", features = ["derive"] }
 superstruct = "0.2.0"
 
+# TODO: remove the rev-reference and go back to release versions once this PR is merged:
+#
+# https://github.com/libp2p/rust-libp2p/pull/2175
 [dependencies.libp2p]
 #version = "0.39.1"
 #default-features = false
-# TODO: Update once https://github.com/libp2p/rust-libp2p/pull/2103 and 
-# https://github.com/libp2p/rust-libp2p/pull/2137 are merged upstream.
 git = "https://github.com/sigp/rust-libp2p"
-rev = "75fd53ec5407a58ae1ff600fd1c68ea49079364a"
+rev = "35eaa10eeacabea3239bfd7216a2f0c836efb719"
 features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns-tokio", "tcp-tokio"]
 
 [dev-dependencies]

beacon_node/eth2_libp2p/src/discovery/enr_ext.rs

@@ -203,15 +203,15 @@ impl CombinedKeyPublicExt for CombinedPublicKey {
                     libp2p::core::identity::secp256k1::PublicKey::decode(&pk_bytes)
                         .expect("valid public key"),
                 );
-                PeerId::from_public_key(libp2p_pk)
+                PeerId::from_public_key(&libp2p_pk)
             }
             Self::Ed25519(pk) => {
                 let pk_bytes = pk.to_bytes();
                 let libp2p_pk = libp2p::core::PublicKey::Ed25519(
                     libp2p::core::identity::ed25519::PublicKey::decode(&pk_bytes)
                         .expect("valid public key"),
                 );
-                PeerId::from_public_key(libp2p_pk)
+                PeerId::from_public_key(&libp2p_pk)
             }
         }
     }
@@ -283,7 +283,7 @@ mod tests {
         let libp2p_sk = libp2p::identity::secp256k1::SecretKey::from_bytes(sk_bytes).unwrap();
         let secp256k1_kp: libp2p::identity::secp256k1::Keypair = libp2p_sk.into();
         let libp2p_kp = Keypair::Secp256k1(secp256k1_kp);
-        let peer_id = libp2p_kp.public().into_peer_id();
+        let peer_id = libp2p_kp.public().to_peer_id();
 
         let enr = discv5::enr::EnrBuilder::new("v4")
             .build(&secret_key)
@@ -304,7 +304,7 @@ mod tests {
         let libp2p_sk = libp2p::identity::ed25519::SecretKey::from_bytes(sk_bytes).unwrap();
         let ed25519_kp: libp2p::identity::ed25519::Keypair = libp2p_sk.into();
         let libp2p_kp = Keypair::Ed25519(ed25519_kp);
-        let peer_id = libp2p_kp.public().into_peer_id();
+        let peer_id = libp2p_kp.public().to_peer_id();
 
         let enr = discv5::enr::EnrBuilder::new("v4").build(&keypair).unwrap();
         let node_id = peer_id_to_node_id(&peer_id).unwrap();

beacon_node/eth2_libp2p/src/service.rs

@@ -143,7 +143,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
                 .with_max_established_outgoing(Some(
                     (config.target_peers as f32 * (1.0 + PEER_EXCESS_FACTOR)) as u32,
                 ))
-                .with_max_established_total(Some(
+                .with_max_established(Some(
                     (config.target_peers as f32 * (1.0 + PEER_EXCESS_FACTOR)) as u32,
                 ))
                 .with_max_established_per_peer(Some(MAX_CONNECTIONS_PER_PEER));

boot_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "boot_node"
-version = "1.4.0"
+version = "1.5.0-rc.0"
 authors = ["Sigma Prime <[email protected]>"]
 edition = "2018"
 

common/eth2/Cargo.toml

@@ -17,7 +17,7 @@ proto_array = { path = "../../consensus/proto_array", optional = true }
 serde_utils = { path = "../../consensus/serde_utils" }
 zeroize = { version = "1.1.1", features = ["zeroize_derive"] }
 eth2_keystore = { path = "../../crypto/eth2_keystore" }
-libsecp256k1 = "0.5.0"
+libsecp256k1 = "0.6.0"
 ring = "0.16.19"
 bytes = "1.0.1"
 account_utils = { path = "../../common/account_utils" }

common/eth2_config/Cargo.toml

@@ -8,3 +8,4 @@ edition = "2018"
 serde = "1.0.116"
 serde_derive = "1.0.116"
 types = { path = "../../consensus/types" }
+paste = "1.0.5"

common/eth2_config/src/lib.rs

@@ -1,7 +1,16 @@
+//! This crate primarily exists to serve the `common/eth2_network_configs` crate, by providing the
+//! canonical list of built-in-networks and some tooling to help include those configurations in the
+//! `lighthouse` binary.
+//!
+//! It also provides some additional structs which are useful to other components of `lighthouse`
+//! (e.g., `Eth2Config`).
+
 use std::env;
 use std::path::PathBuf;
 use types::{ChainSpec, EthSpecId};
 
+pub use paste::paste;
+
 // A macro is used to define this constant so it can be used with `include_bytes!`.
 #[macro_export]
 macro_rules! predefined_networks_dir {
@@ -73,40 +82,157 @@ impl<'a> Eth2NetArchiveAndDirectory<'a> {
     }
 }
 
-macro_rules! define_net {
-    ($title: ident, $macro_title: tt, $name: tt, $genesis_is_known: tt) => {
-        #[macro_use]
-        pub mod $title {
-            use super::*;
-
-            pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory {
-                name: $name,
-                unique_id: $name,
-                genesis_is_known: $genesis_is_known,
-            };
-
-            // A wrapper around `std::include_bytes` which includes a file from a specific network
-            // directory. Used by upstream crates to import files at compile time.
-            #[macro_export]
-            macro_rules! $macro_title {
-                ($base_dir: tt, $filename: tt) => {
-                    include_bytes!(concat!(
-                        $base_dir,
-                        "/",
-                        predefined_networks_dir!(),
-                        "/",
-                        $name,
-                        "/",
-                        $filename
-                    ))
+/// Indicates that the `genesis.ssz.zip` file is present on the filesystem. This means that the
+/// deposit ceremony has concluded and the final genesis `BeaconState` is known.
+const GENESIS_STATE_IS_KNOWN: bool = true;
+
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub struct HardcodedNet {
+    pub name: &'static str,
+    pub genesis_is_known: bool,
+    pub config: &'static [u8],
+    pub deploy_block: &'static [u8],
+    pub boot_enr: &'static [u8],
+    pub genesis_state_bytes: &'static [u8],
+}
+
+/// Defines an `Eth2NetArchiveAndDirectory` for some network.
+///
+/// It also defines a `include_<title>_bytes!` macro which provides a wrapper around
+/// `std::include_bytes`, allowing the inclusion of bytes from the specific testnet directory.
+macro_rules! define_archive {
+    ($name_ident: ident, $name_str: tt, $genesis_is_known: ident) => {
+        paste! {
+            #[macro_use]
+            pub mod $name_ident {
+                use super::*;
+
+                pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory {
+                    name: $name_str,
+                    unique_id: $name_str,
+                    genesis_is_known: $genesis_is_known,
                 };
+
+                /// A wrapper around `std::include_bytes` which includes a file from a specific network
+                /// directory. Used by upstream crates to import files at compile time.
+                #[macro_export]
+                macro_rules! [<include_ $name_ident _file>] {
+                    ($this_crate: ident, $base_dir: tt, $filename: tt) => {
+                        include_bytes!(concat!(
+                            $base_dir,
+                            "/",
+                            $this_crate::predefined_networks_dir!(),
+                            "/",
+                            $name_str,
+                            "/",
+                            $filename
+                        ))
+                    };
+                }
             }
         }
     };
 }
 
-define_net!(pyrmont, include_pyrmont_file, "pyrmont", true);
+/// Creates a `HardcodedNet` definition for some network.
+#[macro_export]
+macro_rules! define_net {
+    ($this_crate: tt, $mod: ident, $include_file: tt) => {{
+        use $this_crate::$mod::ETH2_NET_DIR;
+
+        $this_crate::HardcodedNet {
+            name: ETH2_NET_DIR.name,
+            genesis_is_known: ETH2_NET_DIR.genesis_is_known,
+            config: $this_crate::$include_file!($this_crate, "../", "config.yaml"),
+            deploy_block: $this_crate::$include_file!($this_crate, "../", "deploy_block.txt"),
+            boot_enr: $this_crate::$include_file!($this_crate, "../", "boot_enr.yaml"),
+            genesis_state_bytes: $this_crate::$include_file!($this_crate, "../", "genesis.ssz"),
+        }
+    }};
+}
+
+/// Calls `define_net` on a list of networks, and then defines two more lists:
+///
+/// - `HARDCODED_NETS`: a list of all the networks defined by this macro.
+/// - `HARDCODED_NET_NAMES`: a list of the *names* of the networks defined by this macro.
+#[macro_export]
+macro_rules! define_nets {
+    ($this_crate: ident, $($name_ident: ident, $name_str: tt,)+) => {
+        $this_crate::paste! {
+            $(
+            const [<$name_ident:upper>]: $this_crate::HardcodedNet = $this_crate::define_net!($this_crate, $name_ident, [<include_ $name_ident _file>]);
+            )+
+            const HARDCODED_NETS: &[$this_crate::HardcodedNet] = &[$([<$name_ident:upper>],)+];
+            pub const HARDCODED_NET_NAMES: &[&'static str] = &[$($name_str,)+];
+        }
+    };
+}
+
+/// The canonical macro for defining built-in network configurations.
+///
+/// This macro will provide:
+///
+/// - An `Eth2NetArchiveAndDirectory` for each network.
+/// - `ETH2_NET_DIRS`: a list of all the above `Eth2NetArchiveAndDirectory`.
+/// - The `instantiate_hardcoded_nets` macro (see its documentation).
+///
+/// ## Design Justification
+///
+/// Ultimately, this macro serves as a single list of all the networks. The reason it is structured
+/// in such a complex web-of-macros way is because two requirements of built-in (hard-coded) networks:
+///
+/// 1. We must use `std::include_bytes!` to "bake" arbitrary bytes (genesis states, etc) into the binary.
+/// 2. We must use a `build.rs` script to decompress the genesis state from a zip file, before we
+///    can include those bytes.
+///
+/// Because of these two constraints, we must first define all of the networks and the paths to
+/// their files in this crate. Then, we must use another crate (`eth2_network_configs`) to run a
+/// `build.rs` which will unzip the genesis states. Then, that `eth2_network_configs` crate can
+/// perform the final step of using `std::include_bytes` to bake the files (bytes) into the binary.
+macro_rules! define_hardcoded_nets {
+    ($(($name_ident: ident, $name_str: tt, $genesis_is_known: ident)),+) => {
+        paste! {
+            $(
+            define_archive!($name_ident, $name_str, $genesis_is_known);
+            )+
+
+            pub const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[$($name_ident::ETH2_NET_DIR,)+];
 
-define_net!(mainnet, include_mainnet_file, "mainnet", true);
+            /// This macro is designed to be called by an external crate. When called, it will
+            /// define in that external crate:
+            ///
+            /// - A `HardcodedNet` for each network.
+            /// - `HARDCODED_NETS`: a list of all the above `HardcodedNet`.
+            /// - `HARDCODED_NET_NAMES`: a list of all the names of the above `HardcodedNet` (as `&str`).
+            #[macro_export]
+            macro_rules! instantiate_hardcoded_nets {
+                ($this_crate: ident) => {
+                    $this_crate::define_nets!($this_crate, $($name_ident, $name_str,)+);
+                }
+            }
+        }
+    };
+}
 
-define_net!(prater, include_prater_file, "prater", true);
+// Add a new "built-in" network by adding it to the list below.
+//
+// The last entry must not end with a comma, otherwise compilation will fail.
+//
+// This is the canonical place for defining the built-in network configurations that are present in
+// the `common/eth2_network_config/built_in_network_configs` directory.
+//
+// Each net is defined as a three-tuple:
+//
+// 0. The name of the testnet as an "ident" (i.e. something that can be a Rust variable name).
+// 1. The human-friendly name of the testnet (i.e. usually with "-" instead of "_").
+// 2. A bool indicating if the genesis state is known and present as a `genesis.ssz.zip`.
+//
+// The directory containing the testnet files should match the human-friendly name (element 1).
+define_hardcoded_nets!(
+    (mainnet, "mainnet", GENESIS_STATE_IS_KNOWN),
+    (pyrmont, "pyrmont", GENESIS_STATE_IS_KNOWN),
+    (prater, "prater", GENESIS_STATE_IS_KNOWN),
+    (altair_devnet_0, "altair-devnet-0", GENESIS_STATE_IS_KNOWN),
+    (altair_devnet_1, "altair-devnet-1", GENESIS_STATE_IS_KNOWN),
+    (altair_devnet_2, "altair-devnet-2", GENESIS_STATE_IS_KNOWN)
+);

common/eth2_network_config/build.rs

@@ -1,15 +1,9 @@
 //! Extracts zipped genesis states on first run.
-use eth2_config::{mainnet, prater, pyrmont, Eth2NetArchiveAndDirectory, GENESIS_FILE_NAME};
+use eth2_config::{Eth2NetArchiveAndDirectory, ETH2_NET_DIRS, GENESIS_FILE_NAME};
 use std::fs::File;
 use std::io;
 use zip::ZipArchive;
 
-const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[
-    mainnet::ETH2_NET_DIR,
-    pyrmont::ETH2_NET_DIR,
-    prater::ETH2_NET_DIR,
-];
-
 fn main() {
     for network in ETH2_NET_DIRS {
         match uncompress_state(network) {

common/eth2_network_config/built_in_network_configs/altair-devnet-0/boot_enr.yaml

@@ -0,0 +1,3 @@
+# EF bootnode
+# /ip4/3.125.155.250/udp/9000/p2p/16Uiu2HAm8JSAXCaRruMvWjmY4bdJQhGc6a73VdRxCEpKwp9q56N1
+- enr:-Ku4QOcLld1mPvi9bXxN468zT5ZAX6zHFABOgm6-G76AFaC8C7yOAocpOBgRUKvuxLp-EBt3IqpOLUBZcTiou6VYmtoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpBY9DG-GVBGmf__________gmlkgnY0gmlwhAN9m_qJc2VjcDI1NmsxoQK_TT5hHmoWJpZ4aSv1bdoD5noop1fWU54qz4Tj53t_1oN1ZHCCIyg

common/eth2_network_config/built_in_network_configs/altair-devnet-0/config.yaml

@@ -0,0 +1,73 @@
+# Altair interop devnet config
+
+# Extends the mainnet preset
+PRESET_BASE: 'mainnet'
+
+CONFIG_NAME: "altair-devnet-0"
+
+# Genesis
+# ---------------------------------------------------------------
+MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 10000
+# July 7, 2021, 12pm UTC
+MIN_GENESIS_TIME: 1625659200
+# Location for the altair star is 19h 50m 46.99855
+GENESIS_FORK_VERSION: 0x19504699
+# 86400 seconds (1 day)
+GENESIS_DELAY: 86400
+
+
+# Forking
+# ---------------------------------------------------------------
+# Some forks are disabled for now:
+#  - These may be re-assigned to another fork-version later
+#  - Temporarily set to max uint64 value: 2**64 - 1
+
+# Altair
+ALTAIR_FORK_VERSION: 0x01000000
+ALTAIR_FORK_EPOCH: 10
+# Merge
+MERGE_FORK_VERSION: 0x02000000
+MERGE_FORK_EPOCH: 18446744073709551615
+# Sharding
+SHARDING_FORK_VERSION: 0x03000000
+SHARDING_FORK_EPOCH: 18446744073709551615
+
+# TBD, 2**32 is a placeholder. Merge transition approach is in active R&D.
+MIN_ANCHOR_POW_BLOCK_DIFFICULTY: 4294967296
+
+
+# Time parameters
+# ---------------------------------------------------------------
+# 12 seconds
+SECONDS_PER_SLOT: 12
+# 14 (estimate from Eth1 mainnet)
+SECONDS_PER_ETH1_BLOCK: 14
+# 2**8 (= 256) epochs ~27 hours
+MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
+# 2**8 (= 256) epochs ~27 hours
+SHARD_COMMITTEE_PERIOD: 256
+# 2**11 (= 2,048) Eth1 blocks ~8 hours
+ETH1_FOLLOW_DISTANCE: 2048
+
+
+# Validator cycle
+# ---------------------------------------------------------------
+# 2**2 (= 4)
+INACTIVITY_SCORE_BIAS: 4
+# 2**4 (= 16)
+INACTIVITY_SCORE_RECOVERY_RATE: 16
+# 2**4 * 10**9 (= 16,000,000,000) Gwei
+EJECTION_BALANCE: 16000000000
+# 2**2 (= 4)
+MIN_PER_EPOCH_CHURN_LIMIT: 4
+# 2**16 (= 65,536)
+CHURN_LIMIT_QUOTIENT: 65536
+
+
+# Deposit contract
+# ---------------------------------------------------------------
+# Ethereum Goerli testnet
+DEPOSIT_CHAIN_ID: 5
+DEPOSIT_NETWORK_ID: 5
+DEPOSIT_CONTRACT_ADDRESS: 0x3fe221a147E62eC697723fBfB4dd9505F4F966c7
+

common/eth2_network_config/built_in_network_configs/altair-devnet-0/deploy_block.txt

@@ -0,0 +1 @@
+5088851

common/eth2_network_config/built_in_network_configs/altair-devnet-1/boot_enr.yaml

@@ -0,0 +1,3 @@
+# EF bootnode
+# /ip4/18.192.182.103/udp/9000/p2p/16Uiu2HAmVHXRwinNmyfyN9jVTR7wbKuGd8dEkSLX3iTQThpMNAXi
+- enr:-Ku4QJTSfjugOE_5xyEDYceUlHTd8QiP1QF5q4o6APwk9V7QXV6POQKfhXixhLNMI-_-tGRRorbzR_vMAFe_vQT5XWYBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCFITryGVBHAP__________gmlkgnY0gmlwhBLAtmeJc2VjcDI1NmsxoQP3FwrhFYB60djwRjAoOjttq6du94DtkQuaN99wvgqaIYN1ZHCCIyg