Add Experimental QUIC support (#4577)

## Issue Addressed

#4402 

## Proposed Changes

This PR adds QUIC support to Lighthouse. As this is not officially spec'd this will only work between lighthouse <-> lighthouse connections. We attempt a QUIC connection (if the node advertises it) and if it fails we fallback to TCP. 

This should be a backwards compatible modification. We want to test this functionality on live networks to observe any improvements in bandwidth/latency.

NOTE: This also removes the websockets transport as I believe no one is really using it. It should be mentioned in our release however.


Co-authored-by: João Oliveira <[email protected]>

beacon_node/client/src/lib.rs

@@ -46,20 +46,6 @@ impl<T: BeaconChainTypes> Client<T> {
         self.http_metrics_listen_addr
     }
 
-    /// Returns the ipv4 port of the client's libp2p stack, if it was started.
-    pub fn libp2p_listen_ipv4_port(&self) -> Option<u16> {
-        self.network_globals
-            .as_ref()
-            .and_then(|n| n.listen_port_tcp4())
-    }
-
-    /// Returns the ipv6 port of the client's libp2p stack, if it was started.
-    pub fn libp2p_listen_ipv6_port(&self) -> Option<u16> {
-        self.network_globals
-            .as_ref()
-            .and_then(|n| n.listen_port_tcp6())
-    }
-
     /// Returns the list of libp2p addresses the client is listening to.
     pub fn libp2p_listen_addresses(&self) -> Option<Vec<Multiaddr>> {
         self.network_globals.as_ref().map(|n| n.listen_multiaddrs())

beacon_node/http_api/src/lib.rs

@@ -2825,12 +2825,8 @@ pub fn serve<T: BeaconChainTypes>(
                     })?;
 
                     if let Some(peer_info) = network_globals.peers.read().peer_info(&peer_id) {
-                        let address = if let Some(socket_addr) = peer_info.seen_addresses().next() {
-                            let mut addr = lighthouse_network::Multiaddr::from(socket_addr.ip());
-                            addr.push(lighthouse_network::multiaddr::Protocol::Tcp(
-                                socket_addr.port(),
-                            ));
-                            addr.to_string()
+                        let address = if let Some(multiaddr) = peer_info.seen_multiaddrs().next() {
+                            multiaddr.to_string()
                         } else if let Some(addr) = peer_info.listening_addresses().first() {
                             addr.to_string()
                         } else {
@@ -2878,13 +2874,8 @@ pub fn serve<T: BeaconChainTypes>(
                         .peers()
                         .for_each(|(peer_id, peer_info)| {
                             let address =
-                                if let Some(socket_addr) = peer_info.seen_addresses().next() {
-                                    let mut addr =
-                                        lighthouse_network::Multiaddr::from(socket_addr.ip());
-                                    addr.push(lighthouse_network::multiaddr::Protocol::Tcp(
-                                        socket_addr.port(),
-                                    ));
-                                    addr.to_string()
+                                if let Some(multiaddr) = peer_info.seen_multiaddrs().next() {
+                                    multiaddr.to_string()
                                 } else if let Some(addr) = peer_info.listening_addresses().first() {
                                     addr.to_string()
                                 } else {

beacon_node/http_api/src/test_utils.rs

@@ -152,8 +152,6 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
     let enr = EnrBuilder::new("v4").build(&enr_key).unwrap();
     let network_globals = Arc::new(NetworkGlobals::new(
         enr.clone(),
-        Some(TCP_PORT),
-        None,
         meta_data,
         vec![],
         false,

beacon_node/lighthouse_network/Cargo.toml

@@ -44,12 +44,13 @@ prometheus-client = "0.21.0"
 unused_port = { path = "../../common/unused_port" }
 delay_map = "0.3.0"
 void = "1"
+libp2p-quic= { version = "0.9.2", features=["tokio"]}
 libp2p-mplex = "0.40.0"
 
 [dependencies.libp2p]
 version = "0.52"
 default-features = false
-features = ["websocket", "identify", "yamux", "noise", "gossipsub", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa"]
+features = ["identify", "yamux", "noise", "gossipsub", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa"]
 
 [dev-dependencies]
 slog-term = "2.6.0"
@@ -59,6 +60,3 @@ exit-future = "0.2.0"
 void = "1"
 quickcheck = "0.9.2"
 quickcheck_macros = "0.9.1"
-
-[features]
-libp2p-websocket = []

beacon_node/lighthouse_network/src/config.rs

@@ -58,18 +58,24 @@ pub struct Config {
     /// that no discovery address has been set in the CLI args.
     pub enr_address: (Option<Ipv4Addr>, Option<Ipv6Addr>),
 
-    /// The udp4 port to broadcast to peers in order to reach back for discovery.
+    /// The udp ipv4 port to broadcast to peers in order to reach back for discovery.
     pub enr_udp4_port: Option<u16>,
 
-    /// The tcp4 port to broadcast to peers in order to reach back for libp2p services.
+    /// The quic ipv4 port to broadcast to peers in order to reach back for libp2p services.
+    pub enr_quic4_port: Option<u16>,
+
+    /// The tcp ipv4 port to broadcast to peers in order to reach back for libp2p services.
     pub enr_tcp4_port: Option<u16>,
 
-    /// The udp6 port to broadcast to peers in order to reach back for discovery.
+    /// The udp ipv6 port to broadcast to peers in order to reach back for discovery.
     pub enr_udp6_port: Option<u16>,
 
-    /// The tcp6 port to broadcast to peers in order to reach back for libp2p services.
+    /// The tcp ipv6 port to broadcast to peers in order to reach back for libp2p services.
     pub enr_tcp6_port: Option<u16>,
 
+    /// The quic ipv6 port to broadcast to peers in order to reach back for libp2p services.
+    pub enr_quic6_port: Option<u16>,
+
     /// Target number of connected peers.
     pub target_peers: usize,
 
@@ -102,6 +108,9 @@ pub struct Config {
     /// Disables the discovery protocol from starting.
     pub disable_discovery: bool,
 
+    /// Disables quic support.
+    pub disable_quic_support: bool,
+
     /// Attempt to construct external port mappings with UPnP.
     pub upnp_enabled: bool,
 
@@ -149,57 +158,76 @@ impl Config {
     /// Sets the listening address to use an ipv4 address. The discv5 ip_mode and table filter are
     /// adjusted accordingly to ensure addresses that are present in the enr are globally
     /// reachable.
-    pub fn set_ipv4_listening_address(&mut self, addr: Ipv4Addr, tcp_port: u16, udp_port: u16) {
+    pub fn set_ipv4_listening_address(
+        &mut self,
+        addr: Ipv4Addr,
+        tcp_port: u16,
+        disc_port: u16,
+        quic_port: u16,
+    ) {
         self.listen_addresses = ListenAddress::V4(ListenAddr {
             addr,
-            udp_port,
+            disc_port,
+            quic_port,
             tcp_port,
         });
-        self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), udp_port);
+        self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), disc_port);
         self.discv5_config.table_filter = |enr| enr.ip4().as_ref().map_or(false, is_global_ipv4)
     }
 
     /// Sets the listening address to use an ipv6 address. The discv5 ip_mode and table filter is
     /// adjusted accordingly to ensure addresses that are present in the enr are globally
     /// reachable.
-    pub fn set_ipv6_listening_address(&mut self, addr: Ipv6Addr, tcp_port: u16, udp_port: u16) {
+    pub fn set_ipv6_listening_address(
+        &mut self,
+        addr: Ipv6Addr,
+        tcp_port: u16,
+        disc_port: u16,
+        quic_port: u16,
+    ) {
         self.listen_addresses = ListenAddress::V6(ListenAddr {
             addr,
-            udp_port,
+            disc_port,
+            quic_port,
             tcp_port,
         });
 
-        self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), udp_port);
+        self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), disc_port);
         self.discv5_config.table_filter = |enr| enr.ip6().as_ref().map_or(false, is_global_ipv6)
     }
 
     /// Sets the listening address to use both an ipv4 and ipv6 address. The discv5 ip_mode and
     /// table filter is adjusted accordingly to ensure addresses that are present in the enr are
     /// globally reachable.
+    #[allow(clippy::too_many_arguments)]
     pub fn set_ipv4_ipv6_listening_addresses(
         &mut self,
         v4_addr: Ipv4Addr,
         tcp4_port: u16,
-        udp4_port: u16,
+        disc4_port: u16,
+        quic4_port: u16,
         v6_addr: Ipv6Addr,
         tcp6_port: u16,
-        udp6_port: u16,
+        disc6_port: u16,
+        quic6_port: u16,
     ) {
         self.listen_addresses = ListenAddress::DualStack(
             ListenAddr {
                 addr: v4_addr,
-                udp_port: udp4_port,
+                disc_port: disc4_port,
+                quic_port: quic4_port,
                 tcp_port: tcp4_port,
             },
             ListenAddr {
                 addr: v6_addr,
-                udp_port: udp6_port,
+                disc_port: disc6_port,
+                quic_port: quic6_port,
                 tcp_port: tcp6_port,
             },
         );
         self.discv5_config.listen_config = discv5::ListenConfig::default()
-            .with_ipv4(v4_addr, udp4_port)
-            .with_ipv6(v6_addr, udp6_port);
+            .with_ipv4(v4_addr, disc4_port)
+            .with_ipv6(v6_addr, disc6_port);
 
         self.discv5_config.table_filter = |enr| match (&enr.ip4(), &enr.ip6()) {
             (None, None) => false,
@@ -213,27 +241,32 @@ impl Config {
         match listen_addr {
             ListenAddress::V4(ListenAddr {
                 addr,
-                udp_port,
+                disc_port,
+                quic_port,
                 tcp_port,
-            }) => self.set_ipv4_listening_address(addr, tcp_port, udp_port),
+            }) => self.set_ipv4_listening_address(addr, tcp_port, disc_port, quic_port),
             ListenAddress::V6(ListenAddr {
                 addr,
-                udp_port,
+                disc_port,
+                quic_port,
                 tcp_port,
-            }) => self.set_ipv6_listening_address(addr, tcp_port, udp_port),
+            }) => self.set_ipv6_listening_address(addr, tcp_port, disc_port, quic_port),
             ListenAddress::DualStack(
                 ListenAddr {
                     addr: ip4addr,
-                    udp_port: udp4_port,
+                    disc_port: disc4_port,
+                    quic_port: quic4_port,
                     tcp_port: tcp4_port,
                 },
                 ListenAddr {
                     addr: ip6addr,
-                    udp_port: udp6_port,
+                    disc_port: disc6_port,
+                    quic_port: quic6_port,
                     tcp_port: tcp6_port,
                 },
             ) => self.set_ipv4_ipv6_listening_addresses(
-                ip4addr, tcp4_port, udp4_port, ip6addr, tcp6_port, udp6_port,
+                ip4addr, tcp4_port, disc4_port, quic4_port, ip6addr, tcp6_port, disc6_port,
+                quic6_port,
             ),
         }
     }
@@ -272,7 +305,8 @@ impl Default for Config {
         );
         let listen_addresses = ListenAddress::V4(ListenAddr {
             addr: Ipv4Addr::UNSPECIFIED,
-            udp_port: 9000,
+            disc_port: 9000,
+            quic_port: 9001,
             tcp_port: 9000,
         });
 
@@ -305,10 +339,11 @@ impl Default for Config {
             network_dir,
             listen_addresses,
             enr_address: (None, None),
-
             enr_udp4_port: None,
+            enr_quic4_port: None,
             enr_tcp4_port: None,
             enr_udp6_port: None,
+            enr_quic6_port: None,
             enr_tcp6_port: None,
             target_peers: 50,
             gs_config,
@@ -320,6 +355,7 @@ impl Default for Config {
             disable_peer_scoring: false,
             client_version: lighthouse_version::version_with_platform(),
             disable_discovery: false,
+            disable_quic_support: false,
             upnp_enabled: true,
             network_load: 3,
             private: false,

beacon_node/lighthouse_network/src/discovery/enr.rs

@@ -17,6 +17,8 @@ use std::path::Path;
 use std::str::FromStr;
 use types::{EnrForkId, EthSpec};
 
+use super::enr_ext::{EnrExt, QUIC6_ENR_KEY, QUIC_ENR_KEY};
+
 /// The ENR field specifying the fork id.
 pub const ETH2_ENR_KEY: &str = "eth2";
 /// The ENR field specifying the attestation subnet bitfield.
@@ -142,7 +144,7 @@ pub fn build_or_load_enr<T: EthSpec>(
 
 pub fn create_enr_builder_from_config<T: EnrKey>(
     config: &NetworkConfig,
-    enable_tcp: bool,
+    enable_libp2p: bool,
 ) -> EnrBuilder<T> {
     let mut builder = EnrBuilder::new("v4");
     let (maybe_ipv4_address, maybe_ipv6_address) = &config.enr_address;
@@ -163,7 +165,28 @@ pub fn create_enr_builder_from_config<T: EnrKey>(
         builder.udp6(udp6_port);
     }
 
-    if enable_tcp {
+    if enable_libp2p {
+        // Add QUIC fields to the ENR.
+        // Since QUIC is used as an alternative transport for the libp2p protocols,
+        // the related fields should only be added when both QUIC and libp2p are enabled
+        if !config.disable_quic_support {
+            // If we are listening on ipv4, add the quic ipv4 port.
+            if let Some(quic4_port) = config
+                .enr_quic4_port
+                .or_else(|| config.listen_addrs().v4().map(|v4_addr| v4_addr.quic_port))
+            {
+                builder.add_value(QUIC_ENR_KEY, &quic4_port);
+            }
+
+            // If we are listening on ipv6, add the quic ipv6 port.
+            if let Some(quic6_port) = config
+                .enr_quic6_port
+                .or_else(|| config.listen_addrs().v6().map(|v6_addr| v6_addr.quic_port))
+            {
+                builder.add_value(QUIC6_ENR_KEY, &quic6_port);
+            }
+        }
+
         // If the ENR port is not set, and we are listening over that ip version, use the listening port instead.
         let tcp4_port = config
             .enr_tcp4_port
@@ -218,6 +241,9 @@ fn compare_enr(local_enr: &Enr, disk_enr: &Enr) -> bool {
         // tcp ports must match
         && local_enr.tcp4() == disk_enr.tcp4()
         && local_enr.tcp6() == disk_enr.tcp6()
+        // quic ports must match
+        && local_enr.quic4() == disk_enr.quic4()
+        && local_enr.quic6() == disk_enr.quic6()
         // must match on the same fork
         && local_enr.get(ETH2_ENR_KEY) == disk_enr.get(ETH2_ENR_KEY)
         // take preference over disk udp port if one is not specified