Upgrade to v1.4.0-beta.3 (#4862)

## Issue Addressed

Makes lighthouse compliant with new kzg changes in https://github.com/ethereum/consensus-specs/releases/tag/v1.4.0-beta.3

## Proposed Changes

1. Adds new official trusted setup
2. Refactors kzg to match upstream changes in ethereum/c-kzg-4844#377
3. Updates pre-generated `BlobBundle` to work with official trusted setup. ~~Using json here instead of ssz to account for different value of `MaxBlobCommitmentsPerBlock` in minimal and mainnet. By using json, we can just use one pre generated bundle for both minimal and mainnet. Size of 2 separate ssz bundles is approximately equal to one json bundle cc @jimmygchen~~ 
Dunno what I was doing, ssz works without any issues  
4. Stores trusted_setup as just bytes in eth2_network_config so that we don't have kzg dependency in that lib and in lcli. 


Co-authored-by: realbigsean <[email protected]>
Co-authored-by: realbigsean <[email protected]>

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -481,7 +481,7 @@ pub struct BeaconChain<T: BeaconChainTypes> {
     /// they are collected and combined.
     pub data_availability_checker: Arc<DataAvailabilityChecker<T>>,
     /// The KZG trusted setup used by this chain.
-    pub kzg: Option<Arc<Kzg<<T::EthSpec as EthSpec>::Kzg>>>,
+    pub kzg: Option<Arc<Kzg>>,
 }
 
 type BeaconBlockAndState<T, Payload> = (

beacon_node/beacon_chain/src/blob_verification.rs

@@ -499,7 +499,7 @@ impl<T: EthSpec> KzgVerifiedBlob<T> {
 /// Returns an error if the kzg verification check fails.
 pub fn verify_kzg_for_blob<T: EthSpec>(
     blob: Arc<BlobSidecar<T>>,
-    kzg: &Kzg<T::Kzg>,
+    kzg: &Kzg,
 ) -> Result<KzgVerifiedBlob<T>, AvailabilityCheckError> {
     let _timer = crate::metrics::start_timer(&crate::metrics::KZG_VERIFICATION_SINGLE_TIMES);
     if validate_blob::<T>(kzg, &blob.blob, blob.kzg_commitment, blob.kzg_proof)
@@ -518,7 +518,7 @@ pub fn verify_kzg_for_blob<T: EthSpec>(
 /// in a loop since this function kzg verifies a list of blobs more efficiently.
 pub fn verify_kzg_for_blob_list<T: EthSpec>(
     blob_list: &BlobSidecarList<T>,
-    kzg: &Kzg<T::Kzg>,
+    kzg: &Kzg,
 ) -> Result<(), AvailabilityCheckError> {
     let _timer = crate::metrics::start_timer(&crate::metrics::KZG_VERIFICATION_BATCH_TIMES);
     let (blobs, (commitments, proofs)): (Vec<_>, (Vec<_>, Vec<_>)) = blob_list

beacon_node/beacon_chain/src/data_availability_checker.rs

@@ -50,7 +50,7 @@ pub struct DataAvailabilityChecker<T: BeaconChainTypes> {
     processing_cache: RwLock<ProcessingCache<T::EthSpec>>,
     availability_cache: Arc<OverflowLRUCache<T>>,
     slot_clock: T::SlotClock,
-    kzg: Option<Arc<Kzg<<T::EthSpec as EthSpec>::Kzg>>>,
+    kzg: Option<Arc<Kzg>>,
     log: Logger,
     spec: ChainSpec,
 }
@@ -79,7 +79,7 @@ impl<T: EthSpec> Debug for Availability<T> {
 impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
     pub fn new(
         slot_clock: T::SlotClock,
-        kzg: Option<Arc<Kzg<<T::EthSpec as EthSpec>::Kzg>>>,
+        kzg: Option<Arc<Kzg>>,
         store: BeaconStore<T>,
         log: &Logger,
         spec: ChainSpec,

beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs

@@ -970,6 +970,7 @@ mod test {
     ) -> (
         BeaconChainHarness<DiskHarnessType<E>>,
         Arc<OverflowLRUCache<T>>,
+        TempDir,
     )
     where
         E: EthSpec,
@@ -984,15 +985,15 @@ mod test {
             OverflowLRUCache::<T>::new(capacity, test_store, spec.clone())
                 .expect("should create cache"),
         );
-        (harness, cache)
+        (harness, cache, chain_db_path)
     }
 
     #[tokio::test]
     async fn overflow_cache_test_insert_components() {
         type E = MinimalEthSpec;
         type T = DiskHarnessType<E>;
         let capacity = 4;
-        let (harness, cache) = setup_harness_and_cache::<E, T>(capacity).await;
+        let (harness, cache, _path) = setup_harness_and_cache::<E, T>(capacity).await;
 
         let (pending_block, blobs) = availability_pending_block(&harness).await;
         let root = pending_block.import_data.block_root;
@@ -1104,7 +1105,7 @@ mod test {
         type E = MinimalEthSpec;
         type T = DiskHarnessType<E>;
         let capacity = 4;
-        let (harness, cache) = setup_harness_and_cache::<E, T>(capacity).await;
+        let (harness, cache, _path) = setup_harness_and_cache::<E, T>(capacity).await;
 
         let mut pending_blocks = VecDeque::new();
         let mut pending_blobs = VecDeque::new();
@@ -1255,7 +1256,7 @@ mod test {
         type E = MinimalEthSpec;
         type T = DiskHarnessType<E>;
         let capacity = E::slots_per_epoch() as usize;
-        let (harness, cache) = setup_harness_and_cache::<E, T>(capacity).await;
+        let (harness, cache, _path) = setup_harness_and_cache::<E, T>(capacity).await;
 
         let n_epochs = 4;
         let mut pending_blocks = VecDeque::new();
@@ -1395,7 +1396,7 @@ mod test {
         type E = MinimalEthSpec;
         type T = DiskHarnessType<E>;
         let capacity = E::slots_per_epoch() as usize;
-        let (harness, cache) = setup_harness_and_cache::<E, T>(capacity).await;
+        let (harness, cache, _path) = setup_harness_and_cache::<E, T>(capacity).await;
 
         let n_epochs = 4;
         let mut pending_blocks = VecDeque::new();
@@ -1573,7 +1574,7 @@ mod test {
         type E = MinimalEthSpec;
         type T = DiskHarnessType<E>;
         let capacity = STATE_LRU_CAPACITY * 2;
-        let (harness, cache) = setup_harness_and_cache::<E, T>(capacity).await;
+        let (harness, cache, _path) = setup_harness_and_cache::<E, T>(capacity).await;
 
         let mut pending_blocks = VecDeque::new();
         let mut states = Vec::new();

beacon_node/beacon_chain/src/kzg_utils.rs

@@ -1,31 +1,26 @@
-use kzg::{Error as KzgError, Kzg, KzgPreset};
+use kzg::{Blob as KzgBlob, Error as KzgError, Kzg};
 use types::{Blob, EthSpec, Hash256, KzgCommitment, KzgProof};
 
 /// Converts a blob ssz List object to an array to be used with the kzg
 /// crypto library.
-fn ssz_blob_to_crypto_blob<T: EthSpec>(
-    blob: &Blob<T>,
-) -> Result<<<T as EthSpec>::Kzg as KzgPreset>::Blob, KzgError> {
-    T::blob_from_bytes(blob.as_ref())
+fn ssz_blob_to_crypto_blob<T: EthSpec>(blob: &Blob<T>) -> Result<KzgBlob, KzgError> {
+    KzgBlob::from_bytes(blob.as_ref())
 }
 
 /// Validate a single blob-commitment-proof triplet from a `BlobSidecar`.
 pub fn validate_blob<T: EthSpec>(
-    kzg: &Kzg<T::Kzg>,
+    kzg: &Kzg,
     blob: &Blob<T>,
     kzg_commitment: KzgCommitment,
     kzg_proof: KzgProof,
 ) -> Result<bool, KzgError> {
-    kzg.verify_blob_kzg_proof(
-        &ssz_blob_to_crypto_blob::<T>(blob)?,
-        kzg_commitment,
-        kzg_proof,
-    )
+    let kzg_blob = ssz_blob_to_crypto_blob::<T>(blob)?;
+    kzg.verify_blob_kzg_proof(&kzg_blob, kzg_commitment, kzg_proof)
 }
 
 /// Validate a batch of blob-commitment-proof triplets from multiple `BlobSidecars`.
 pub fn validate_blobs<T: EthSpec>(
-    kzg: &Kzg<T::Kzg>,
+    kzg: &Kzg,
     expected_kzg_commitments: &[KzgCommitment],
     blobs: Vec<&Blob<T>>,
     kzg_proofs: &[KzgProof],
@@ -40,36 +35,38 @@ pub fn validate_blobs<T: EthSpec>(
 
 /// Compute the kzg proof given an ssz blob and its kzg commitment.
 pub fn compute_blob_kzg_proof<T: EthSpec>(
-    kzg: &Kzg<T::Kzg>,
+    kzg: &Kzg,
     blob: &Blob<T>,
     kzg_commitment: KzgCommitment,
 ) -> Result<KzgProof, KzgError> {
-    // Avoid this blob clone
-    kzg.compute_blob_kzg_proof(&ssz_blob_to_crypto_blob::<T>(blob)?, kzg_commitment)
+    let kzg_blob = ssz_blob_to_crypto_blob::<T>(blob)?;
+    kzg.compute_blob_kzg_proof(&kzg_blob, kzg_commitment)
 }
 
 /// Compute the kzg commitment for a given blob.
 pub fn blob_to_kzg_commitment<T: EthSpec>(
-    kzg: &Kzg<T::Kzg>,
+    kzg: &Kzg,
     blob: &Blob<T>,
 ) -> Result<KzgCommitment, KzgError> {
-    kzg.blob_to_kzg_commitment(&ssz_blob_to_crypto_blob::<T>(blob)?)
+    let kzg_blob = ssz_blob_to_crypto_blob::<T>(blob)?;
+    kzg.blob_to_kzg_commitment(&kzg_blob)
 }
 
 /// Compute the kzg proof for a given blob and an evaluation point z.
 pub fn compute_kzg_proof<T: EthSpec>(
-    kzg: &Kzg<T::Kzg>,
+    kzg: &Kzg,
     blob: &Blob<T>,
     z: Hash256,
 ) -> Result<(KzgProof, Hash256), KzgError> {
     let z = z.0.into();
-    kzg.compute_kzg_proof(&ssz_blob_to_crypto_blob::<T>(blob)?, &z)
+    let kzg_blob = ssz_blob_to_crypto_blob::<T>(blob)?;
+    kzg.compute_kzg_proof(&kzg_blob, &z)
         .map(|(proof, z)| (proof, Hash256::from_slice(&z.to_vec())))
 }
 
 /// Verify a `kzg_proof` for a `kzg_commitment` that evaluating a polynomial at `z` results in `y`
 pub fn verify_kzg_proof<T: EthSpec>(
-    kzg: &Kzg<T::Kzg>,
+    kzg: &Kzg,
     kzg_commitment: KzgCommitment,
     kzg_proof: KzgProof,
     z: Hash256,

beacon_node/beacon_chain/src/test_utils.rs

@@ -16,6 +16,7 @@ use crate::{
 };
 use bls::get_withdrawal_credentials;
 use eth2::types::SignedBlockContentsTuple;
+use eth2_network_config::TRUSTED_SETUP_BYTES;
 use execution_layer::test_utils::generate_genesis_header;
 use execution_layer::{
     auth::JwtKey,
@@ -492,10 +493,9 @@ where
             .validator_keypairs
             .expect("cannot build without validator keypairs");
         let chain_config = self.chain_config.unwrap_or_default();
-        let trusted_setup: TrustedSetup =
-            serde_json::from_reader(eth2_network_config::get_trusted_setup::<E::Kzg>())
-                .map_err(|e| format!("Unable to read trusted setup file: {}", e))
-                .unwrap();
+        let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP_BYTES)
+            .map_err(|e| format!("Unable to read trusted setup file: {}", e))
+            .unwrap();
 
         let mut builder = BeaconChainBuilder::new(self.eth_spec_instance)
             .logger(log.clone())
@@ -571,10 +571,9 @@ pub fn mock_execution_layer_from_parts<T: EthSpec>(
         HARNESS_GENESIS_TIME + spec.seconds_per_slot * T::slots_per_epoch() * epoch.as_u64()
     });
 
-    let trusted_setup: TrustedSetup =
-        serde_json::from_reader(eth2_network_config::get_trusted_setup::<T::Kzg>())
-            .map_err(|e| format!("Unable to read trusted setup file: {}", e))
-            .expect("should have trusted setup");
+    let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP_BYTES)
+        .map_err(|e| format!("Unable to read trusted setup file: {}", e))
+        .expect("should have trusted setup");
     let kzg = Kzg::new_from_trusted_setup(trusted_setup).expect("should create kzg");
 
     MockExecutionLayer::new(

beacon_node/beacon_chain/tests/store_tests.rs

@@ -14,7 +14,7 @@ use beacon_chain::{
     migrate::MigratorConfig, BeaconChain, BeaconChainError, BeaconChainTypes, BeaconSnapshot,
     BlockError, ChainConfig, NotifyExecutionLayer, ServerSentEventHandler, WhenSlotSkipped,
 };
-use eth2_network_config::get_trusted_setup;
+use eth2_network_config::TRUSTED_SETUP_BYTES;
 use kzg::TrustedSetup;
 use lazy_static::lazy_static;
 use logging::test_logger;
@@ -2154,10 +2154,9 @@ async fn weak_subjectivity_sync_test(slots: Vec<Slot>, checkpoint_slot: Slot) {
     let store = get_store(&temp2);
     let spec = test_spec::<E>();
     let seconds_per_slot = spec.seconds_per_slot;
-    let trusted_setup: TrustedSetup =
-        serde_json::from_reader(get_trusted_setup::<<E as EthSpec>::Kzg>())
-            .map_err(|e| println!("Unable to read trusted setup file: {}", e))
-            .unwrap();
+    let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP_BYTES)
+        .map_err(|e| println!("Unable to read trusted setup file: {}", e))
+        .unwrap();
 
     let mock =
         mock_execution_layer_from_parts(&harness.spec, harness.runtime.task_executor.clone(), None);