Snap v4 final

gradle/verification-metadata.xml

@@ -54,6 +54,11 @@
             <sha256 value="3e650b44d944cbbf886f2b04fb03f9d829a0b73c084d602ea481ff8571fba8cb" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.fasterxml" name="oss-parent" version="27">
+         <artifact name="oss-parent-27.pom">
+            <sha256 value="b9b8f388fd628057b1249756468b86726c8fd5816ce14d313cb40003a509beeb" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.fasterxml" name="oss-parent" version="38">
          <artifact name="oss-parent-38.pom">
             <sha256 value="c83f8f45dfdca8d0b6b3661c60b3f84780f671b12e06f91ad5d1c1a1d1f966e8" origin="Generated by Gradle"/>
@@ -658,6 +663,32 @@
             <sha256 value="5bd9d261b09a5e0e1d472fdd0d943e4ccafa191612b0f76c2e8f18f471049c27" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="me.champeau.jmh" name="jmh-gradle-plugin" version="0.7.0">
+         <artifact name="jmh-gradle-plugin-0.7.0.jar">
+            <sha256 value="a4cb2052aee8271973f53e02bc94f60ddd4eb20f2eb04bd82008c6eb036d3e50" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="jmh-gradle-plugin-0.7.0.module">
+            <sha256 value="becf0cfd30e5a0195687c339185168553884cc2bafcd4a16b0fd7c15ffb3c56a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="me.champeau.jmh" name="jmh-gradle-plugin" version="0.7.1">
+         <artifact name="jmh-gradle-plugin-0.7.1.jar">
+            <sha256 value="15605b30a146a3e6243335db65b91ed658223a650330be68cf0819c354cbee2a" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="jmh-gradle-plugin-0.7.1.module">
+            <sha256 value="caddafb7ced1ae6bc96a253b67cdd46caf09b41137402443e3708191928eb475" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="me.champeau.jmh" name="me.champeau.jmh.gradle.plugin" version="0.7.0">
+         <artifact name="me.champeau.jmh.gradle.plugin-0.7.0.pom">
+            <sha256 value="818e873701053111c82904f452569b0de72b736959531e7d72024e3c7a15b429" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="me.champeau.jmh" name="me.champeau.jmh.gradle.plugin" version="0.7.1">
+         <artifact name="me.champeau.jmh.gradle.plugin-0.7.1.pom">
+            <sha256 value="5009b99203a5f9e7c9e0d4846cac599982189c944fd134b0162edfe85eae8ab2" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="net.bytebuddy" name="byte-buddy" version="1.12.10">
          <artifact name="byte-buddy-1.12.10.jar">
             <sha256 value="1a1ac9ce65eddcea54ead958387bb0b3863d02a2ffe856ab6a57ac79737c19cf" origin="Generated by Gradle"/>
@@ -1159,6 +1190,14 @@
             <sha256 value="5188ee615c309a06eb3968db8286fe5b9c513a6977a5dbcb919edef5651d5a8d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.lz4" name="lz4-java" version="1.8.0">
+         <artifact name="lz4-java-1.8.0.jar">
+            <sha256 value="d74a3334fb35195009b338a951f918203d6bbca3d1d359033dc33edd1cadc9ef" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="lz4-java-1.8.0.pom">
+            <sha256 value="0db8adb51e132454a5c406debb2f1a0df81f93dd59fbe20cb947102994689034" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.mapdb" name="mapdb" version="2.0-beta13">
          <artifact name="mapdb-2.0-beta13.jar">
             <sha256 value="1268e9ec22ff770ef7e63d7cc72563406ad239422c791acc8b9ee4fdfba0bb1e" origin="Generated by Gradle"/>
@@ -1233,11 +1272,24 @@
             <sha256 value="a96e671816c1ff8803bdec74c9241f025bdfb277da5d2b4ee02266405936f994" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2" name="ow2" version="1.3">
+         <artifact name="ow2-1.3.pom">
+            <sha256 value="51215c67d2c068d8b7d2f6f80f51372a098075deccc448d4bdd7b987ba8328fb" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.ow2" name="ow2" version="1.5">
          <artifact name="ow2-1.5.pom">
             <sha256 value="0f8a1b116e760b8fe6389c51b84e4b07a70fc11082d4f936e453b583dd50b43b" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2.asm" name="asm" version="9.0">
+         <artifact name="asm-9.0.jar">
+            <sha256 value="0df97574914aee92fd349d0cb4e00f3345d45b2c239e0bb50f0a90ead47888e0" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="asm-9.0.module">
+            <sha256 value="8af81096ed3affa39a4729fc900a55b663894911d67c4d4bef0ea424393dd3f9" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.ow2.asm" name="asm" version="9.1">
          <artifact name="asm-9.1.jar">
             <sha256 value="cda4de455fab48ff0bcb7c48b4639447d4de859a7afc30a094a986f0936beba2" origin="Generated by Gradle"/>
@@ -1334,6 +1386,22 @@
             <sha256 value="fb309c5dd2021cc364cc37fe88a1bbc36e4f8c1503a8fd4accc3011f2b630758" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.slf4j" name="slf4j-api" version="1.7.25">
+         <artifact name="slf4j-api-1.7.25.jar">
+            <sha256 value="18c4a0095d5c1da6b817592e767bb23d29dd2f560ad74df75ff3961dbde25b79" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="slf4j-api-1.7.25.pom">
+            <sha256 value="7cd9d7a0b5d93dfd461a148891b43509cf403a9c7f9fb49060d3554df1c81e1e" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.slf4j" name="slf4j-api" version="1.7.30">
+         <artifact name="slf4j-api-1.7.30.jar">
+            <sha256 value="cdba07964d1bb40a0761485c6b1e8c2f8fd9eb1d19c53928ac0d7f9510105c57" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="slf4j-api-1.7.30.pom">
+            <sha256 value="7e0747751e9b67e19dcb5206f04ea22cc03d250c422426402eadd03513f2c314" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.slf4j" name="slf4j-api" version="1.7.36">
          <artifact name="slf4j-api-1.7.36.jar">
             <sha256 value="d3ef575e3e4979678dc01bf1dcce51021493b4d11fb7f1be8ad982877c16a1c0" origin="Generated by Gradle"/>
@@ -1342,6 +1410,16 @@
             <sha256 value="fb046a9c229437928bb11c2d27c8b5d773eb8a25e60cbd253d985210dedc2684" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.slf4j" name="slf4j-parent" version="1.7.25">
+         <artifact name="slf4j-parent-1.7.25.pom">
+            <sha256 value="18f5c52120db036e88d6136f8839c832d074bdda95c756c6f429249d2db54ac6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.slf4j" name="slf4j-parent" version="1.7.30">
+         <artifact name="slf4j-parent-1.7.30.pom">
+            <sha256 value="11647956e48a0c5bfb3ac33f6da7e83f341002b6857efd335a505b687be34b75" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.slf4j" name="slf4j-parent" version="1.7.36">
          <artifact name="slf4j-parent-1.7.36.pom">
             <sha256 value="bb388d37fbcdd3cde64c3cede21838693218dc451f04040c5df360a78ed7e812" origin="Generated by Gradle"/>

rskj-core/build.gradle

@@ -413,4 +413,4 @@ static def amendPathIfNeeded(details) {
         def newPath = originalFile.parentFile.name + "/" + jarName + "_" + originalFile.name
         details.path = newPath
     }
-}
+}

rskj-core/src/main/java/co/rsk/RskContext.java

@@ -194,6 +194,7 @@ public class RskContext implements NodeContext, NodeBootstrapper {
     private SyncProcessor syncProcessor;
     private BlockSyncService blockSyncService;
     private SyncPool syncPool;
+    private SnapshotProcessor snapshotProcessor;
     private Web3 web3;
     private JsonRpcWeb3FilterHandler jsonRpcWeb3FilterHandler;
     private JsonRpcWeb3ServerHandler jsonRpcWeb3ServerHandler;
@@ -863,7 +864,8 @@ public synchronized RskModule getRskModule() {
                     getBlockStore(),
                     getReceiptStore(),
                     getWeb3InformationRetriever(),
-                    getBlockChainFlusher());
+                    getBlockChainFlusher(),
+                    getRskSystemProperties());
         }
 
         return rskModule;
@@ -1458,7 +1460,10 @@ protected synchronized SyncConfiguration buildSyncConfiguration() {
                 rskSystemProperties.getChunkSize(),
                 rskSystemProperties.getMaxRequestedBodies(),
                 rskSystemProperties.getLongSyncLimit(),
-                rskSystemProperties.getTopBest());
+                rskSystemProperties.getTopBest(),
+                rskSystemProperties.isSnapshotSyncEnabled(),
+                rskSystemProperties.getSnapshotChunkTimeout(),
+                rskSystemProperties.getSnapshotSyncLimit());
     }
 
     protected synchronized StateRootHandler buildStateRootHandler() {
@@ -1914,7 +1919,8 @@ protected SyncProcessor getSyncProcessor() {
                     getDifficultyCalculator(),
                     getPeersInformation(),
                     getGenesis(),
-                    getCompositeEthereumListener());
+                    getCompositeEthereumListener(),
+                    getSnapshotProcessor());
         }
 
         return syncProcessor;
@@ -1952,6 +1958,21 @@ private SyncPool getSyncPool() {
         return syncPool;
     }
 
+    private SnapshotProcessor getSnapshotProcessor() {
+        if (snapshotProcessor == null) {
+            snapshotProcessor = new SnapshotProcessor(
+                    getBlockchain(),
+                    getTrieStore(),
+                    getPeersInformation(),
+                    getBlockStore(),
+                    getTransactionPool(),
+                    getRskSystemProperties().getSnapshotChunkSize(),
+                    getRskSystemProperties().isSnapshotParallelEnabled()
+            );
+        }
+        return snapshotProcessor;
+    }
+
     private Web3 getWeb3() {
         if (web3 == null) {
             web3 = buildWeb3();
@@ -2072,6 +2093,7 @@ private NodeMessageHandler getNodeMessageHandler() {
                     getRskSystemProperties(),
                     getNodeBlockProcessor(),
                     getSyncProcessor(),
+                    getSnapshotProcessor(),
                     getChannelManager(),
                     getTransactionGateway(),
                     getPeerScoringManager(),

rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java

@@ -375,6 +375,18 @@ public int getLongSyncLimit() {
         return configFromFiles.getInt("sync.longSyncLimit");
     }
 
+    public boolean isSnapshotSyncEnabled() { return configFromFiles.getBoolean("sync.snapshot.enabled");}
+
+    public int getSnapshotChunkTimeout() {
+        return configFromFiles.getInt("sync.snapshot.chunkRequestTimeout");
+    }
+
+    public boolean isSnapshotParallelEnabled() { return configFromFiles.getBoolean("sync.snapshot.parallel");}
+
+    public int getSnapshotChunkSize() { return configFromFiles.getInt("sync.snapshot.chunkSize");}
+
+    public int getSnapshotSyncLimit() { return configFromFiles.getInt("sync.snapshot.limit");}
+
     // its fixed, cannot be set by config file
     public int getChunkSize() {
         return CHUNK_SIZE;
@@ -490,4 +502,5 @@ private void fetchMethodTimeout(Config configElement, Map<String, Long> methodTi
                 .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey(), Long.parseLong(entry.getValue().toString())))
                 .forEach(entry -> methodTimeoutMap.put(entry.getKey(), entry.getValue()));
     }
+
 }

rskj-core/src/main/java/co/rsk/core/bc/TransactionPoolImpl.java

@@ -153,6 +153,11 @@
         return bestBlock;
     }
 
+    public void setBestBlock(Block bestBlock) {
+        this.bestBlock = bestBlock;
+    }
+
+
     @Override
     public synchronized PendingState getPendingState() {
         return getPendingState(getCurrentRepository());
@@ -314,7 +319,7 @@
 
         //we need to update the bestBlock before calling retractBlock
         //or else the transactions would be validated against outdated account state.
-        this.bestBlock = newBlock;
+        this.setBestBlock(newBlock);
 
         if (fork != null) {
             for (Block blk : fork.getOldBlocks()) {

rskj-core/src/main/java/co/rsk/db/BlocksIndex.java

@@ -88,6 +88,7 @@ public interface BlocksIndex {
 
     void close();
 
+    void remove(long number);
     /**
      * Removes the block with the given number and hash from the storage.
      *

rskj-core/src/main/java/co/rsk/db/MapDBBlocksIndex.java

@@ -166,4 +166,9 @@ public void flush() {
     public void close() {
         indexDB.close();
     }
+
+    @Override
+    public void remove(long number) {
+        this.index.remove(number);
+    }
 }

rskj-core/src/main/java/co/rsk/net/BlockProcessor.java

@@ -19,6 +19,7 @@
 package co.rsk.net;
 
 import co.rsk.net.messages.NewBlockHashesMessage;
+import co.rsk.net.messages.SnapStateChunkRequestMessage;
 import org.ethereum.core.Block;
 import org.ethereum.core.BlockHeader;
 
@@ -63,4 +64,6 @@
     void processSkeletonRequest(Peer sender, long requestId, long startNumber);
 
     boolean canBeIgnoredForUnclesRewards(long blockNumber);
+
+    void processStateChunkRequest(Peer sender, SnapStateChunkRequestMessage snapStateChunkRequestMessage);
 }

rskj-core/src/main/java/co/rsk/net/BlockSyncService.java

@@ -55,6 +55,7 @@ public class BlockSyncService {
     private final BlockNodeInformation nodeInformation; // keep tabs on which nodes know which blocks.
     private final RskSystemProperties config;
     private final BlockValidator blockHeaderValidator;
+    private static final Logger loggerSnapExperiment = LoggerFactory.getLogger("snapExperiment");
 
     // this is tightly coupled with NodeProcessorService and SyncProcessor,
     // and we should use the same objects everywhere to ensure consistency
@@ -205,7 +206,7 @@ private Set<Block> getConnectedBlocks(List<Block> remainingBlocks, Peer sender,
             }
 
             connectionsResult.put(block.getHash(), blockchain.tryToConnect(block));
-
+            loggerSnapExperiment.debug("BlockMessage block executed number: [{}] hash: [{}] from: [{}]", block.getNumber(), block.getPrintableHash(), sender.getPeerNodeID());
             if (BlockUtils.blockInSomeBlockChain(block, blockchain)) {
                 this.store.removeBlock(block);
                 connected.add(block);

rskj-core/src/main/java/co/rsk/net/NodeBlockProcessor.java

@@ -299,6 +299,11 @@ public boolean canBeIgnoredForUnclesRewards(long blockNumber) {
         return blockSyncService.canBeIgnoredForUnclesRewards(blockNumber);
     }
 
+    @Override
+    public void processStateChunkRequest(Peer sender, SnapStateChunkRequestMessage snapStateChunkRequestMessage) {
+
+    }
+
     /**
      *
      * @param skeletonBlockNumber a block number that belongs to the skeleton