queue;
- private final MessageCounter messageCounter = new MessageCounter();
+ private final MessageCounter messageCounter;
private final int messageQueueMaxSize;
private volatile boolean recentIdleTime = false;
@@ -92,16 +89,43 @@ public class NodeMessageHandler implements MessageHandler, InternalService, Runn
* Creates a new node message handler.
*/
public NodeMessageHandler(RskSystemProperties config,
+ BlockProcessor blockProcessor,
+ SyncProcessor syncProcessor,
+ SnapshotProcessor snapshotProcessor,
+ @Nullable ChannelManager channelManager,
+ @Nullable TransactionGateway transactionGateway,
+ @Nullable PeerScoringManager peerScoringManager,
+ StatusResolver statusResolver) {
+ this(
+ config,
+ blockProcessor,
+ syncProcessor,
+ snapshotProcessor,
+ channelManager,
+ transactionGateway,
+ peerScoringManager,
+ statusResolver,
+ null,
+ null
+ );
+ }
+
+ @VisibleForTesting
+ NodeMessageHandler(RskSystemProperties config,
BlockProcessor blockProcessor,
SyncProcessor syncProcessor,
+ SnapshotProcessor snapshotProcessor,
@Nullable ChannelManager channelManager,
@Nullable TransactionGateway transactionGateway,
@Nullable PeerScoringManager peerScoringManager,
- StatusResolver statusResolver) {
+ StatusResolver statusResolver,
+ Thread thread,
+ MessageCounter messageCounter) {
this.config = config;
this.channelManager = channelManager;
this.blockProcessor = blockProcessor;
this.syncProcessor = syncProcessor;
+ this.snapshotProcessor = snapshotProcessor;
this.transactionGateway = transactionGateway;
this.statusResolver = statusResolver;
this.peerScoringManager = peerScoringManager;
@@ -111,13 +135,15 @@ public NodeMessageHandler(RskSystemProperties config,
config.bannedMinerList().stream().map(RskAddress::new).collect(Collectors.toSet())
);
this.messageQueueMaxSize = config.getMessageQueueMaxSize();
- this.thread = new Thread(this, "message handler");
+ this.thread = thread == null ? new Thread(this, "message handler") : thread;
+ this.messageCounter = messageCounter == null ? new MessageCounter() : messageCounter;
}
@VisibleForTesting
NodeMessageHandler(RskSystemProperties config,
BlockProcessor blockProcessor,
SyncProcessor syncProcessor,
+ SnapshotProcessor snapshotProcessor,
@Nullable ChannelManager channelManager,
@Nullable TransactionGateway transactionGateway,
@Nullable PeerScoringManager peerScoringManager,
@@ -127,6 +153,7 @@ public NodeMessageHandler(RskSystemProperties config,
this.channelManager = channelManager;
this.blockProcessor = blockProcessor;
this.syncProcessor = syncProcessor;
+ this.snapshotProcessor = snapshotProcessor;
this.transactionGateway = transactionGateway;
this.statusResolver = statusResolver;
this.peerScoringManager = peerScoringManager;
@@ -137,6 +164,7 @@ public NodeMessageHandler(RskSystemProperties config,
);
this.messageQueueMaxSize = config.getMessageQueueMaxSize();
this.thread = new Thread(this, "message handler");
+ this.messageCounter = new MessageCounter();
}
/**
@@ -151,7 +179,8 @@ public synchronized void processMessage(final Peer sender, @Nonnull final Messag
MessageType messageType = message.getMessageType();
logger.trace("Process message type: {}", messageType);
- MessageVisitor mv = new MessageVisitor(config, blockProcessor, syncProcessor, transactionGateway, peerScoringManager, channelManager, sender);
+ MessageVisitor mv = new MessageVisitor(config, blockProcessor, syncProcessor,
+ snapshotProcessor, transactionGateway, peerScoringManager, channelManager, sender);
message.accept(mv);
}
@@ -258,6 +287,7 @@ void addMessage(Peer sender, Message message, double score, NodeMsgTraceInfo nod
// also, while queue implementation stays unbounded, offer() will never return false
messageCounter.increment(sender);
MessageTask messageTask = new MessageTask(sender, message, score, nodeMsgTraceInfo);
+
boolean messageAdded = this.queue.offer(messageTask);
if (!messageAdded) {
messageCounter.decrement(sender);
@@ -325,7 +355,7 @@ public void run() {
Thread.currentThread().interrupt();
break;
} catch (Exception e) {
- logger.error("Got unexpected error while processing task: {}", task, e);
+ logger.error("Got unexpected error while processing task:", e);
} catch (IllegalAccessError e) { // Usually this is been thrown by DB instances when closed
logger.warn("Message handler got `{}`. Exiting", e.getClass().getSimpleName(), e);
return;
diff --git a/rskj-core/src/main/java/co/rsk/net/Peer.java b/rskj-core/src/main/java/co/rsk/net/Peer.java
index e148600f0a1..b839dfad2ac 100644
--- a/rskj-core/src/main/java/co/rsk/net/Peer.java
+++ b/rskj-core/src/main/java/co/rsk/net/Peer.java
@@ -32,4 +32,5 @@ public interface Peer {
double score(long currentTime, MessageType type);
void imported(boolean best);
+ boolean isSnapCapable();
}
diff --git a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java
new file mode 100644
index 00000000000..862ddc6fed0
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java
@@ -0,0 +1,522 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net;
+
+import co.rsk.config.InternalService;
+import co.rsk.core.BlockDifficulty;
+import co.rsk.net.messages.*;
+import co.rsk.net.sync.*;
+import co.rsk.trie.TrieDTO;
+import co.rsk.trie.TrieDTOInOrderIterator;
+import co.rsk.trie.TrieDTOInOrderRecoverer;
+import co.rsk.trie.TrieStore;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.ethereum.core.Block;
+import org.ethereum.core.Blockchain;
+import org.ethereum.core.TransactionPool;
+import org.ethereum.db.BlockStore;
+import org.ethereum.util.RLP;
+import org.ethereum.util.RLPElement;
+import org.ethereum.util.RLPList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.math.BigInteger;
+import java.util.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.stream.Collectors;
+
+/**
+ * Snapshot Synchronization consist in 3 steps:
+ * 1. Status: exchange message with the server, to know which block we are going to sync and what the size of the Unitrie of that block is.
+ * it also exchanges previous blocks (4k) and the block of the snapshot, which has a root hash of the state.
+ * 2. State chunks: share the state in chunks of N nodes. Each chunk is independently verifiable.
+ * 3. Rebuild the state: Rebuild the state in the client side, save it to the db and save also the blocks corresponding to the snapshot.
+ *
+ * After this process, the node should be able to start the long sync to the tip and then the backward sync to the genesis.
+ */
+public class SnapshotProcessor implements InternalService {
+
+ private static final Logger logger = LoggerFactory.getLogger("snapshotprocessor");
+
+ public static final int BLOCK_NUMBER_CHECKPOINT = 5000;
+ public static final int BLOCK_CHUNK_SIZE = 400;
+ public static final int BLOCKS_REQUIRED = 6000;
+ public static final long CHUNK_ITEM_SIZE = 1024L;
+ private final Blockchain blockchain;
+ private final TrieStore trieStore;
+ private final BlockStore blockStore;
+ private final int chunkSize;
+ private final SnapshotPeersInformation peersInformation;
+ private final TransactionPool transactionPool;
+ private long messageId = 0;
+
+ // flag for parallel requests
+ private final boolean parallel;
+
+ private final BlockingQueue requestQueue = new LinkedBlockingQueue<>();
+
+ private volatile Boolean isRunning;
+ private final Thread thread;
+ public SnapshotProcessor(Blockchain blockchain,
+ TrieStore trieStore,
+ SnapshotPeersInformation peersInformation,
+ BlockStore blockStore,
+ TransactionPool transactionPool,
+ int chunkSize,
+ boolean isParallelEnabled) {
+ this(blockchain, trieStore, peersInformation, blockStore, transactionPool, chunkSize, isParallelEnabled, null);
+ }
+
+ @VisibleForTesting
+ SnapshotProcessor(Blockchain blockchain,
+ TrieStore trieStore,
+ SnapshotPeersInformation peersInformation,
+ BlockStore blockStore,
+ TransactionPool transactionPool,
+ int chunkSize,
+ boolean isParallelEnabled,
+ @Nullable SyncMessageHandler.Listener listener) {
+ this.blockchain = blockchain;
+ this.trieStore = trieStore;
+ this.peersInformation = peersInformation;
+ this.chunkSize = chunkSize;
+ this.blockStore = blockStore;
+ this.transactionPool = transactionPool;
+ this.parallel = isParallelEnabled;
+ this.thread = new Thread(new SyncMessageHandler("SNAP requests", requestQueue, listener) {
+
+ @Override
+ public boolean isRunning() {
+ return isRunning;
+ }
+ }, "snap sync request handler");
+ }
+
+ public void startSyncing() {
+ // get more than one peer, use the peer queue
+ // TODO(snap-poc) deal with multiple peers algorithm here
+ Peer peer = peersInformation.getBestSnapPeerCandidates().get(0);
+ logger.info("CLIENT - Starting Snapshot sync.");
+ requestSnapStatus(peer);
+ }
+
+ // TODO(snap-poc) should be called on errors too
+ private void stopSyncing(SnapSyncState state) {
+ state.finish();
+ }
+
+ /**
+ * STATUS
+ */
+ private void requestSnapStatus(Peer peer) {
+ SnapStatusRequestMessage message = new SnapStatusRequestMessage();
+ peer.sendMessage(message);
+ }
+
+ public void processSnapStatusRequest(Peer sender, SnapStatusRequestMessage requestMessage) {
+ if (isRunning != Boolean.TRUE) {
+ logger.warn("processSnapStatusRequest: invalid state, isRunning: [{}]", isRunning);
+ return;
+ }
+
+ try {
+ requestQueue.put(new SyncMessageHandler.Job(sender, requestMessage) {
+ @Override
+ public void run() {
+ processSnapStatusRequestInternal(sender, requestMessage);
+ }
+ });
+ } catch (InterruptedException e) {
+ logger.warn("SnapStatusRequestMessage processing was interrupted", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ void processSnapStatusRequestInternal(Peer sender, SnapStatusRequestMessage ignoredRequestMessage) {
+ logger.debug("SERVER - Processing snapshot status request.");
+ long bestBlockNumber = blockchain.getBestBlock().getNumber();
+ long checkpointBlockNumber = bestBlockNumber - (bestBlockNumber % BLOCK_NUMBER_CHECKPOINT);
+ logger.debug("SERVER - checkpointBlockNumber: {}, bestBlockNumber: {}", checkpointBlockNumber, bestBlockNumber);
+ List blocks = Lists.newArrayList();
+ List difficulties = Lists.newArrayList();
+ for (long i = checkpointBlockNumber - BLOCK_CHUNK_SIZE; i < checkpointBlockNumber; i++) {
+ Block block = blockchain.getBlockByNumber(i);
+ blocks.add(block);
+ difficulties.add(blockStore.getTotalDifficultyForHash(block.getHash().getBytes()));
+ }
+
+ logger.trace("SERVER - Sending snapshot status response. From block {} to block {} - chunksize {}", blocks.get(0).getNumber(), blocks.get(blocks.size() - 1).getNumber(), BLOCK_CHUNK_SIZE);
+ Block checkpointBlock = blockchain.getBlockByNumber(checkpointBlockNumber);
+ blocks.add(checkpointBlock);
+ logger.trace("SERVER - adding checkpoint block: {}", checkpointBlock.getNumber());
+ difficulties.add(blockStore.getTotalDifficultyForHash(checkpointBlock.getHash().getBytes()));
+ byte[] rootHash = checkpointBlock.getStateRoot();
+ Optional opt = trieStore.retrieveDTO(rootHash);
+
+ long trieSize = 0;
+ if (opt.isPresent()) {
+ trieSize = opt.get().getTotalSize();
+ } else {
+ logger.debug("SERVER - trie is notPresent");
+ }
+ logger.debug("SERVER - processing snapshot status request - rootHash: {} trieSize: {}", rootHash, trieSize);
+ SnapStatusResponseMessage responseMessage = new SnapStatusResponseMessage(blocks, difficulties, trieSize);
+ sender.sendMessage(responseMessage);
+ }
+
+ public void processSnapStatusResponse(SnapSyncState state, Peer sender, SnapStatusResponseMessage responseMessage) {
+ List blocksFromResponse = responseMessage.getBlocks();
+ List difficultiesFromResponse = responseMessage.getDifficulties();
+ Block lastBlock = blocksFromResponse.get(blocksFromResponse.size() - 1);
+
+ state.setLastBlock(lastBlock);
+ state.setLastBlockDifficulty(lastBlock.getCumulativeDifficulty());
+ state.setRemoteRootHash(lastBlock.getStateRoot());
+ state.setRemoteTrieSize(responseMessage.getTrieSize());
+
+ for (int i = 0; i < blocksFromResponse.size(); i++) {
+ state.addBlock(new ImmutablePair<>(blocksFromResponse.get(i), difficultiesFromResponse.get(i)));
+ }
+ logger.debug("CLIENT - Processing snapshot status response - last blockNumber: {} triesize: {}", lastBlock.getNumber(), state.getRemoteTrieSize());
+ logger.debug("Blocks included in the response: {} from {} to {}", blocksFromResponse.size(), blocksFromResponse.get(0).getNumber(), blocksFromResponse.get(blocksFromResponse.size() - 1).getNumber());
+ requestBlocksChunk(sender, blocksFromResponse.get(0).getNumber());
+ generateChunkRequestTasks(state);
+ startRequestingChunks(state);
+ }
+
+ /**
+ * BLOCK CHUNK
+ */
+ private void requestBlocksChunk(Peer sender, long blockNumber) {
+ logger.debug("CLIENT - Requesting block chunk to node {} - block {}", sender.getPeerNodeID(), blockNumber);
+ sender.sendMessage(new SnapBlocksRequestMessage(blockNumber));
+ }
+
+ public void processSnapBlocksRequest(Peer sender, SnapBlocksRequestMessage requestMessage) {
+ if (isRunning != Boolean.TRUE) {
+ logger.warn("processSnapBlocksRequest: invalid state, isRunning: [{}]", isRunning);
+ return;
+ }
+
+ try {
+ requestQueue.put(new SyncMessageHandler.Job(sender, requestMessage) {
+ @Override
+ public void run() {
+ processSnapBlocksRequestInternal(sender, requestMessage);
+ }
+ });
+ } catch (InterruptedException e) {
+ logger.warn("SnapBlocksRequestMessage processing was interrupted", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ void processSnapBlocksRequestInternal(Peer sender, SnapBlocksRequestMessage requestMessage) {
+ logger.debug("SERVER - Processing snap blocks request");
+ List blocks = Lists.newArrayList();
+ List difficulties = Lists.newArrayList();
+ long startingBlockNumber = requestMessage.getBlockNumber() - BLOCK_CHUNK_SIZE;
+ for (long i = startingBlockNumber; i < requestMessage.getBlockNumber(); i++) {
+ Block block = blockchain.getBlockByNumber(i);
+ blocks.add(block);
+ difficulties.add(blockStore.getTotalDifficultyForHash(block.getHash().getBytes()));
+ }
+ logger.debug("SERVER - Sending snap blocks response. From block {} to block {} - chunksize {}", blocks.get(0).getNumber(), blocks.get(blocks.size() - 1).getNumber(), BLOCK_CHUNK_SIZE);
+ SnapBlocksResponseMessage responseMessage = new SnapBlocksResponseMessage(blocks, difficulties);
+ sender.sendMessage(responseMessage);
+ }
+
+ public void processSnapBlocksResponse(SnapSyncState state, Peer sender, SnapBlocksResponseMessage responseMessage) {
+ long lastRequiredBlock = state.getLastBlock().getNumber() - BLOCKS_REQUIRED;
+ List blocksFromResponse = responseMessage.getBlocks();
+ logger.debug("CLIENT - Processing snap blocks response. Receiving from block {} to block {} Objective: {}.", blocksFromResponse.get(0).getNumber(), blocksFromResponse.get(blocksFromResponse.size() - 1).getNumber(), lastRequiredBlock);
+ List difficultiesFromResponse = responseMessage.getDifficulties();
+
+ for (int i = 0; i < blocksFromResponse.size(); i++) {
+ state.addBlock(new ImmutablePair<>(blocksFromResponse.get(i), difficultiesFromResponse.get(i)));
+ }
+ long nextChunk = blocksFromResponse.get(0).getNumber();
+ logger.debug("CLIENT - SnapBlock - nexChunk : {} - lastRequired {}, missing {}", nextChunk, lastRequiredBlock, nextChunk - lastRequiredBlock);
+ if (nextChunk > lastRequiredBlock) {
+ requestBlocksChunk(sender, nextChunk);
+ } else {
+ logger.info("CLIENT - Finished Snap blocks request sending.");
+ }
+ }
+
+ /**
+ * STATE CHUNK
+ */
+ private void requestStateChunk(Peer peer, long from, long blockNumber, int chunkSize) {
+ logger.debug("CLIENT - Requesting state chunk to node {} - block {} - chunkNumber {}", peer.getPeerNodeID(), blockNumber, from / chunkSize);
+ SnapStateChunkRequestMessage message = new SnapStateChunkRequestMessage(messageId++, blockNumber, from, chunkSize);
+ peer.sendMessage(message);
+ }
+
+ public void processStateChunkRequest(Peer sender, SnapStateChunkRequestMessage requestMessage) {
+ if (isRunning != Boolean.TRUE) {
+ logger.warn("processStateChunkRequest: invalid state, isRunning: [{}]", isRunning);
+ return;
+ }
+
+ try {
+ requestQueue.put(new SyncMessageHandler.Job(sender, requestMessage) {
+ @Override
+ public void run() {
+ processStateChunkRequestInternal(sender, requestMessage);
+ }
+ });
+ } catch (InterruptedException e) {
+ logger.warn("SnapStateChunkRequestMessage processing was interrupted", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ void processStateChunkRequestInternal(Peer sender, SnapStateChunkRequestMessage request) {
+ long startChunk = System.currentTimeMillis();
+
+ List trieEncoded = new ArrayList<>();
+ Block block = blockchain.getBlockByNumber(request.getBlockNumber());
+ final long to = request.getFrom() + (request.getChunkSize() * CHUNK_ITEM_SIZE);
+ logger.debug("SERVER - Processing state chunk request from node {}. From {} to calculated {} being chunksize {}", sender.getPeerNodeID(), request.getFrom(), to, request.getChunkSize());
+ logger.debug("SERVER - Sending state chunk from {} to {}", request.getFrom(), to);
+ TrieDTOInOrderIterator it = new TrieDTOInOrderIterator(trieStore, block.getStateRoot(), request.getFrom(), to);
+
+ // First we add the root nodes on the left of the current node. They are used to validate the chunk.
+ List preRootNodes = it.getPreRootNodes().stream().map((t) -> RLP.encodeList(RLP.encodeElement(t.getEncoded()), RLP.encodeElement(getBytes(t.getLeftHash())))).collect(Collectors.toList());
+ byte[] preRootNodesBytes = !preRootNodes.isEmpty() ? RLP.encodeList(preRootNodes.toArray(new byte[0][0])) : RLP.encodedEmptyList();
+
+ // Then we add the nodes corresponding to the chunk.
+ TrieDTO first = it.peek();
+ TrieDTO last = null;
+ while (it.hasNext()) {
+ TrieDTO e = it.next();
+ if (it.hasNext() || it.isEmpty()) {
+ last = e;
+ trieEncoded.add(RLP.encodeElement(e.getEncoded()));
+ }
+ }
+ byte[] firstNodeLeftHash = RLP.encodeElement(first.getLeftHash());
+ byte[] nodesBytes = RLP.encodeList(trieEncoded.toArray(new byte[0][0]));
+ byte[] lastNodeHashes = last != null ? RLP.encodeList(RLP.encodeElement(getBytes(last.getLeftHash())), RLP.encodeElement(getBytes(last.getRightHash()))) : RLP.encodedEmptyList();
+ // Last we add the root nodes on the right of the last visited node. They are used to validate the chunk.
+ List postRootNodes = it.getNodesLeftVisiting().stream().map((t) -> RLP.encodeList(RLP.encodeElement(t.getEncoded()), RLP.encodeElement(getBytes(t.getRightHash())))).collect(Collectors.toList());
+ byte[] postRootNodesBytes = !postRootNodes.isEmpty() ? RLP.encodeList(postRootNodes.toArray(new byte[0][0])) : RLP.encodedEmptyList();
+ byte[] chunkBytes = RLP.encodeList(preRootNodesBytes, nodesBytes, firstNodeLeftHash, lastNodeHashes, postRootNodesBytes);
+
+ SnapStateChunkResponseMessage responseMessage = new SnapStateChunkResponseMessage(request.getId(), chunkBytes, request.getBlockNumber(), request.getFrom(), to, it.isEmpty());
+
+ long totalChunkTime = System.currentTimeMillis() - startChunk;
+
+ logger.debug("SERVER - Sending state chunk from {} of {} bytes to node {}, totalTime {}ms", request.getFrom(), chunkBytes.length, sender.getPeerNodeID(), totalChunkTime);
+ sender.sendMessage(responseMessage);
+ }
+
+ public void processStateChunkResponse(SnapSyncState state, Peer peer, SnapStateChunkResponseMessage responseMessage) {
+ logger.debug("CLIENT - State chunk received chunkNumber {}. From {} to {} of total size {}", responseMessage.getFrom() / CHUNK_ITEM_SIZE, responseMessage.getFrom(), responseMessage.getTo(), state.getRemoteTrieSize());
+
+ PriorityQueue queue = state.getSnapStateChunkQueue();
+ queue.add(responseMessage);
+
+ while (!queue.isEmpty()) {
+ SnapStateChunkResponseMessage nextMessage = queue.peek();
+ long nextExpectedFrom = state.getNextExpectedFrom();
+ logger.debug("CLIENT - State chunk dequeued from: {} - expected: {}", nextMessage.getFrom(), nextExpectedFrom);
+ if (nextMessage.getFrom() == nextExpectedFrom) {
+ try {
+ processOrderedStateChunkResponse(state, peer, queue.poll());
+ state.setNextExpectedFrom(nextExpectedFrom + chunkSize * CHUNK_ITEM_SIZE);
+ } catch (Exception e) {
+ logger.error("Error while processing chunk response. {}", e.getMessage(), e);
+ onStateChunkResponseError(peer, nextMessage);
+ }
+ } else {
+ break;
+ }
+ }
+
+ if (!responseMessage.isComplete()) {
+ logger.debug("CLIENT - State chunk response not complete. Requesting next chunk.");
+ executeNextChunkRequestTask(state, peer);
+ }
+ }
+
+ @VisibleForTesting
+ void onStateChunkResponseError(Peer peer, SnapStateChunkResponseMessage responseMessage) {
+ logger.error("Error while processing chunk response from {} of peer {}. Asking for chunk again.", responseMessage.getFrom(), peer.getPeerNodeID());
+ Peer alternativePeer = peersInformation.getBestSnapPeerCandidates().stream()
+ .filter(listedPeer -> !listedPeer.getPeerNodeID().equals(peer.getPeerNodeID()))
+ .findFirst()
+ .orElse(peer);
+ logger.debug("Requesting state chunk \"from\" {} to peer {}", responseMessage.getFrom(), peer.getPeerNodeID());
+ requestStateChunk(alternativePeer, responseMessage.getFrom(), responseMessage.getBlockNumber(), chunkSize);
+ }
+
+
+ private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, SnapStateChunkResponseMessage message) throws Exception {
+ logger.debug("CLIENT - Processing State chunk received from {} to {}", message.getFrom(), message.getTo());
+ peersInformation.getOrRegisterPeer(peer);
+ state.onNewChunk();
+
+ RLPList nodeLists = RLP.decodeList(message.getChunkOfTrieKeyValue());
+ final RLPList preRootElements = RLP.decodeList(nodeLists.get(0).getRLPData());
+ final RLPList trieElements = RLP.decodeList(nodeLists.get(1).getRLPData());
+ byte[] firstNodeLeftHash = nodeLists.get(2).getRLPData();
+ final RLPList lastNodeHashes = RLP.decodeList(nodeLists.get(3).getRLPData());
+ final RLPList postRootElements = RLP.decodeList(nodeLists.get(4).getRLPData());
+ List preRootNodes = new ArrayList<>();
+ List nodes = new ArrayList<>();
+ List postRootNodes = new ArrayList<>();
+
+
+ for (int i = 0; i < preRootElements.size(); i++) {
+ final RLPList trieElement = (RLPList) preRootElements.get(i);
+ final byte[] value = trieElement.get(0).getRLPData();
+ final byte[] leftHash = trieElement.get(1).getRLPData();
+ TrieDTO node = TrieDTO.decodeFromSync(value);
+ node.setLeftHash(leftHash);
+ preRootNodes.add(node);
+ }
+
+ if (trieElements.size() > 0) {
+ for (int i = 0; i < trieElements.size(); i++) {
+ final RLPElement trieElement = trieElements.get(i);
+ byte[] value = trieElement.getRLPData();
+ nodes.add(TrieDTO.decodeFromSync(value));
+ }
+ nodes.get(0).setLeftHash(firstNodeLeftHash);
+ }
+
+ if (lastNodeHashes.size() > 0) {
+ TrieDTO lastNode = nodes.get(nodes.size() - 1);
+ lastNode.setLeftHash(lastNodeHashes.get(0).getRLPData());
+ lastNode.setRightHash(lastNodeHashes.get(1).getRLPData());
+ }
+
+ for (int i = 0; i < postRootElements.size(); i++) {
+ final RLPList trieElement = (RLPList) postRootElements.get(i);
+ final byte[] value = trieElement.get(0).getRLPData();
+ final byte[] rightHash = trieElement.get(1).getRLPData();
+ TrieDTO node = TrieDTO.decodeFromSync(value);
+ node.setRightHash(rightHash);
+ postRootNodes.add(node);
+ }
+
+ if (TrieDTOInOrderRecoverer.verifyChunk(state.getRemoteRootHash(), preRootNodes, nodes, postRootNodes)) {
+ state.getAllNodes().addAll(nodes);
+ state.setStateSize(state.getStateSize().add(BigInteger.valueOf(trieElements.size())));
+ state.setStateChunkSize(state.getStateChunkSize().add(BigInteger.valueOf(message.getChunkOfTrieKeyValue().length)));
+ if (!message.isComplete()) {
+ executeNextChunkRequestTask(state, peer);
+ } else {
+ boolean result = rebuildStateAndSave(state);
+ logger.info("CLIENT - Snapshot sync finished {}! ", result ? "successfully" : "with errors");
+ stopSyncing(state);
+ }
+ } else {
+ logger.error("Error while verifying chunk response: {}", message);
+ throw new Exception("Error verifying chunk.");
+ }
+ }
+
+ /**
+ * Once state share is received, rebuild the trie, save it in db and save all the blocks.
+ */
+ private boolean rebuildStateAndSave(SnapSyncState state) {
+ logger.info("CLIENT - Recovering trie...");
+ final TrieDTO[] nodeArray = state.getAllNodes().toArray(new TrieDTO[0]);
+ Optional result = TrieDTOInOrderRecoverer.recoverTrie(nodeArray, this.trieStore::saveDTO);
+
+ if (result.isPresent() && Arrays.equals(state.getRemoteRootHash(), result.get().calculateHash())) {
+ logger.info("CLIENT - State final validation OK!");
+
+ this.blockchain.removeBlocksByNumber(0);
+ //genesis is removed so backwards sync will always start.
+
+ BlockConnectorHelper blockConnector = new BlockConnectorHelper(this.blockStore);
+ state.connectBlocks(blockConnector);
+ logger.info("CLIENT - Setting last block as best block...");
+ this.blockchain.setStatus(state.getLastBlock(), state.getLastBlockDifficulty());
+ this.transactionPool.setBestBlock(state.getLastBlock());
+ return true;
+ }
+ logger.error("CLIENT - State final validation FAILED");
+ return false;
+ }
+
+ private void generateChunkRequestTasks(SnapSyncState state) {
+ long from = 0;
+ logger.debug("Generating chunk request tasks... chunksize {}", chunkSize);
+ while (from < state.getRemoteTrieSize()) {
+ ChunkTask task = new ChunkTask(state.getLastBlock().getNumber(), from);
+ state.getChunkTaskQueue().add(task);
+ from += chunkSize * CHUNK_ITEM_SIZE;
+ }
+ }
+
+ private void startRequestingChunks(SnapSyncState state) {
+ List bestPeerCandidates = peersInformation.getBestSnapPeerCandidates();
+ List peerList = bestPeerCandidates.subList(0, !parallel ? 1 : bestPeerCandidates.size());
+ for (Peer peer : peerList) {
+ executeNextChunkRequestTask(state, peer);
+ }
+ }
+
+ private void executeNextChunkRequestTask(SnapSyncState state, Peer peer) {
+ Queue taskQueue = state.getChunkTaskQueue();
+ if (!taskQueue.isEmpty()) {
+ ChunkTask task = taskQueue.poll();
+
+ requestStateChunk(peer, task.getFrom(), task.getBlockNumber(), chunkSize);
+ } else {
+ logger.warn("No more chunk request tasks.");
+ }
+ }
+
+ private static byte[] getBytes(byte[] result) {
+ return result != null ? result : new byte[0];
+ }
+
+ @Override
+ public void start() {
+ if (isRunning != null) {
+ logger.warn("Invalid state, isRunning: [{}]", isRunning);
+ return;
+ }
+
+ isRunning = Boolean.TRUE;
+ thread.start();
+ }
+
+ @Override
+ public void stop() {
+ if (isRunning != Boolean.TRUE) {
+ logger.warn("Invalid state, isRunning: [{}]", isRunning);
+ return;
+ }
+
+ isRunning = Boolean.FALSE;
+ thread.interrupt();
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java b/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java
index c6eff9d7ec9..7c282118e40 100644
--- a/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java
+++ b/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java
@@ -68,6 +68,7 @@ public class SyncProcessor implements SyncEventsHandler {
private final PeersInformation peersInformation;
private final Map pendingMessages;
private final AtomicBoolean isSyncing = new AtomicBoolean();
+ private final SnapshotProcessor snapshotProcessor;
private volatile long initialBlockNumber;
private volatile long highestBlockNumber;
@@ -77,6 +78,7 @@ public class SyncProcessor implements SyncEventsHandler {
private SyncState syncState;
private long lastRequestId;
+ @VisibleForTesting
public SyncProcessor(Blockchain blockchain,
BlockStore blockStore,
ConsensusValidationMainchainView consensusValidationMainchainView,
@@ -89,6 +91,23 @@ public SyncProcessor(Blockchain blockchain,
PeersInformation peersInformation,
Genesis genesis,
EthereumListener ethereumListener) {
+
+ this(blockchain, blockStore, consensusValidationMainchainView, blockSyncService, syncConfiguration, blockFactory, blockHeaderValidationRule, syncBlockValidatorRule, difficultyCalculator, peersInformation, genesis, ethereumListener, null);
+ }
+
+ public SyncProcessor(Blockchain blockchain,
+ BlockStore blockStore,
+ ConsensusValidationMainchainView consensusValidationMainchainView,
+ BlockSyncService blockSyncService,
+ SyncConfiguration syncConfiguration,
+ BlockFactory blockFactory,
+ BlockHeaderValidationRule blockHeaderValidationRule,
+ SyncBlockValidatorRule syncBlockValidatorRule,
+ DifficultyCalculator difficultyCalculator,
+ PeersInformation peersInformation,
+ Genesis genesis,
+ EthereumListener ethereumListener,
+ SnapshotProcessor snapshotProcessor) {
this.blockchain = blockchain;
this.blockStore = blockStore;
this.consensusValidationMainchainView = consensusValidationMainchainView;
@@ -112,6 +131,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) {
};
this.peersInformation = peersInformation;
+ this.snapshotProcessor = snapshotProcessor;
setSyncState(new PeerAndModeDecidingSyncState(syncConfiguration, this, peersInformation, blockStore));
}
@@ -204,6 +224,18 @@ public void processBlockResponse(Peer peer, BlockResponseMessage message) {
}
}
+ public void processSnapStatusResponse(Peer sender, SnapStatusResponseMessage responseMessage) {
+ syncState.onSnapStatus(sender, responseMessage);
+ }
+
+ public void processSnapBlocksResponse(Peer sender, SnapBlocksResponseMessage responseMessage) {
+ syncState.onSnapBlocks(sender, responseMessage);
+ }
+
+ public void processStateChunkResponse(Peer peer, SnapStateChunkResponseMessage responseMessage) {
+ syncState.onSnapStateChunk(peer, responseMessage);
+ }
+
@Override
public void sendSkeletonRequest(Peer peer, long height) {
logger.debug("Send skeleton request to node {} height {}", peer.getPeerNodeID(), height);
@@ -246,7 +278,7 @@ public void onTimePassed(Duration timePassed) {
}
@Override
- public void startSyncing(Peer peer) {
+ public void startBlockForwardSyncing(Peer peer) {
NodeID nodeID = peer.getPeerNodeID();
logger.info("Start syncing with node {}", nodeID);
byte[] bestBlockHash = peersInformation.getPeer(peer).getStatus().getBestBlockHash();
@@ -256,6 +288,12 @@ public void startSyncing(Peer peer) {
blockHeaderValidationRule, peer, bestBlockHash));
}
+ @Override
+ public void startSnapSync() {
+ logger.info("Start Snap syncing");
+ setSyncState(new SnapSyncState(this, snapshotProcessor, syncConfiguration));
+ }
+
@Override
public void startDownloadingBodies(
List> pendingHeaders, Map> skeletons, Peer peer) {
diff --git a/rskj-core/src/main/java/co/rsk/net/messages/MessageType.java b/rskj-core/src/main/java/co/rsk/net/messages/MessageType.java
index 27db4c7e83e..2b8da7a78d0 100644
--- a/rskj-core/src/main/java/co/rsk/net/messages/MessageType.java
+++ b/rskj-core/src/main/java/co/rsk/net/messages/MessageType.java
@@ -256,9 +256,46 @@ public Message createMessage(BlockFactory blockFactory, RLPList list) {
byte[] hash = list.get(0).getRLPData();
return new NewBlockHashMessage(hash);
}
- };
+ },
+ SNAP_STATE_CHUNK_REQUEST_MESSAGE(20) {
+ @Override
+ public Message createMessage(BlockFactory blockFactory, RLPList list) {
+ return SnapStateChunkRequestMessage.create(blockFactory, list);
+ }
+ },
+ SNAP_STATE_CHUNK_RESPONSE_MESSAGE(21) {
+ @Override
+ public Message createMessage(BlockFactory blockFactory, RLPList list) {
+ return SnapStateChunkResponseMessage.create(blockFactory, list);
+ }
+ },
+ SNAP_STATUS_REQUEST_MESSAGE(22) {
+ @Override
+ public Message createMessage(BlockFactory blockFactory, RLPList list) {
+ return new SnapStatusRequestMessage();
+ }
+ },
+ SNAP_STATUS_RESPONSE_MESSAGE(23) {
+ @Override
+ public Message createMessage(BlockFactory blockFactory, RLPList list) {
+ return SnapStatusResponseMessage.decodeMessage(blockFactory, list);
+ }
+ },
+ SNAP_BLOCKS_REQUEST_MESSAGE(24) {
+ @Override
+ public Message createMessage(BlockFactory blockFactory, RLPList list) {
+ return SnapBlocksRequestMessage.decodeMessage(blockFactory, list);
+ }
+ },
+ SNAP_BLOCKS_RESPONSE_MESSAGE(25) {
+ @Override
+ public Message createMessage(BlockFactory blockFactory, RLPList list) {
+ return SnapBlocksResponseMessage.decodeMessage(blockFactory, list);
+ }
+ },
+ ;
- private int type;
+ private final int type;
MessageType(int type) {
this.type = type;
diff --git a/rskj-core/src/main/java/co/rsk/net/messages/MessageVisitor.java b/rskj-core/src/main/java/co/rsk/net/messages/MessageVisitor.java
index 43047fa46c1..9fdfd4687da 100644
--- a/rskj-core/src/main/java/co/rsk/net/messages/MessageVisitor.java
+++ b/rskj-core/src/main/java/co/rsk/net/messages/MessageVisitor.java
@@ -46,6 +46,7 @@ public class MessageVisitor {
private final BlockProcessor blockProcessor;
private final SyncProcessor syncProcessor;
+ private final SnapshotProcessor snapshotProcessor;
private final TransactionGateway transactionGateway;
private final Peer sender;
private final PeerScoringManager peerScoringManager;
@@ -55,6 +56,7 @@ public class MessageVisitor {
public MessageVisitor(RskSystemProperties config,
BlockProcessor blockProcessor,
SyncProcessor syncProcessor,
+ SnapshotProcessor snapshotProcessor,
TransactionGateway transactionGateway,
PeerScoringManager peerScoringManager,
ChannelManager channelManager,
@@ -62,6 +64,7 @@ public MessageVisitor(RskSystemProperties config,
this.blockProcessor = blockProcessor;
this.syncProcessor = syncProcessor;
+ this.snapshotProcessor = snapshotProcessor;
this.transactionGateway = transactionGateway;
this.peerScoringManager = peerScoringManager;
this.channelManager = channelManager;
@@ -184,6 +187,36 @@ public void apply(NewBlockHashesMessage message) {
blockProcessor.processNewBlockHashesMessage(sender, message);
}
+ public void apply(SnapStatusRequestMessage message) {
+ logger.debug("snapshot status request message apply");
+ this.snapshotProcessor.processSnapStatusRequest(sender, message);
+ }
+
+ public void apply(SnapStatusResponseMessage message) {
+ logger.debug("snapshot status response message apply blocks[{}] - trieSize[{}]", message.getBlocks().size(), message.getTrieSize());
+ this.syncProcessor.processSnapStatusResponse(sender, message);
+ }
+
+ public void apply(SnapBlocksRequestMessage message) {
+ logger.debug("snapshot blocks request message apply : {}", message);
+ this.snapshotProcessor.processSnapBlocksRequest(sender, message);
+ }
+
+ public void apply(SnapBlocksResponseMessage message) {
+ logger.debug("snapshot blocks response message apply : {}", message);
+ this.syncProcessor.processSnapBlocksResponse(sender, message);
+ }
+
+ public void apply(SnapStateChunkRequestMessage message) {
+ logger.debug("snapshot chunk request : {}", message.getId());
+ this.snapshotProcessor.processStateChunkRequest(sender, message);
+ }
+
+ public void apply(SnapStateChunkResponseMessage message) {
+ logger.debug("snapshot chunk response : {}", message.getId());
+ this.syncProcessor.processStateChunkResponse(sender, message);
+ }
+
public void apply(TransactionsMessage message) {
if (blockProcessor.hasBetterBlockToSync()) {
loggerMessageProcess.debug("Message[{}] not processed.", message.getMessageType());
diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksRequestMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksRequestMessage.java
new file mode 100644
index 00000000000..b5437477383
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksRequestMessage.java
@@ -0,0 +1,62 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.messages;
+
+import org.bouncycastle.util.BigIntegers;
+import org.ethereum.core.BlockFactory;
+import org.ethereum.util.RLP;
+import org.ethereum.util.RLPList;
+
+import java.math.BigInteger;
+
+public class SnapBlocksRequestMessage extends Message {
+ private final long blockNumber;
+
+ public SnapBlocksRequestMessage(long blockNumber) {
+ this.blockNumber = blockNumber;
+ }
+
+ @Override
+ public MessageType getMessageType() {
+ return MessageType.SNAP_BLOCKS_REQUEST_MESSAGE;
+ }
+
+ @Override
+ public byte[] getEncodedMessage() {
+ byte[] encodedBlockNumber = RLP.encodeBigInteger(BigInteger.valueOf(blockNumber));
+ return RLP.encodeList(encodedBlockNumber);
+ }
+
+ public static Message decodeMessage(BlockFactory blockFactory, RLPList list) {
+ byte[] rlpBlockNumber = list.get(0).getRLPData();
+
+ long blockNumber = rlpBlockNumber == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpBlockNumber).longValue();
+
+ return new SnapBlocksRequestMessage(blockNumber);
+ }
+
+ public long getBlockNumber() {
+ return this.blockNumber;
+ }
+
+ @Override
+ public void accept(MessageVisitor v) {
+ v.apply(this);
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksResponseMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksResponseMessage.java
new file mode 100644
index 00000000000..df0185c458c
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksResponseMessage.java
@@ -0,0 +1,80 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.messages;
+
+import co.rsk.core.BlockDifficulty;
+import com.google.common.collect.Lists;
+import org.ethereum.core.Block;
+import org.ethereum.core.BlockFactory;
+import org.ethereum.util.RLP;
+import org.ethereum.util.RLPList;
+
+import java.math.BigInteger;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class SnapBlocksResponseMessage extends Message {
+ private final List blocks;
+ private final List difficulties;
+
+ public SnapBlocksResponseMessage(List blocks, List difficulties) {
+ this.blocks = blocks;
+ this.difficulties = difficulties;
+ }
+
+ @Override
+ public MessageType getMessageType() {
+ return MessageType.SNAP_BLOCKS_RESPONSE_MESSAGE;
+ }
+
+ public List getDifficulties() {
+ return difficulties;
+ }
+
+ public List getBlocks() {
+ return this.blocks;
+ }
+
+ @Override
+ public byte[] getEncodedMessage() {
+ List rlpBlocks = this.blocks.stream().map(Block::getEncoded).map(RLP::encode).collect(Collectors.toList());
+ List rlpDifficulties = this.difficulties.stream().map(BlockDifficulty::getBytes).map(RLP::encode).collect(Collectors.toList());
+ return RLP.encodeList(RLP.encodeList(rlpBlocks.toArray(new byte[][]{})),
+ RLP.encodeList(rlpDifficulties.toArray(new byte[][]{})));
+ }
+
+ public static Message decodeMessage(BlockFactory blockFactory, RLPList list) {
+ List blocks = Lists.newArrayList();
+ List blockDifficulties = Lists.newArrayList();
+ RLPList blocksRLP = RLP.decodeList(list.get(0).getRLPData());
+ for (int i = 0; i < blocksRLP.size(); i++) {
+ blocks.add(blockFactory.decodeBlock(blocksRLP.get(i).getRLPData()));
+ }
+ RLPList difficultiesRLP = RLP.decodeList(list.get(1).getRLPData());
+ for (int i = 0; i < difficultiesRLP.size(); i++) {
+ blockDifficulties.add(new BlockDifficulty(new BigInteger(difficultiesRLP.get(i).getRLPData())));
+ }
+ return new SnapBlocksResponseMessage(blocks, blockDifficulties);
+ }
+
+ @Override
+ public void accept(MessageVisitor v) {
+ v.apply(this);
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkRequestMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkRequestMessage.java
new file mode 100644
index 00000000000..3195ff22aa8
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkRequestMessage.java
@@ -0,0 +1,91 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.messages;
+
+import org.bouncycastle.util.BigIntegers;
+import org.ethereum.core.BlockFactory;
+import org.ethereum.util.RLP;
+import org.ethereum.util.RLPList;
+
+import java.math.BigInteger;
+
+public class SnapStateChunkRequestMessage extends MessageWithId {
+ private final long id;
+ private final long from;
+ private final long chunkSize;
+ private final long blockNumber;
+
+ public SnapStateChunkRequestMessage(long id, long blockNumber, long from, long chunkSize) {
+ this.id = id;
+ this.from = from;
+ this.chunkSize = chunkSize;
+ this.blockNumber = blockNumber;
+ }
+
+ @Override
+ public MessageType getMessageType() {
+ return MessageType.SNAP_STATE_CHUNK_REQUEST_MESSAGE;
+ }
+
+ @Override
+ public void accept(MessageVisitor v) {
+ v.apply(this);
+ }
+
+ @Override
+ public long getId() {
+ return this.id;
+ }
+
+ @Override
+ protected byte[] getEncodedMessageWithoutId() {
+ byte[] rlpBlockNumber = RLP.encodeBigInteger(BigInteger.valueOf(this.blockNumber));
+ byte[] rlpFrom = RLP.encodeBigInteger(BigInteger.valueOf(this.from));
+ byte[] rlpChunkSize = RLP.encodeBigInteger(BigInteger.valueOf(this.chunkSize));
+ return RLP.encodeList(rlpBlockNumber, rlpFrom, rlpChunkSize);
+ }
+
+ public static Message create(BlockFactory blockFactory, RLPList list) {
+ try {
+ byte[] rlpId = list.get(0).getRLPData();
+ RLPList message = (RLPList) RLP.decode2(list.get(1).getRLPData()).get(0);
+ byte[] rlpBlockNumber = message.get(0).getRLPData();
+ byte[] rlpFrom = message.get(1).getRLPData();
+ byte[] rlpChunkSize = message.get(2).getRLPData();
+ long id = rlpId == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpId).longValue();
+ long blockNumber = rlpBlockNumber == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpBlockNumber).longValue();
+ long from = rlpFrom == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpFrom).longValue();
+ long chunkSize = rlpChunkSize == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpChunkSize).longValue();
+ return new SnapStateChunkRequestMessage(id, blockNumber, from, chunkSize);
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ public long getFrom() {
+ return from;
+ }
+
+ public long getChunkSize() {
+ return chunkSize;
+ }
+ public long getBlockNumber() {
+ return blockNumber;
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkResponseMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkResponseMessage.java
new file mode 100644
index 00000000000..f0af95538cf
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkResponseMessage.java
@@ -0,0 +1,115 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.messages;
+
+import org.bouncycastle.util.BigIntegers;
+import org.ethereum.core.BlockFactory;
+import org.ethereum.util.RLP;
+import org.ethereum.util.RLPList;
+
+import java.math.BigInteger;
+
+public class SnapStateChunkResponseMessage extends MessageWithId {
+ private final long to;
+ private final long id;
+ private final byte[] chunkOfTrieKeyValue;
+
+ private final long from;
+
+ private final boolean complete;
+ private final long blockNumber;
+
+ public SnapStateChunkResponseMessage(long id, byte[] chunkOfTrieKeyValue, long blockNumber, long from, long to, boolean complete) {
+ this.id = id;
+ this.chunkOfTrieKeyValue = chunkOfTrieKeyValue;
+ this.blockNumber = blockNumber;
+ this.from = from;
+ this.to = to;
+ this.complete = complete;
+ }
+
+ @Override
+ public MessageType getMessageType() {
+ return MessageType.SNAP_STATE_CHUNK_RESPONSE_MESSAGE;
+ }
+
+ @Override
+ public void accept(MessageVisitor v) {
+ v.apply(this);
+ }
+
+ @Override
+ public long getId() {
+ return this.id;
+ }
+
+
+ @Override
+ protected byte[] getEncodedMessageWithoutId() {
+ try {
+ byte[] rlpBlockNumber = RLP.encodeBigInteger(BigInteger.valueOf(this.blockNumber));
+ byte[] rlpFrom = RLP.encodeBigInteger(BigInteger.valueOf(this.from));
+ byte[] rlpTo = RLP.encodeBigInteger(BigInteger.valueOf(this.to));
+ byte[] rlpComplete = new byte[]{this.complete ? (byte) 1 : (byte) 0};
+ return RLP.encodeList(chunkOfTrieKeyValue, rlpBlockNumber, rlpFrom, rlpTo, rlpComplete);
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ public static Message create(BlockFactory blockFactory, RLPList list) {
+ try {
+ byte[] rlpId = list.get(0).getRLPData();
+ RLPList message = (RLPList) RLP.decode2(list.get(1).getRLPData()).get(0);
+ byte[] chunkOfTrieKeys = message.get(0).getRLPData();
+ byte[] rlpBlockNumber = message.get(1).getRLPData();
+ byte[] rlpFrom = message.get(2).getRLPData();
+ byte[] rlpTo = message.get(3).getRLPData();
+ byte[] rlpComplete = message.get(4).getRLPData();
+ long id = rlpId == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpId).longValue();
+ long blockNumber = rlpBlockNumber == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpBlockNumber).longValue();
+ long from = rlpFrom == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpFrom).longValue();
+ long to = rlpTo == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpTo).longValue();
+ boolean complete = rlpComplete == null ? Boolean.FALSE : rlpComplete[0] != 0;
+ return new SnapStateChunkResponseMessage(id, chunkOfTrieKeys, blockNumber, from, to, complete);
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ public byte[] getChunkOfTrieKeyValue() {
+ return chunkOfTrieKeyValue;
+ }
+
+ public long getFrom() {
+ return from;
+ }
+
+ public boolean isComplete() {
+ return complete;
+ }
+
+ public long getBlockNumber() {
+ return blockNumber;
+ }
+
+ public long getTo() {
+ return to;
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusRequestMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusRequestMessage.java
new file mode 100644
index 00000000000..39e822ba788
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusRequestMessage.java
@@ -0,0 +1,42 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.messages;
+
+import org.ethereum.util.RLP;
+
+public class SnapStatusRequestMessage extends Message {
+
+ public SnapStatusRequestMessage() {
+ }
+
+ @Override
+ public MessageType getMessageType() {
+ return MessageType.SNAP_STATUS_REQUEST_MESSAGE;
+ }
+
+ @Override
+ public byte[] getEncodedMessage() {
+ return RLP.encodedEmptyList();
+ }
+
+ @Override
+ public void accept(MessageVisitor v) {
+ v.apply(this);
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusResponseMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusResponseMessage.java
new file mode 100644
index 00000000000..e70851edc69
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusResponseMessage.java
@@ -0,0 +1,92 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.messages;
+
+import co.rsk.core.BlockDifficulty;
+import com.google.common.collect.Lists;
+import org.bouncycastle.util.BigIntegers;
+import org.ethereum.core.Block;
+import org.ethereum.core.BlockFactory;
+import org.ethereum.util.RLP;
+import org.ethereum.util.RLPList;
+
+import java.math.BigInteger;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class SnapStatusResponseMessage extends Message {
+ private final List blocks;
+ private final List difficulties;
+ private final long trieSize;
+
+ public List getBlocks() {
+ return this.blocks;
+ }
+
+ public long getTrieSize() {
+ return this.trieSize;
+ }
+
+ public SnapStatusResponseMessage(List blocks, List difficulties, long trieSize) {
+ this.blocks = blocks;
+ this.difficulties = difficulties;
+ this.trieSize = trieSize;
+ }
+
+ @Override
+ public MessageType getMessageType() {
+ return MessageType.SNAP_STATUS_RESPONSE_MESSAGE;
+ }
+
+ public List getDifficulties() {
+ return difficulties;
+ }
+
+ @Override
+ public byte[] getEncodedMessage() {
+ List rlpBlocks = this.blocks.stream().map(Block::getEncoded).map(RLP::encode).collect(Collectors.toList());
+ List rlpDifficulties = this.difficulties.stream().map(BlockDifficulty::getBytes).map(RLP::encode).collect(Collectors.toList());
+ byte[] rlpTrieSize = RLP.encodeBigInteger(BigInteger.valueOf(this.trieSize));
+
+ return RLP.encodeList(RLP.encodeList(rlpBlocks.toArray(new byte[][]{})), RLP.encodeList(rlpDifficulties.toArray(new byte[][]{})), rlpTrieSize);
+ }
+
+ public static Message decodeMessage(BlockFactory blockFactory, RLPList list) {
+ RLPList rlpBlocks = RLP.decodeList(list.get(0).getRLPData());
+ RLPList rlpDifficulties = RLP.decodeList(list.get(1).getRLPData());
+ List blocks = Lists.newArrayList();
+ List difficulties = Lists.newArrayList();
+ for (int i = 0; i < rlpBlocks.size(); i++) {
+ blocks.add(blockFactory.decodeBlock(rlpBlocks.get(i).getRLPData()));
+ }
+ for (int i = 0; i < rlpDifficulties.size(); i++) {
+ difficulties.add(new BlockDifficulty(new BigInteger(rlpDifficulties.get(i).getRLPData())));
+ }
+
+ byte[] rlpTrieSize = list.get(2).getRLPData();
+ long trieSize = rlpTrieSize == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpTrieSize).longValue();
+
+ return new SnapStatusResponseMessage(blocks, difficulties, trieSize);
+ }
+
+ @Override
+ public void accept(MessageVisitor v) {
+ v.apply(this);
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/BaseSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/BaseSyncState.java
index 4845202c7e5..a2b0fa2cf9b 100644
--- a/rskj-core/src/main/java/co/rsk/net/sync/BaseSyncState.java
+++ b/rskj-core/src/main/java/co/rsk/net/sync/BaseSyncState.java
@@ -19,6 +19,9 @@
import co.rsk.net.Peer;
import co.rsk.net.messages.BodyResponseMessage;
+import co.rsk.net.messages.SnapBlocksResponseMessage;
+import co.rsk.net.messages.SnapStateChunkResponseMessage;
+import co.rsk.net.messages.SnapStatusResponseMessage;
import com.google.common.annotations.VisibleForTesting;
import org.ethereum.core.BlockHeader;
import org.ethereum.core.BlockIdentifier;
@@ -51,30 +54,34 @@ public void tick(Duration duration) {
}
}
- protected void onMessageTimeOut() {
- }
+ protected void onMessageTimeOut() { /* empty */ }
@Override
- public void newBlockHeaders(List chunk) {
- }
+ public void newBlockHeaders(List chunk) { /* empty */ }
@Override
- public void newBody(BodyResponseMessage message, Peer peer) {
- }
+ public void newBody(BodyResponseMessage message, Peer peer) { /* empty */ }
@Override
- public void newConnectionPointData(byte[] hash) {
- }
+ public void newConnectionPointData(byte[] hash) { /* empty */ }
@Override
- public void newPeerStatus() { }
+ public void newPeerStatus() { /* empty */ }
@Override
- public void newSkeleton(List skeleton, Peer peer) {
- }
+ public void newSkeleton(List skeleton, Peer peer) { /* empty */ }
+
+ @Override
+ public void onSnapStatus(Peer sender, SnapStatusResponseMessage responseMessage) { /* empty */ }
+
+ @Override
+ public void onSnapBlocks(Peer sender, SnapBlocksResponseMessage responseMessage) { /* empty */ }
+
+ @Override
+ public void onSnapStateChunk(Peer peer, SnapStateChunkResponseMessage responseMessage) { /* empty */ }
@Override
- public void onEnter() { }
+ public void onEnter() { /* empty */ }
@VisibleForTesting
public void messageSent() {
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorException.java b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorException.java
new file mode 100644
index 00000000000..67c297a81ca
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorException.java
@@ -0,0 +1,38 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.sync;
+
+public class BlockConnectorException extends RuntimeException {
+ private final long blockNumber;
+ private final long childBlockNumber;
+
+ public BlockConnectorException(final long blockNumber, final long childBlockNumber) {
+ super(String.format("Block with number %s is not child's (%s) parent.", blockNumber, childBlockNumber));
+ this.blockNumber = blockNumber;
+ this.childBlockNumber = childBlockNumber;
+ }
+
+ public long getBlockNumber() {
+ return blockNumber;
+ }
+
+ public long getChildBlockNumber() {
+ return childBlockNumber;
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java
new file mode 100644
index 00000000000..58191074de5
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java
@@ -0,0 +1,85 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.sync;
+
+import co.rsk.core.BlockDifficulty;
+import org.apache.commons.lang3.tuple.Pair;
+import org.ethereum.core.Block;
+import org.ethereum.db.BlockStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class BlockConnectorHelper {
+ private static final Logger logger = LoggerFactory.getLogger("SnapBlockConnector");
+ private final BlockStore blockStore;
+
+ public BlockConnectorHelper(BlockStore blockStore) {
+ this.blockStore = blockStore;
+ }
+
+ public void startConnecting(List> blockAndDifficultiesList) {
+ if (blockAndDifficultiesList.isEmpty()) {
+ logger.debug("Block list is empty, nothing to connect");
+ return;
+ }
+
+ blockAndDifficultiesList.sort(new BlockAndDiffComparator());
+ Block child = null;
+ logger.info("Start connecting blocks ranging from {} to {} - Total: {}",
+ blockAndDifficultiesList.get(0).getKey().getNumber(),
+ blockAndDifficultiesList.get(blockAndDifficultiesList.size() - 1).getKey().getNumber(),
+ blockAndDifficultiesList.size());
+
+ int blockIndex = blockAndDifficultiesList.size() - 1;
+ if (blockStore.isEmpty()) {
+ Pair blockAndDifficulty = blockAndDifficultiesList.get(blockIndex);
+ child = blockAndDifficulty.getLeft();
+ logger.debug("BlockStore is empty, setting child block number the last block from the list: {}", child.getNumber());
+ blockStore.saveBlock(child, blockAndDifficulty.getRight(), true);
+ logger.debug("Block number: {} saved", child.getNumber());
+ blockIndex--;
+ } else {
+ logger.debug("BlockStore is not empty, getting best block");
+ child = blockStore.getBestBlock();
+ logger.debug("Best block number: {}", child.getNumber());
+ }
+ while (blockIndex >= 0) {
+ Pair currentBlockAndDifficulty = blockAndDifficultiesList.get(blockIndex);
+ Block currentBlock = currentBlockAndDifficulty.getLeft();
+ logger.trace("Connecting block number: {}", currentBlock.getNumber());
+
+ if (!currentBlock.isParentOf(child)) {
+ throw new BlockConnectorException(currentBlock.getNumber(), child.getNumber());
+ }
+ blockStore.saveBlock(currentBlock, currentBlockAndDifficulty.getRight(), true);
+ child = currentBlock;
+ blockIndex--;
+ }
+ logger.info("Finished connecting blocks. Last saved block: {}",child.getNumber());
+ }
+
+ static class BlockAndDiffComparator implements java.util.Comparator> {
+ @Override
+ public int compare(Pair o1, Pair o2) {
+ return Long.compare(o1.getLeft().getNumber(), o2.getLeft().getNumber());
+ }
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/ChunkTask.java b/rskj-core/src/main/java/co/rsk/net/sync/ChunkTask.java
new file mode 100644
index 00000000000..5c9c4cfa88c
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/sync/ChunkTask.java
@@ -0,0 +1,37 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2024 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.sync;
+
+public class ChunkTask {
+ private final long blockNumber;
+ private final long from;
+
+ public ChunkTask(long blockNumber, long from) {
+ this.blockNumber = blockNumber;
+ this.from = from;
+ }
+
+ public long getBlockNumber() {
+ return blockNumber;
+ }
+
+ public long getFrom() {
+ return from;
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/PeerAndModeDecidingSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/PeerAndModeDecidingSyncState.java
index efdee1bd9a8..1389d0e9ad1 100644
--- a/rskj-core/src/main/java/co/rsk/net/sync/PeerAndModeDecidingSyncState.java
+++ b/rskj-core/src/main/java/co/rsk/net/sync/PeerAndModeDecidingSyncState.java
@@ -69,7 +69,13 @@ public void onEnter() {
}
private void tryStartSyncing() {
- if (tryStartLongForwardSync()) {
+ logger.trace("Starting tryStartSyncing");
+
+ if (tryStartSnapshotSync()) {
+ return;
+ }
+
+ if (tryStartBlockForwardSync()) {
return;
}
@@ -80,22 +86,57 @@ private void tryStartSyncing() {
syncEventsHandler.onLongSyncUpdate(false, null);
}
- private boolean tryStartLongForwardSync() {
+ private boolean tryStartSnapshotSync() {
+ if (!syncConfiguration.isClientSnapSyncEnabled()) {
+ logger.trace("Snap syncing disabled");
+ return false;
+ }
+
+ // TODO(snap-poc) deal with multiple peers logic here
+ // TODO: To be handled when we implement the multiple peers
+ //List bestPeers = peersInformation.getBestPeerCandidates();
+
+ // TODO: for now, use pre-configured snap boot nodes instead (until snap nodes discovery is implemented)
+ SnapshotPeersInformation snapPeersInformation = peersInformation;
+ Optional bestPeerOpt = snapPeersInformation.getBestSnapPeer();
+ Optional peerBestBlockNumOpt = bestPeerOpt.flatMap(this::getPeerBestBlockNumber);
+
+ if (!bestPeerOpt.isPresent() || !peerBestBlockNumOpt.isPresent()) {
+ logger.trace("Snap syncing not possible, no valid peer");
+ return false;
+ }
+
+ // we consider Snap as part of the Long Sync
+ if (!isValidSnapDistance(peerBestBlockNumOpt.get())) {
+ logger.debug("Snap syncing not required (long sync not required)");
+ return false;
+ }
+
+ // we consider Snap as part of the Long Sync
+ syncEventsHandler.onLongSyncUpdate(true, peerBestBlockNumOpt.get());
+
+ // send the LIST
+ syncEventsHandler.startSnapSync();
+ return true;
+ }
+
+ private boolean tryStartBlockForwardSync() {
Optional bestPeerOpt = peersInformation.getBestPeer();
Optional peerBestBlockNumOpt = bestPeerOpt.flatMap(this::getPeerBestBlockNumber);
+
if (!bestPeerOpt.isPresent() || !peerBestBlockNumOpt.isPresent()) {
logger.trace("Forward syncing not possible, no valid peer");
return false;
}
if (!shouldLongSync(peerBestBlockNumOpt.get())) {
- logger.debug("Forward syncing not required");
+ logger.trace("Forward syncing not required");
return false;
}
// start "long" / "forward" sync
syncEventsHandler.onLongSyncUpdate(true, peerBestBlockNumOpt.get());
- syncEventsHandler.startSyncing(bestPeerOpt.get());
+ syncEventsHandler.startBlockForwardSyncing(bestPeerOpt.get());
return true;
}
@@ -122,6 +163,11 @@ private boolean shouldLongSync(long peerBestBlockNumber) {
return distanceToTip > syncConfiguration.getLongSyncLimit() || checkGenesisConnected();
}
+ private boolean isValidSnapDistance(long peerBestBlockNumber) {
+ long distanceToTip = peerBestBlockNumber - blockStore.getBestBlock().getNumber();
+ return distanceToTip > syncConfiguration.getSnapshotSyncLimit();
+ }
+
private Optional getPeerBestBlockNumber(Peer peer) {
return Optional.ofNullable(peersInformation.getPeer(peer))
.flatMap(pi -> Optional.ofNullable(pi.getStatus()).map(Status::getBestBlockNumber));
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java b/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java
index 6400e68f91f..593e814beb2 100644
--- a/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java
+++ b/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java
@@ -31,7 +31,14 @@
import java.security.SecureRandom;
import java.time.Instant;
-import java.util.*;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Random;
+import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -41,21 +48,19 @@
* TODO(mc) remove this after the logical node abstraction is created, since it will wrap
* things such as the underlying communication channel.
*/
-public class PeersInformation {
+public class PeersInformation implements SnapshotPeersInformation {
private static final int TIME_LIMIT_FAILURE_RECORD = 600;
private static final int MAX_SIZE_FAILURE_RECORDS = 10;
-
private final ChannelManager channelManager;
private final SyncConfiguration syncConfiguration;
private final Blockchain blockchain;
private final Map failedPeers;
private final PeerScoringManager peerScoringManager;
private final Comparator> peerComparator;
- private Map peerStatuses = new HashMap<>();
private final double topBest;
private final Random random;
-
+ private Map peerStatuses = new HashMap<>();
public PeersInformation(ChannelManager channelManager,
SyncConfiguration syncConfiguration,
@@ -66,10 +71,10 @@ public PeersInformation(ChannelManager channelManager,
@VisibleForTesting
PeersInformation(ChannelManager channelManager,
- SyncConfiguration syncConfiguration,
- Blockchain blockchain,
- PeerScoringManager peerScoringManager,
- Random random) {
+ SyncConfiguration syncConfiguration,
+ Blockchain blockchain,
+ PeerScoringManager peerScoringManager,
+ Random random) {
this.channelManager = channelManager;
this.syncConfiguration = syncConfiguration;
this.blockchain = blockchain;
@@ -117,9 +122,9 @@ public SyncPeerStatus getPeer(Peer peer) {
return this.peerStatuses.get(peer);
}
- public Optional getBestPeer() {
+ private Optional getBestPeer(Stream> bestCandidatesStream) {
if (topBest > 0.0D) {
- List> entries = getBestCandidatesStream()
+ List> entries = bestCandidatesStream
.sorted(this.peerComparator.reversed())
.collect(Collectors.toList());
@@ -143,6 +148,18 @@ public Optional getBestPeer() {
.map(Map.Entry::getKey);
}
+ public Optional getBestPeer() {
+ return getBestPeer(getBestCandidatesStream());
+ }
+
+ @Override
+ public Optional getBestSnapPeer() {
+ return getBestPeer(
+ getBestCandidatesStream()
+ .filter(this::isSnapPeerCandidateOrCapable)
+ );
+ }
+
public Optional getBestOrEqualPeer() {
return getTrustedPeers()
.filter(e -> isMyDifficultyLowerThan(e.getKey(), false))
@@ -179,6 +196,14 @@ public List getBestPeerCandidates() {
.collect(Collectors.toList());
}
+ @Override
+ public List getBestSnapPeerCandidates() {
+ return getBestCandidatesStream()
+ .filter(entry -> entry.getKey().isSnapCapable())
+ .map(Map.Entry::getKey)
+ .collect(Collectors.toList());
+ }
+
public Set knownNodeIds() {
return peerStatuses.keySet().stream()
.map(Peer::getPeerNodeID)
@@ -260,4 +285,12 @@ private Instant getFailInstant(Peer peer) {
public void clearOldFailedPeers() {
failedPeers.values().removeIf(Instant.now().minusSeconds(TIME_LIMIT_FAILURE_RECORD)::isAfter);
}
+
+ private boolean isSnapPeerCandidate(Map.Entry entry) {
+ return syncConfiguration.getNodeIdToSnapshotTrustedPeerMap().containsKey(entry.getKey().getPeerNodeID().toString());
+ }
+
+ private boolean isSnapPeerCandidateOrCapable(Map.Entry entry) {
+ return isSnapPeerCandidate(entry) || entry.getKey().isSnapCapable();
+ }
}
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java
new file mode 100644
index 00000000000..65494b86129
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java
@@ -0,0 +1,268 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.sync;
+
+import co.rsk.core.BlockDifficulty;
+import co.rsk.net.Peer;
+import co.rsk.net.SnapshotProcessor;
+import co.rsk.net.messages.SnapBlocksResponseMessage;
+import co.rsk.net.messages.SnapStateChunkResponseMessage;
+import co.rsk.net.messages.SnapStatusResponseMessage;
+import co.rsk.trie.TrieDTO;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.tuple.Pair;
+import org.ethereum.core.Block;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.math.BigInteger;
+import java.time.Duration;
+import java.util.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+public class SnapSyncState extends BaseSyncState {
+
+ private static final Logger logger = LoggerFactory.getLogger("SnapSyncState");
+
+ private final SnapshotProcessor snapshotProcessor;
+
+ // queue for processing of SNAP responses
+ private final BlockingQueue responseQueue = new LinkedBlockingQueue<>();
+
+ // priority queue for ordering chunk responses
+ private final PriorityQueue snapStateChunkQueue = new PriorityQueue<>(
+ Comparator.comparingLong(SnapStateChunkResponseMessage::getFrom)
+ );
+
+ private final Queue chunkTaskQueue = new LinkedList<>();
+
+ private BigInteger stateSize = BigInteger.ZERO;
+ private BigInteger stateChunkSize = BigInteger.ZERO;
+ private final List allNodes;
+
+ private long remoteTrieSize;
+ private byte[] remoteRootHash;
+ private final List> blocks;
+ private Block lastBlock;
+ private BlockDifficulty lastBlockDifficulty;
+
+ private long nextExpectedFrom = 0L;
+
+ private volatile Boolean isRunning;
+ private final Thread thread;
+
+ public SnapSyncState(SyncEventsHandler syncEventsHandler, SnapshotProcessor snapshotProcessor, SyncConfiguration syncConfiguration) {
+ this(syncEventsHandler, snapshotProcessor, syncConfiguration, null);
+ }
+
+ @VisibleForTesting
+ SnapSyncState(SyncEventsHandler syncEventsHandler, SnapshotProcessor snapshotProcessor,
+ SyncConfiguration syncConfiguration, @Nullable SyncMessageHandler.Listener listener) {
+ super(syncEventsHandler, syncConfiguration);
+ this.snapshotProcessor = snapshotProcessor; // TODO(snap-poc) code in SnapshotProcessor should be moved here probably
+ this.allNodes = Lists.newArrayList();
+ this.blocks = Lists.newArrayList();
+ this.thread = new Thread(new SyncMessageHandler("SNAP responses", responseQueue, listener) {
+
+ @Override
+ public boolean isRunning() {
+ return isRunning;
+ }
+ }, "snap sync response handler");
+ }
+
+ @Override
+ public void onEnter() {
+ if (isRunning != null) {
+ logger.warn("Invalid state, isRunning: [{}]", isRunning);
+ return;
+ }
+ isRunning = Boolean.TRUE;
+ thread.start();
+ snapshotProcessor.startSyncing();
+ }
+
+ @Override
+ public void onSnapStatus(Peer sender, SnapStatusResponseMessage responseMessage) {
+ try {
+ responseQueue.put(new SyncMessageHandler.Job(sender, responseMessage) {
+ @Override
+ public void run() {
+ snapshotProcessor.processSnapStatusResponse(SnapSyncState.this, sender, responseMessage);
+ }
+ });
+ } catch (InterruptedException e) {
+ logger.warn("SnapStatusResponseMessage processing was interrupted", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @Override
+ public void onSnapBlocks(Peer sender, SnapBlocksResponseMessage responseMessage) {
+ try {
+ responseQueue.put(new SyncMessageHandler.Job(sender, responseMessage) {
+ @Override
+ public void run() {
+ snapshotProcessor.processSnapBlocksResponse(SnapSyncState.this, sender, responseMessage);
+ }
+ });
+ } catch (InterruptedException e) {
+ logger.warn("SnapBlocksResponseMessage processing was interrupted", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @Override
+ public void onSnapStateChunk(Peer sender, SnapStateChunkResponseMessage responseMessage) {
+ try {
+ responseQueue.put(new SyncMessageHandler.Job(sender, responseMessage) {
+ @Override
+ public void run() {
+ snapshotProcessor.processStateChunkResponse(SnapSyncState.this, sender, responseMessage);
+ }
+ });
+ } catch (InterruptedException e) {
+ logger.warn("SnapStateChunkResponseMessage processing was interrupted", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ public void onNewChunk() {
+ resetTimeElapsed();
+ }
+
+ @Override
+ public void tick(Duration duration) {
+ // TODO(snap-poc) handle multiple peers casuistry, similarly to co.rsk.net.sync.DownloadingBodiesSyncState.tick
+
+ timeElapsed = timeElapsed.plus(duration);
+ if (timeElapsed.compareTo(syncConfiguration.getTimeoutWaitingSnapChunk()) >= 0) {
+ onMessageTimeOut();
+ }
+ }
+
+ @Override
+ protected void onMessageTimeOut() {
+ // TODO: call syncEventsHandler.onErrorSyncing() and punish peers after SNAP feature discovery is implemented
+
+ finish();
+ }
+
+ public Block getLastBlock() {
+ return lastBlock;
+ }
+
+ public void setLastBlock(Block lastBlock) {
+ this.lastBlock = lastBlock;
+ }
+
+ public long getNextExpectedFrom() {
+ return nextExpectedFrom;
+ }
+
+ public void setNextExpectedFrom(long nextExpectedFrom) {
+ this.nextExpectedFrom = nextExpectedFrom;
+ }
+
+ public BlockDifficulty getLastBlockDifficulty() {
+ return lastBlockDifficulty;
+ }
+
+ public void setLastBlockDifficulty(BlockDifficulty lastBlockDifficulty) {
+ this.lastBlockDifficulty = lastBlockDifficulty;
+ }
+
+ public byte[] getRemoteRootHash() {
+ return remoteRootHash;
+ }
+
+ public void setRemoteRootHash(byte[] remoteRootHash) {
+ this.remoteRootHash = remoteRootHash;
+ }
+
+ public long getRemoteTrieSize() {
+ return remoteTrieSize;
+ }
+
+ public void setRemoteTrieSize(long remoteTrieSize) {
+ this.remoteTrieSize = remoteTrieSize;
+ }
+
+ public void addBlock(Pair blockPair) {
+ blocks.add(blockPair);
+ }
+
+ public void addAllBlocks(List> blocks) {
+ this.blocks.addAll(blocks);
+ }
+
+ public void connectBlocks(BlockConnectorHelper blockConnectorHelper) {
+ blockConnectorHelper.startConnecting(blocks);
+ }
+
+ public List getAllNodes() {
+ return allNodes;
+ }
+
+ public BigInteger getStateSize() {
+ return stateSize;
+ }
+
+ public void setStateSize(BigInteger stateSize) {
+ this.stateSize = stateSize;
+ }
+
+ public BigInteger getStateChunkSize() {
+ return stateChunkSize;
+ }
+
+ public void setStateChunkSize(BigInteger stateChunkSize) {
+ this.stateChunkSize = stateChunkSize;
+ }
+
+ public PriorityQueue getSnapStateChunkQueue() {
+ return snapStateChunkQueue;
+ }
+
+ public Queue getChunkTaskQueue() {
+ return chunkTaskQueue;
+ }
+
+ public void finish() {
+ if (isRunning != Boolean.TRUE) {
+ logger.warn("Invalid state, isRunning: [{}]", isRunning);
+ return;
+ }
+
+ isRunning = Boolean.FALSE;
+ thread.interrupt();
+
+ syncEventsHandler.stopSyncing();
+ }
+
+ @VisibleForTesting
+ public void setRunning() {
+ isRunning = true;
+ }
+
+
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java
new file mode 100644
index 00000000000..52bed4900e4
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java
@@ -0,0 +1,34 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2024 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+package co.rsk.net.sync;
+
+import co.rsk.net.Peer;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * This is mostly a workaround because SyncProcessor needs to access Peer instances.
+ * TODO(mc) remove this after the logical node abstraction is created, since it will wrap
+ * things such as the underlying communication channel.
+ */
+public interface SnapshotPeersInformation {
+ Optional getBestSnapPeer();
+ List getBestSnapPeerCandidates();
+ SyncPeerStatus getOrRegisterPeer(Peer peer);
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java
index 929e9af7908..beeb1ac70e7 100644
--- a/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java
+++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java
@@ -18,17 +18,22 @@
package co.rsk.net.sync;
import com.google.common.annotations.VisibleForTesting;
+import org.ethereum.net.rlpx.Node;
import javax.annotation.concurrent.Immutable;
import java.time.Duration;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
@Immutable
public final class SyncConfiguration {
@VisibleForTesting
- public static final SyncConfiguration DEFAULT = new SyncConfiguration(5, 60, 30, 5, 20, 192, 20, 10, 0);
+ public static final SyncConfiguration DEFAULT = new SyncConfiguration(5, 60, 30, 5, 20, 192, 20, 10, 0, false, false, 60, 0);
@VisibleForTesting
- public static final SyncConfiguration IMMEDIATE_FOR_TESTING = new SyncConfiguration(1, 1, 3, 1, 5, 192, 20, 10, 0);
+ public static final SyncConfiguration IMMEDIATE_FOR_TESTING = new SyncConfiguration(1, 1, 3, 1, 5, 192, 20, 10, 0, false, false, 60, 0);
private final int expectedPeers;
private final Duration timeoutWaitingPeers;
@@ -39,17 +44,28 @@ public final class SyncConfiguration {
private final int longSyncLimit;
private final int maxRequestedBodies;
private final double topBest;
+ private final boolean isServerSnapSyncEnabled;
+ private final boolean isClientSnapSyncEnabled;
+
+ private final Duration timeoutWaitingSnapChunk;
+
+ private final int snapshotSyncLimit;
+ private final Map nodeIdToSnapshotTrustedPeerMap;
/**
- * @param expectedPeers The expected number of peers we would want to start finding a connection point.
- * @param timeoutWaitingPeers Timeout in minutes to start finding the connection point when we have at least one peer
- * @param timeoutWaitingRequest Timeout in seconds to wait for syncing requests
+ * @param expectedPeers The expected number of peers we would want to start finding a connection point.
+ * @param timeoutWaitingPeers Timeout in minutes to start finding the connection point when we have at least one peer
+ * @param timeoutWaitingRequest Timeout in seconds to wait for syncing requests
* @param expirationTimePeerStatus Expiration time in minutes for peer status
- * @param maxSkeletonChunks Maximum amount of chunks included in a skeleton message
- * @param chunkSize Amount of blocks contained in a chunk
- * @param maxRequestedBodies Amount of bodies to request at the same time when synchronizing backwards.
- * @param longSyncLimit Distance to the tip of the peer's blockchain to enable long synchronization.
- * @param topBest % of top best nodes that will be considered for random selection.
+ * @param maxSkeletonChunks Maximum amount of chunks included in a skeleton message
+ * @param chunkSize Amount of blocks contained in a chunk
+ * @param maxRequestedBodies Amount of bodies to request at the same time when synchronizing backwards.
+ * @param longSyncLimit Distance to the tip of the peer's blockchain to enable long synchronization.
+ * @param topBest % of top best nodes that will be considered for random selection.
+ * @param isServerSnapSyncEnabled Flag that indicates if server-side snap sync is enabled
+ * @param isClientSnapSyncEnabled Flag that indicates if client-side snap sync is enabled
+ * @param timeoutWaitingSnapChunk Specific request timeout for snap sync
+ * @param snapshotSyncLimit Distance to the tip of the peer's blockchain to enable snap synchronization.
*/
public SyncConfiguration(
int expectedPeers,
@@ -60,7 +76,42 @@ public SyncConfiguration(
int chunkSize,
int maxRequestedBodies,
int longSyncLimit,
- double topBest) {
+ double topBest,
+ boolean isServerSnapSyncEnabled,
+ boolean isClientSnapSyncEnabled,
+ int timeoutWaitingSnapChunk,
+ int snapshotSyncLimit) {
+ this(expectedPeers,
+ timeoutWaitingPeers,
+ timeoutWaitingRequest,
+ expirationTimePeerStatus,
+ maxSkeletonChunks,
+ chunkSize,
+ maxRequestedBodies,
+ longSyncLimit,
+ topBest,
+ isServerSnapSyncEnabled,
+ isClientSnapSyncEnabled,
+ timeoutWaitingSnapChunk,
+ snapshotSyncLimit,
+ Collections.emptyList());
+ }
+
+ public SyncConfiguration(
+ int expectedPeers,
+ int timeoutWaitingPeers,
+ int timeoutWaitingRequest,
+ int expirationTimePeerStatus,
+ int maxSkeletonChunks,
+ int chunkSize,
+ int maxRequestedBodies,
+ int longSyncLimit,
+ double topBest,
+ boolean isServerSnapSyncEnabled,
+ boolean isClientSnapSyncEnabled,
+ int timeoutWaitingSnapChunk,
+ int snapshotSyncLimit,
+ List snapBootNodes) {
this.expectedPeers = expectedPeers;
this.timeoutWaitingPeers = Duration.ofSeconds(timeoutWaitingPeers);
this.timeoutWaitingRequest = Duration.ofSeconds(timeoutWaitingRequest);
@@ -70,6 +121,18 @@ public SyncConfiguration(
this.maxRequestedBodies = maxRequestedBodies;
this.longSyncLimit = longSyncLimit;
this.topBest = topBest;
+ this.isServerSnapSyncEnabled = isServerSnapSyncEnabled;
+ this.isClientSnapSyncEnabled = isClientSnapSyncEnabled;
+ // TODO(snap-poc) re-visit the need of this specific timeout as the algorithm evolves
+ this.timeoutWaitingSnapChunk = Duration.ofSeconds(timeoutWaitingSnapChunk);
+ this.snapshotSyncLimit = snapshotSyncLimit;
+
+
+
+ List snapBootNodesList = snapBootNodes != null ? snapBootNodes : Collections.emptyList();
+
+ nodeIdToSnapshotTrustedPeerMap = Collections.unmodifiableMap(snapBootNodesList.stream()
+ .collect(Collectors.toMap(peer -> peer.getId().toString(), peer -> peer)));
}
public final int getExpectedPeers() {
@@ -105,6 +168,26 @@ public int getLongSyncLimit() {
}
public double getTopBest() {
- return topBest;
+ return topBest;
+ }
+
+ public boolean isServerSnapSyncEnabled() {
+ return isServerSnapSyncEnabled;
+ }
+
+ public boolean isClientSnapSyncEnabled() {
+ return isClientSnapSyncEnabled;
+ }
+
+ public Duration getTimeoutWaitingSnapChunk() {
+ return timeoutWaitingSnapChunk;
+ }
+
+ public int getSnapshotSyncLimit() {
+ return snapshotSyncLimit;
+ }
+
+ public Map getNodeIdToSnapshotTrustedPeerMap() {
+ return nodeIdToSnapshotTrustedPeerMap;
}
}
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java
index 9d50e0f61c0..e3cb4d93c83 100644
--- a/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java
+++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java
@@ -43,7 +43,7 @@ public interface SyncEventsHandler {
void startDownloadingSkeleton(long connectionPoint, Peer peer);
- void startSyncing(Peer peer);
+ void startBlockForwardSyncing(Peer peer);
void backwardDownloadBodies(Block parent, List toRequest, Peer peer);
@@ -58,4 +58,6 @@ public interface SyncEventsHandler {
void startFindingConnectionPoint(Peer peer);
void backwardSyncing(Peer peer);
+
+ void startSnapSync();
}
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java
new file mode 100644
index 00000000000..b2f282cc468
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java
@@ -0,0 +1,143 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2024 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.net.sync;
+
+import co.rsk.net.Peer;
+import co.rsk.net.messages.Message;
+import co.rsk.util.FormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.BlockingQueue;
+
+public abstract class SyncMessageHandler implements Runnable {
+
+ private static final Logger logger = LoggerFactory.getLogger("syncprocessor");
+
+ private final String name;
+
+ private final BlockingQueue jobQueue;
+
+ private final Listener listener;
+
+ protected SyncMessageHandler(String name, BlockingQueue jobQueue) {
+ this(name, jobQueue, null);
+ }
+
+ protected SyncMessageHandler(String name, BlockingQueue jobQueue, Listener listener) {
+ this.name = name;
+ this.jobQueue = jobQueue;
+ this.listener = listener;
+ }
+
+ public abstract boolean isRunning();
+
+ @Override
+ public void run() {
+ logger.debug("Starting processing queue of messages for: [{}]", name);
+
+ if (listener != null) {
+ listener.onStart();
+ }
+
+ Job job = null;
+ Instant jobStart = Instant.MIN;
+ while (isRunning()) {
+ try {
+ job = jobQueue.take();
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("Processing msg: [{}] from: [{}] for: [{}]", job.getMsg().getMessageType(), job.getSender(), name);
+ jobStart = Instant.now();
+ }
+
+ job.run();
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("Finished processing of msg: [{}] from: [{}] for: [{}] after [{}] seconds.",
+ job.getMsg().getMessageType(), job.getSender(), name,
+ FormatUtils.formatNanosecondsToSeconds(Duration.between(jobStart, Instant.now()).toNanos()));
+ }
+
+ if (listener != null) {
+ listener.onJobRun(job);
+ if (jobQueue.isEmpty()) {
+ listener.onQueueEmpty();
+ }
+ }
+ } catch (InterruptedException e) {
+ logger.warn("Queue processing was interrupted for: [{}]", name, e);
+ if (listener != null) {
+ listener.onInterrupted();
+ }
+ Thread.currentThread().interrupt();
+ break;
+ } catch (Exception e) {
+ logger.error("Unexpected error processing msg: '[{}]' for: [{}]", job, name, e);
+ if (listener != null) {
+ listener.onException(e);
+ }
+ }
+ }
+
+ if (listener != null) {
+ listener.onComplete();
+ }
+
+ logger.debug("Finished processing queue of messages for: [{}]", name);
+ }
+
+ public interface Listener {
+ void onStart();
+ void onJobRun(Job job);
+ void onQueueEmpty();
+ void onInterrupted();
+ void onException(Exception e);
+ void onComplete();
+ }
+
+ public static abstract class Job implements Runnable {
+ private final Peer sender;
+
+ private final Message msg;
+
+ public Job(Peer sender, Message msg) {
+ this.sender = sender;
+ this.msg = msg;
+ }
+
+ public Peer getSender() {
+ return sender;
+ }
+
+ public Message getMsg() {
+ return msg;
+ }
+
+ @Override
+ public String toString() {
+ return "SyncMessageHandler{" +
+ "sender=" + sender +
+ ", msgType=" + msg.getMessageType() +
+ '}';
+ }
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java
index adf3ed7076e..e3b9b48fa47 100644
--- a/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java
+++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java
@@ -19,6 +19,9 @@
import co.rsk.net.Peer;
import co.rsk.net.messages.BodyResponseMessage;
+import co.rsk.net.messages.SnapBlocksResponseMessage;
+import co.rsk.net.messages.SnapStateChunkResponseMessage;
+import co.rsk.net.messages.SnapStatusResponseMessage;
import org.ethereum.core.BlockHeader;
import org.ethereum.core.BlockIdentifier;
@@ -40,6 +43,12 @@ public interface SyncState {
void newSkeleton(List skeletonChunk, Peer peer);
+ void onSnapStatus(Peer sender, SnapStatusResponseMessage responseMessage);
+
+ void onSnapBlocks(Peer sender, SnapBlocksResponseMessage responseMessage);
+
+ void onSnapStateChunk(Peer peer, SnapStateChunkResponseMessage responseMessage);
+
void onEnter();
void tick(Duration duration);
diff --git a/rskj-core/src/main/java/co/rsk/trie/MultiTrieStore.java b/rskj-core/src/main/java/co/rsk/trie/MultiTrieStore.java
index e73bb202c98..b94ccad7386 100644
--- a/rskj-core/src/main/java/co/rsk/trie/MultiTrieStore.java
+++ b/rskj-core/src/main/java/co/rsk/trie/MultiTrieStore.java
@@ -61,6 +61,11 @@ public void save(Trie trie) {
getCurrentStore().save(trie);
}
+ @Override
+ public void saveDTO(TrieDTO trieDTO) {
+ getCurrentStore().saveDTO(trieDTO);
+ }
+
/**
* It's not enough to just flush the current one b/c it may occur, if the epoch size doesn't match the flush size,
* that some epoch may never get flushed
@@ -87,6 +92,19 @@ public Optional retrieve(byte[] rootHash) {
return Optional.empty();
}
+ @Override
+ public Optional retrieveDTO(byte[] rootHash) {
+ for (TrieStore epochTrieStore : epochs) {
+ byte[] message = epochTrieStore.retrieveValue(rootHash);
+ if (message == null) {
+ continue;
+ }
+ return Optional.of(TrieDTO.decodeFromMessage(message, this));
+ }
+
+ return Optional.empty();
+ }
+
@Override
public byte[] retrieveValue(byte[] hash) {
for (TrieStore epochTrieStore : epochs) {
diff --git a/rskj-core/src/main/java/co/rsk/trie/NodeReference.java b/rskj-core/src/main/java/co/rsk/trie/NodeReference.java
index 52a8997d93f..27427d81232 100644
--- a/rskj-core/src/main/java/co/rsk/trie/NodeReference.java
+++ b/rskj-core/src/main/java/co/rsk/trie/NodeReference.java
@@ -62,6 +62,23 @@ public boolean isEmpty() {
return lazyHash == null && lazyNode == null;
}
+ /**
+ * The hash or empty if this is an empty reference.
+ * If the hash is not present but its node is known, it will be calculated.
+ */
+ public Optional getHash() {
+ if (lazyHash != null) {
+ return Optional.of(lazyHash);
+ }
+
+ if (lazyNode == null) {
+ return Optional.empty();
+ }
+
+ lazyHash = lazyNode.getHash();
+ return Optional.of(lazyHash);
+ }
+
/**
* The node or empty if this is an empty reference.
* If the node is not present but its hash is known, it will be retrieved from the store.
@@ -90,23 +107,6 @@ public Optional getNode() {
return node;
}
- /**
- * The hash or empty if this is an empty reference.
- * If the hash is not present but its node is known, it will be calculated.
- */
- public Optional getHash() {
- if (lazyHash != null) {
- return Optional.of(lazyHash);
- }
-
- if (lazyNode == null) {
- return Optional.empty();
- }
-
- lazyHash = lazyNode.getHash();
- return Optional.of(lazyHash);
- }
-
/**
* The hash or empty if this is an empty reference.
* If the hash is not present but its node is known, it will be calculated.
diff --git a/rskj-core/src/main/java/co/rsk/trie/SharedPathSerializer.java b/rskj-core/src/main/java/co/rsk/trie/SharedPathSerializer.java
index 2031f7a89ce..e5579faab70 100644
--- a/rskj-core/src/main/java/co/rsk/trie/SharedPathSerializer.java
+++ b/rskj-core/src/main/java/co/rsk/trie/SharedPathSerializer.java
@@ -18,8 +18,12 @@
package co.rsk.trie;
import co.rsk.bitcoinj.core.VarInt;
+import org.apache.commons.lang3.tuple.Pair;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
public class SharedPathSerializer {
@@ -46,14 +50,19 @@ public boolean isPresent() {
}
public void serializeInto(ByteBuffer buffer) {
- serializeInto(this.sharedPath,buffer);
+ serializeInto(this.sharedPath, buffer);
}
- public static void serializeInto(TrieKeySlice sharedPath,ByteBuffer buffer) {
+ public static void serializeInto(TrieKeySlice sharedPath, ByteBuffer buffer) {
if (!isPresent(sharedPath)) {
return;
}
int lshared = sharedPath.length();
+ final byte[] encode = sharedPath.encode();
+ serializeBytes(buffer, lshared, encode);
+ }
+
+ public static void serializeBytes(ByteBuffer buffer, int lshared, byte[] encode) {
if (1 <= lshared && lshared <= 32) {
// first byte in [0..31]
buffer.put((byte) (lshared - 1));
@@ -64,8 +73,21 @@ public static void serializeInto(TrieKeySlice sharedPath,ByteBuffer buffer) {
buffer.put((byte) 255);
buffer.put(new VarInt(lshared).encode());
}
+ buffer.put(encode);
+ }
- buffer.put(sharedPath.encode());
+ public static void writeBytes(ByteArrayOutputStream buffer, int lshared, byte[] encode) throws IOException {
+ if (1 <= lshared && lshared <= 32) {
+ // first byte in [0..31]
+ buffer.write((byte) (lshared - 1));
+ } else if (160 <= lshared && lshared <= 382) {
+ // first byte in [32..254]
+ buffer.write((byte) (lshared - 128));
+ } else {
+ buffer.write((byte) 255);
+ buffer.write(new VarInt(lshared).encode());
+ }
+ buffer.write(encode);
}
// Returns the size of the path prefix when path needs encoding.
@@ -76,6 +98,10 @@ private static int lsharedSize(TrieKeySlice sharedPath) {
return 0;
}
int lshared = sharedPath.length();
+ return calculateVarIntSize(lshared);
+ }
+
+ public static int calculateVarIntSize(int lshared) {
if (1 <= lshared && lshared <= 32) {
return 1;
}
@@ -107,6 +133,7 @@ public static int getPathBitsLength(ByteBuffer message) {
}
return lshared;
}
+
public static TrieKeySlice deserialize(ByteBuffer message, boolean sharedPrefixPresent) {
if (!sharedPrefixPresent) {
return TrieKeySlice.empty();
@@ -120,6 +147,23 @@ public static TrieKeySlice deserialize(ByteBuffer message, boolean sharedPrefixP
return TrieKeySlice.fromEncoded(encodedKey, 0, lshared, lencoded);
}
+ public static Pair deserializeEncoded(ByteBuffer message, boolean sharedPrefixPresent, ByteArrayOutputStream encoder) throws IOException {
+ if (!sharedPrefixPresent) {
+ return null;
+ }
+
+ int lshared = getPathBitsLength(message);
+
+ int lencoded = PathEncoder.calculateEncodedLength(lshared);
+ byte[] encodedKey = new byte[lencoded];
+ message.get(encodedKey);
+ final byte[] result = Arrays.copyOfRange(encodedKey, 0, lencoded);
+ if (encoder != null) {
+ writeBytes(encoder, lshared, result);
+ }
+ return Pair.of(lshared, result);
+ }
+
private static VarInt readVarInt(ByteBuffer message) {
// read without touching the buffer position so when we read into bytes it contains the header
int first = Byte.toUnsignedInt(message.get(message.position()));
diff --git a/rskj-core/src/main/java/co/rsk/trie/TrieDTO.java b/rskj-core/src/main/java/co/rsk/trie/TrieDTO.java
new file mode 100644
index 00000000000..d7ca5fc3fdb
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/trie/TrieDTO.java
@@ -0,0 +1,563 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2023 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.trie;
+
+import co.rsk.bitcoinj.core.VarInt;
+import co.rsk.core.types.ints.Uint24;
+import co.rsk.core.types.ints.Uint8;
+import co.rsk.crypto.Keccak256;
+import co.rsk.util.HexUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.ethereum.crypto.Keccak256Helper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.function.Function;
+
+public class TrieDTO {
+
+ private static final Logger logger = LoggerFactory.getLogger(TrieDTO.class);
+
+ // ----- FLAGS:
+ private boolean hasLongVal;
+
+ private boolean sharedPrefixPresent;
+ private boolean leftNodePresent;
+ private boolean rightNodePresent;
+
+ private boolean leftNodeEmbedded;
+ private boolean rightNodeEmbedded;
+ // -----
+
+ // left and right are used when embedded nodes
+ private byte[] left;
+ private byte[] right;
+
+ // hashes are used when not embedded nodes
+ private byte[] leftHash;
+ private byte[] rightHash;
+
+ // encoded has the essential data to rebuild the trie.
+ private byte[] encoded;
+
+ // source has the source data retrieved from the DB.
+ private byte[] source;
+
+ //
+ private byte[] value;
+
+ // childrenSize is the size its children (in bytes)
+ private VarInt childrenSize;
+ private byte[] path;
+ private Integer pathLength;
+ private byte flags;
+ private TrieDTO leftNode;
+ private TrieDTO rightNode;
+ private byte[] hash;
+
+ public static TrieDTO decodeFromMessage(byte[] src, TrieStore ds) {
+ return decodeFromMessage(src, ds, false, null);
+ }
+
+ public static TrieDTO decodeFromMessage(byte[] src, TrieStore ds, boolean preloadChildren, byte[] hash) {
+ TrieDTO result = new TrieDTO();
+ try {
+ ByteArrayOutputStream encoder = new ByteArrayOutputStream();
+ ByteBuffer srcWrap = ByteBuffer.wrap(src);
+ byte flags = srcWrap.get();
+ //1.flags
+ encoder.write(flags);
+ result.flags = flags;
+ result.hasLongVal = (flags & 0b00100000) == 0b00100000;
+ result.sharedPrefixPresent = (flags & 0b00010000) == 0b00010000;
+ result.leftNodePresent = (flags & 0b00001000) == 0b00001000;
+ result.rightNodePresent = (flags & 0b00000100) == 0b00000100;
+ result.leftNodeEmbedded = (flags & 0b00000010) == 0b00000010;
+ result.rightNodeEmbedded = (flags & 0b00000001) == 0b00000001;
+
+ //(*optional) 2.sharedPath - if sharedPrefixPresent
+ final Pair pathTuple = SharedPathSerializer.deserializeEncoded(srcWrap, result.sharedPrefixPresent, encoder);
+ result.pathLength = pathTuple != null ? pathTuple.getKey() : null;
+ result.path = pathTuple != null ? pathTuple.getValue() : null;
+
+ //(*optional) 3.left - if present & !embedded => hash
+ handleLeft(result, srcWrap, encoder, ds, preloadChildren, hash);
+
+ //(*optional) 3.right - if present & !embedded => hash
+ handleRight(result, srcWrap, encoder, ds, preloadChildren, hash);
+
+ result.childrenSize = new VarInt(0);
+ if (result.leftNodePresent || result.rightNodePresent) {
+ //(*optional) 4.childrenSize - if any children present
+ result.childrenSize = readVarInt(srcWrap, encoder);
+ }
+
+ handleValue(result, srcWrap, encoder, ds);
+
+ if (srcWrap.hasRemaining()) {
+ throw new IllegalArgumentException("The srcWrap had more data than expected");
+ }
+ result.hash = hash;
+ result.encoded = encoder.toByteArray();
+ result.source = ArrayUtils.clone(src);
+ } catch (IOException e) {
+ throw new RuntimeException("Error while decoding.", e);
+ }
+ return result;
+ }
+
+ private static void handleLeft(TrieDTO result, ByteBuffer srcWrap, ByteArrayOutputStream encoder, TrieStore ds, boolean preloadChildren, byte[] hash) throws IOException {
+ if (result.leftNodePresent && result.leftNodeEmbedded) {
+ result.leftNode = TrieDTO.decodeFromMessage(readChildEmbedded(srcWrap, decodeUint8(), Uint8.BYTES), ds, false, hash);
+ result.left = result.leftNode.getEncoded();
+ result.leftHash = null;
+ encoder.write(encodeUint24(result.left.length));
+ encoder.write(result.left);
+ } else if (result.leftNodePresent) {
+ byte[] valueHash = new byte[Keccak256Helper.DEFAULT_SIZE_BYTES];
+ srcWrap.get(valueHash);
+ result.left = preloadChildren ? valueHash : null;
+ result.leftHash = valueHash;
+ }
+ }
+ private static void handleRight(TrieDTO result, ByteBuffer srcWrap, ByteArrayOutputStream encoder, TrieStore ds, boolean preloadChildren, byte[] hash) throws IOException {
+ if (result.rightNodePresent && result.rightNodeEmbedded) {
+ result.rightNode = TrieDTO.decodeFromMessage(readChildEmbedded(srcWrap, decodeUint8(), Uint8.BYTES), ds, false, hash);
+ result.right = result.rightNode.getEncoded();
+ result.rightHash = null;
+ encoder.write(encodeUint24(result.right.length));
+ encoder.write(result.right);
+ } else if (result.rightNodePresent) {
+ byte[] valueHash = new byte[Keccak256Helper.DEFAULT_SIZE_BYTES];
+ srcWrap.get(valueHash);
+ result.right = preloadChildren ? valueHash : null;
+ result.rightHash = valueHash;
+ }
+
+ }
+
+ private static void handleValue(TrieDTO result, ByteBuffer srcWrap, ByteArrayOutputStream encoder, TrieStore ds) throws IOException {
+ if (result.hasLongVal) {
+ byte[] valueHashBytes = new byte[Keccak256Helper.DEFAULT_SIZE_BYTES];
+ srcWrap.get(valueHashBytes);
+ byte[] lvalueBytes = new byte[Uint24.BYTES];
+ srcWrap.get(lvalueBytes);
+ byte[] value = ds.retrieveValue(valueHashBytes);
+ encoder.write(value);
+ result.value = value;
+ } else {
+ int remaining = srcWrap.remaining();
+ byte[] value = new byte[remaining];
+ srcWrap.get(value);
+ //(*optional) 5.value - if !longValue => value
+ encoder.write(value);
+ result.value = value;
+ }
+
+ }
+ public static TrieDTO decodeFromSync(byte[] src) {
+ TrieDTO result = new TrieDTO();
+ try {
+ ByteBuffer srcWrap = ByteBuffer.wrap(src);
+ result.flags = srcWrap.get();
+ result.hasLongVal = (result.flags & 0b00100000) == 0b00100000;
+ result.sharedPrefixPresent = (result.flags & 0b00010000) == 0b00010000;
+ result.leftNodePresent = (result.flags & 0b00001000) == 0b00001000;
+ result.rightNodePresent = (result.flags & 0b00000100) == 0b00000100;
+ result.leftNodeEmbedded = (result.flags & 0b00000010) == 0b00000010;
+ result.rightNodeEmbedded = (result.flags & 0b00000001) == 0b00000001;
+
+ final Pair pathTuple = SharedPathSerializer.deserializeEncoded(srcWrap, result.sharedPrefixPresent, null);
+ result.pathLength = pathTuple != null ? pathTuple.getKey() : null;
+ result.path = pathTuple != null ? pathTuple.getValue() : null;
+
+ result.leftHash = null;
+ if (result.leftNodeEmbedded) {
+ final byte[] leftBytes = readChildEmbedded(srcWrap, decodeUint24(), Uint24.BYTES);
+ result.leftNode = TrieDTO.decodeFromSync(leftBytes);
+ result.left = result.leftNode.toMessage();
+ }
+
+ result.rightHash = null;
+ if (result.rightNodeEmbedded) {
+ final byte[] rightBytes = readChildEmbedded(srcWrap, decodeUint24(), Uint24.BYTES);
+ result.rightNode = TrieDTO.decodeFromSync(rightBytes);
+ result.right = result.rightNode.toMessage();
+ }
+
+ result.childrenSize = new VarInt(0);
+ if (result.leftNodePresent || result.rightNodePresent) {
+ result.childrenSize = readVarInt(srcWrap, null);
+ }
+
+ int remaining = srcWrap.remaining();
+ byte[] value = new byte[remaining];
+ srcWrap.get(value);
+ result.value = value;
+ //result.source = ArrayUtils.clone(src);
+ } catch (IOException e) {
+ logger.trace("Error while decoding: {}", e.getMessage());
+ }
+ return result;
+ }
+
+ private static byte[] readChildEmbedded(ByteBuffer srcWrap, Function decode, int uintBytes) {
+ byte[] lengthBytes = new byte[uintBytes];
+ srcWrap.get(lengthBytes);
+ byte[] serializedNode = decode.apply(lengthBytes);
+ srcWrap.get(serializedNode);
+ return serializedNode;
+ }
+
+ /**
+ * @return the tree size in bytes as specified in RSKIP107
+ *
+ * This method will EXPAND internal encoding caches without removing them afterwards.
+ * It shouldn't be called from outside. It's still public for NodeReference call
+ */
+ public VarInt getChildrenSize() {
+ if (childrenSize == null) {
+ childrenSize = new VarInt(0);
+ }
+ return childrenSize;
+ }
+
+ public long getSize() {
+ long externalValueLength = this.hasLongVal ? this.value.length : 0L;
+ final long leftSize = getLeftSize();
+ final long rightSize = getRightSize();
+ return externalValueLength + this.source.length + leftSize + rightSize;
+ }
+
+ public long getLeftSize() {
+ return leftNodeEmbedded ? this.leftNode.getTotalSize() : 0L;
+ }
+
+ public long getRightSize() {
+ return rightNodeEmbedded ? this.rightNode.getTotalSize() : 0L;
+ }
+
+ public long getTotalSize() {
+ long externalValueLength = this.hasLongVal ? this.value.length : 0L;
+ long nodeSize = externalValueLength + this.source.length;
+ return this.getChildrenSize().value + nodeSize;
+ }
+
+ private static VarInt readVarInt(ByteBuffer message, ByteArrayOutputStream encoder) throws IOException {
+ // read without touching the buffer position so when we read into bytes it contains the header
+ int first = Byte.toUnsignedInt(message.get(message.position()));
+ byte[] bytes;
+ if (first < 253) {
+ bytes = new byte[1];
+ } else if (first == 253) {
+ bytes = new byte[3];
+ } else if (first == 254) {
+ bytes = new byte[5];
+ } else {
+ bytes = new byte[9];
+ }
+
+ message.get(bytes);
+ if (encoder != null) {
+ encoder.write(bytes);
+ }
+ return new VarInt(bytes, 0);
+ }
+
+ public byte[] getRightHash() {
+ return this.rightNodePresent && !this.rightNodeEmbedded ? rightHash : null;
+ }
+
+ public byte[] getLeftHash() {
+ return this.leftNodePresent && !this.leftNodeEmbedded ? leftHash : null;
+ }
+
+ public byte[] getEncoded() {
+ return encoded;
+ }
+
+ public byte[] getSource() {
+ return this.source;
+ }
+
+ public byte[] getValue() {
+ return value;
+ }
+
+ public boolean isTerminal() {
+ if (!this.leftNodePresent && !this.rightNodePresent) {
+ return true;
+ }
+
+ boolean isLeftTerminal = !this.leftNodePresent || this.leftNodeEmbedded;
+ boolean isRightTerminal = !this.rightNodePresent || this.rightNodeEmbedded;
+
+ return isLeftTerminal && isRightTerminal;
+ }
+
+ public byte[] getLeft() {
+ return left;
+ }
+
+ public byte[] getRight() {
+ return right;
+ }
+
+ public void setLeft(byte[] left) {
+ this.left = left;
+ }
+
+ public void setRight(byte[] right) {
+ this.right = right;
+ }
+
+ public TrieDTO getLeftNode() {
+ return leftNode;
+ }
+
+ public TrieDTO getRightNode() {
+ return rightNode;
+ }
+
+ public void setLeftHash(byte[] hash) {
+ this.leftHash = hash;
+ }
+
+ public void setRightHash(byte[] hash) {
+ this.rightHash = hash;
+ }
+
+ public boolean isLeftNodePresent() {
+ return leftNodePresent;
+ }
+
+ public boolean isRightNodePresent() {
+ return rightNodePresent;
+ }
+
+ public boolean isLeftNodeEmbedded() {
+ return leftNodeEmbedded;
+ }
+
+ public boolean isRightNodeEmbedded() {
+ return rightNodeEmbedded;
+ }
+
+ public boolean isSharedPrefixPresent() {
+ return sharedPrefixPresent;
+ }
+
+ public byte[] getPath() {
+ return this.path;
+ }
+
+ public Integer getPathLength() {
+ return pathLength;
+ }
+
+ public boolean isHasLongVal() {
+ return hasLongVal;
+ }
+
+ @Override
+ public String toString() {
+ return "Node{" + HexUtils.toJsonHex(this.path) + "}:" + this.childrenSize.value;
+ }
+
+ public String toDescription() {
+ return "Node{" + HexUtils.toJsonHex(this.path) + "}:\n" +
+ "{isTerminal()=" + this.isTerminal() + "},\n" +
+ "{getSize()=" + this.getSize() + "},\n" +
+ "{getTotalSize()=" + this.getTotalSize() + "},\n" +
+ "{valueSize=" + getSizeString(this.value) + "},\n" +
+ "{childrenSize=" + this.childrenSize.value + "},\n" +
+ "{rightSize=" + getSizeString(this.right) + "},\n" +
+ "{leftSize=" + getSizeString(this.left) + "},\n" +
+ "{hasLongVal=" + this.hasLongVal + "},\n" +
+ "{sharedPrefixPresent=" + this.sharedPrefixPresent + "},\n" +
+ "{leftNodePresent=" + this.leftNodePresent + "},\n" +
+ "{rightNodePresent=" + this.rightNodePresent + "},\n" +
+ "{leftNodeEmbedded=" + this.leftNodeEmbedded + "},\n" +
+ "{rightNodeEmbedded=" + this.rightNodeEmbedded + "},\n" +
+ "{left=" + HexUtils.toJsonHex(this.left) + "},\n" +
+ "{right=" + HexUtils.toJsonHex(this.right) + "},\n" +
+ "{hash=" + HexUtils.toJsonHex(this.hash) + "},\n" +
+ "{calculateHash()=" + calculateHashString() + "},\n" +
+ "{calculateSourceHash()=" + calculateSourceHash() + "},\n" +
+ "{leftHash=" + HexUtils.toJsonHex(this.leftHash) + "},\n" +
+ "{rightHash=" + HexUtils.toJsonHex(this.rightHash) + "},\n" +
+ "{value=" + HexUtils.toJsonHex(this.value) + "},\n" +
+ "{toMessage()=" + HexUtils.toJsonHex(this.toMessage()) + "},\n" +
+ "{source=" + HexUtils.toJsonHex(this.source) + "}\n";
+ }
+
+ /**
+ * Based on {@link Trie:toMessage()}
+ */
+ public byte[] toMessage() {
+// ByteBuffer buffer = ByteBuffer.allocate(
+// 1 + // flags
+// (this.sharedPrefixPresent ? SharedPathSerializer.calculateVarIntSize(this.pathLength) + this.path.length : 0) +
+// serializedLength(leftNodePresent, leftNodeEmbedded, left) +
+// serializedLength(rightNodePresent, rightNodeEmbedded, right) +
+// ((leftNodePresent || rightNodePresent) ? childrenSize.getSizeInBytes() : 0) +
+// (hasLongVal ? Keccak256Helper.DEFAULT_SIZE_BYTES + Uint24.BYTES : value.length)
+// );
+
+ int sharedPrefixSize = this.sharedPrefixPresent ? SharedPathSerializer.calculateVarIntSize(this.pathLength) + this.path.length : 0;
+ int leftNodeSize = serializedLength(leftNodePresent, leftNodeEmbedded, left);
+ int rightNodeSize = serializedLength(rightNodePresent, rightNodeEmbedded, right);
+ int childrenSizeBytes = (leftNodePresent || rightNodePresent) ? childrenSize.getSizeInBytes() : 0;
+ int valueSize = hasLongVal ? Keccak256Helper.DEFAULT_SIZE_BYTES + Uint24.BYTES : value.length;
+
+ int totalSize = 1 + sharedPrefixSize + leftNodeSize + rightNodeSize + childrenSizeBytes + valueSize;
+
+ ByteBuffer buffer = ByteBuffer.allocate(totalSize);
+
+ buffer.put(flags);
+ if (this.sharedPrefixPresent) {
+ SharedPathSerializer.serializeBytes(buffer, this.pathLength, this.path);
+ }
+
+ toMessageHandleLeftNode(buffer);
+ toMessageHandleRightNode(buffer);
+
+ if (leftNodePresent || rightNodePresent) {
+ buffer.put(childrenSize.encode());
+ }
+ if (hasLongVal) {
+ byte[] valueHash = new Keccak256(Keccak256Helper.keccak256(getValue())).getBytes();
+ buffer.put(valueHash);
+ buffer.put(encodeUint24(value.length));
+ } else if (this.getValue().length > 0) {
+ buffer.put(this.getValue());
+ }
+ return buffer.array();
+ }
+
+ private void toMessageHandleLeftNode(ByteBuffer buffer){
+ if (leftNodePresent) {
+ if (leftNodeEmbedded) {
+ buffer.put(encodeUint8(this.left.length));
+ buffer.put(this.left);
+ } else {
+ buffer.put(this.leftHash);
+ }
+ }
+ }
+
+ private void toMessageHandleRightNode(ByteBuffer buffer){
+ if (rightNodePresent) {
+ if (rightNodeEmbedded) {
+ buffer.put(encodeUint8(this.right.length));
+ buffer.put(this.right);
+ } else {
+ buffer.put(this.rightHash);
+ }
+ }
+ }
+ public int serializedLength(boolean isPresent, boolean isEmbeddable, byte[] value) {
+ if (isPresent) {
+ if (isEmbeddable) {
+ return Uint8.BYTES + value.length;
+ }
+ return Keccak256Helper.DEFAULT_SIZE_BYTES;
+ }
+
+ return 0;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o){
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()){
+ return false;
+ }
+ TrieDTO trieDTO = (TrieDTO) o;
+ return hasLongVal == trieDTO.hasLongVal
+ && sharedPrefixPresent == trieDTO.sharedPrefixPresent
+ && leftNodePresent == trieDTO.leftNodePresent
+ && rightNodePresent == trieDTO.rightNodePresent
+ && leftNodeEmbedded == trieDTO.leftNodeEmbedded
+ && rightNodeEmbedded == trieDTO.rightNodeEmbedded
+ && Arrays.equals(value, trieDTO.value)
+ && Objects.equals(childrenSize.value, trieDTO.childrenSize.value)
+ && Arrays.equals(path, trieDTO.path);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hash(hasLongVal, sharedPrefixPresent, leftNodePresent, rightNodePresent, leftNodeEmbedded, rightNodeEmbedded, childrenSize);
+ result = 31 * result + Arrays.hashCode(left);
+ result = 31 * result + Arrays.hashCode(right);
+ result = 31 * result + Arrays.hashCode(value);
+ result = 31 * result + Arrays.hashCode(path);
+ return result;
+ }
+
+ public String calculateHashString() {
+ return HexUtils.toJsonHex(calculateHash());
+ }
+
+ public byte[] calculateHash() {
+ return Keccak256Helper.keccak256(this.toMessage());
+ }
+
+ private String calculateSourceHash() {
+ if (this.source != null) {
+ return HexUtils.toJsonHex(Keccak256Helper.keccak256(this.source));
+ }
+ return "";
+ }
+
+ private String getSizeString(byte[] value) {
+ return "" + (value != null ? value.length : 0L);
+ }
+
+ private static Function decodeUint8() {
+ return (byte[] lengthBytes) -> {
+ Uint8 length = Uint8.decode(lengthBytes, 0);
+ return new byte[length.intValue()];
+ };
+ }
+
+ private static Function decodeUint24() {
+ return (byte[] lengthBytes) -> {
+ Uint24 length = Uint24.decode(lengthBytes, 0);
+ return new byte[length.intValue()];
+ };
+ }
+
+ private static byte[] encodeUint24(int length) {
+ return new Uint24(length).encode();
+ }
+
+ private static byte[] encodeUint8(int length) {
+ return new Uint8(length).encode();
+ }
+
+}
diff --git a/rskj-core/src/main/java/co/rsk/trie/TrieDTOInOrderIterator.java b/rskj-core/src/main/java/co/rsk/trie/TrieDTOInOrderIterator.java
new file mode 100644
index 00000000000..9d84c8a64b4
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/trie/TrieDTOInOrderIterator.java
@@ -0,0 +1,197 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2022 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.trie;
+
+import java.util.*;
+
+public class TrieDTOInOrderIterator implements Iterator {
+
+ private final Deque visiting;
+ private final TrieStore ds;
+
+ private long from;
+ private final long to;
+ private final boolean preloadChildren;
+
+ // preRoots are all the root nodes on the left of the current node. They are used to validate the chunk.
+ // When initializing the iterator, everytime we turn right, we add the node to the list.
+ private final List preRootNodes = new ArrayList<>();
+
+ public TrieDTOInOrderIterator(TrieStore ds, byte[] root, long from, long to) {
+ this(ds, root, from, to, false);
+ }
+
+ public TrieDTOInOrderIterator(TrieStore ds, byte[] root, long from, long to, boolean preloadChildren) {
+ Objects.requireNonNull(root);
+ this.ds = ds;
+ this.visiting = new LinkedList<>();
+ this.from = from;
+ this.to = to;
+ this.preloadChildren = preloadChildren;
+ // find the leftmost node, pushing all the intermediate nodes onto the visiting stack
+ findByChildrenSize(from, getNode(root), visiting);
+ // now the leftmost unvisited node is on top of the visiting stack
+ }
+
+ private TrieDTO findByChildrenSize(long offset, TrieDTO nodeDTO, Deque visiting) {
+ // TODO poner los nodos padres intermedios en el stack, tenemos que serializarlos para poder validar el chunk completo.
+ if (!nodeDTO.isTerminal()) {
+
+ if (isLeftNotEmbedded(nodeDTO)){
+ TrieDTO left = getNode(nodeDTO.getLeftHash());
+
+ if (left == null) {
+ throw new NullPointerException("Left node is null");
+ }
+
+ long maxLeftSize = left.getTotalSize();
+
+ if (offset <= maxLeftSize) {
+ visiting.push(nodeDTO);
+ return findByChildrenSize(offset, left, visiting);
+ }
+ maxLeftSize += nodeDTO.getSize();
+ if (offset <= maxLeftSize) {
+ return pushAndReturn(nodeDTO, visiting, (offset - left.getTotalSize()));
+ }
+ } else if (nodeDTO.isLeftNodePresent() && nodeDTO.isLeftNodeEmbedded() && (offset <= nodeDTO.getLeftSize())) {
+ return pushAndReturn(nodeDTO, visiting, offset);
+ }
+
+ if (nodeDTO.isRightNodePresent() && !nodeDTO.isRightNodeEmbedded()) {
+ TrieDTO right = getNode(nodeDTO.getRightHash());
+ if (right == null) {
+ throw new NullPointerException("Right node is null.");
+ }
+
+ long maxParentSize = nodeDTO.getTotalSize() - right.getTotalSize();
+ long maxRightSize = nodeDTO.getTotalSize();
+
+ if (maxParentSize < offset && offset <= maxRightSize) {
+ preRootNodes.add(nodeDTO);
+ return findByChildrenSize(offset - maxParentSize, right, visiting);
+ }
+ } else if (nodeDTO.isRightNodeEmbedded() && (offset <= nodeDTO.getTotalSize())) {
+ long leftAndParentSize = nodeDTO.getTotalSize() - nodeDTO.getRightSize();
+ return pushAndReturn(nodeDTO, visiting, offset - leftAndParentSize);
+ }
+ }
+ if (nodeDTO.getTotalSize() >= offset) {
+ return pushAndReturn(nodeDTO, visiting, offset);
+ } else {
+ return nodeDTO;
+ }
+ }
+
+ private boolean isLeftNotEmbedded(TrieDTO nodeDTO){
+ return nodeDTO.isLeftNodePresent() && !nodeDTO.isLeftNodeEmbedded();
+ }
+
+ private TrieDTO pushAndReturn(TrieDTO nodeDTO, Deque visiting, long offset) {
+ this.from -= offset;
+ visiting.push(nodeDTO);
+ return nodeDTO;
+ }
+
+ /**
+ * return the leftmost node that has not yet been visited that node is normally on top of the stack
+ */
+ @Override
+ @SuppressWarnings("squid:S2272") // NoSuchElementException is thrown by {@link Deque#pop()} when it's empty
+ public TrieDTO next() {
+ TrieDTO result = this.visiting.peek();
+ // if the node has a right child, its leftmost node is next
+ long offset = getOffset(result);
+ if (this.from + offset < this.to) {
+ this.visiting.pop(); // remove the node from the stack
+ if (result.getRightHash() != null) {
+ TrieDTO rightNode = pushNode(result.getRightHash(), this.visiting);
+ // find the leftmost node of the right child
+ if (rightNode != null) {
+ pushLeftmostNode(rightNode);
+ }
+ // note "node" has been replaced on the stack by its right child
+ }
+ } // else: no right subtree, go back up the stack
+ // next node on stack will be next returned
+ this.from += offset;
+ return result;
+ }
+
+ /**
+ * Find the leftmost node from this root, pushing all the intermediate nodes onto the visiting stack
+ *
+ * @param nodeKey the root of the subtree for which we are trying to reach the leftmost node
+ */
+ private void pushLeftmostNode(TrieDTO nodeKey) {
+ // find the leftmost node
+ if (nodeKey.getLeftHash() != null) {
+ TrieDTO leftNode = pushNode(nodeKey.getLeftHash(), visiting);
+ if (leftNode != null) {
+ pushLeftmostNode(leftNode); // recurse on next left node
+ }
+ }
+ }
+
+ private TrieDTO pushNode(byte[] root, Deque visiting) {
+ final TrieDTO result = getNode(root);
+ if (result != null) {
+ visiting.push(result);
+ }
+ return result;
+ }
+
+ private TrieDTO getNode(byte[] hash) {
+ if (hash != null) {
+ byte[] node = this.ds.retrieveValue(hash);
+ return TrieDTO.decodeFromMessage(node, this.ds, this.preloadChildren, hash);
+ } else {
+ return null;
+ }
+ }
+
+ public TrieDTO peek() {
+ return this.visiting.peek();
+ }
+
+ public static long getOffset(TrieDTO visiting) {
+ return visiting.isTerminal() ? visiting.getTotalSize() : visiting.getSize();
+ }
+
+ public List getNodesLeftVisiting() {
+ return new ArrayList<>(visiting);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return this.from < this.to && !visiting.isEmpty();
+ }
+
+ public boolean isEmpty() {
+ return visiting.isEmpty();
+ }
+
+ public long getFrom() {
+ return from;
+ }
+
+ public List getPreRootNodes() {
+ return preRootNodes;
+ }
+}
diff --git a/rskj-core/src/main/java/co/rsk/trie/TrieDTOInOrderRecoverer.java b/rskj-core/src/main/java/co/rsk/trie/TrieDTOInOrderRecoverer.java
new file mode 100644
index 00000000000..32fd83e52c6
--- /dev/null
+++ b/rskj-core/src/main/java/co/rsk/trie/TrieDTOInOrderRecoverer.java
@@ -0,0 +1,110 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2022 RSK Labs Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+
+package co.rsk.trie;
+
+import co.rsk.crypto.Keccak256;
+import com.google.common.collect.Lists;
+import org.ethereum.crypto.Keccak256Helper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.function.Consumer;
+
+public class TrieDTOInOrderRecoverer {
+
+ private static final Logger logger = LoggerFactory.getLogger(TrieDTOInOrderRecoverer.class);
+
+ private TrieDTOInOrderRecoverer() {
+ throw new UnsupportedOperationException("This is a utility class and cannot be instantiated");
+ }
+
+ public static Optional recoverTrie(TrieDTO[] trieCollection, Consumer processTrieDTO) {
+ Optional result = recoverSubtree(trieCollection, 0, trieCollection.length - 1, processTrieDTO);
+ result.ifPresent(processTrieDTO);
+ return result;
+ }
+
+ private static Optional recoverSubtree(TrieDTO[] trieCollection, int start, int end, Consumer processTrieDTO) {
+ if (end - start < 0) {
+ return Optional.empty();
+ }
+ if (end - start == 0) {
+ return Optional.of(fromTrieDTO(trieCollection[start], Optional.empty(), Optional.empty()));
+ }
+ int indexRoot = findRoot(trieCollection, start, end);
+ Optional left = recoverSubtree(trieCollection, start, indexRoot - 1, processTrieDTO);
+ left.ifPresent(processTrieDTO);
+ Optional right = recoverSubtree(trieCollection, indexRoot + 1, end, processTrieDTO);
+ right.ifPresent(processTrieDTO);
+ return Optional.of(fromTrieDTO(trieCollection[indexRoot], left, right));
+ }
+
+ public static boolean verifyChunk(byte[] remoteRootHash, List preRootNodes, List nodes, List postRootNodes) {
+ List allNodes = Lists.newArrayList(preRootNodes);
+ allNodes.addAll(nodes);
+ allNodes.addAll(postRootNodes);
+ if (allNodes.isEmpty()) {
+ logger.warn("Received empty chunk");
+ return false;
+ }
+ TrieDTO[] nodeArray = allNodes.toArray(new TrieDTO[0]);
+ Optional result = TrieDTOInOrderRecoverer.recoverTrie(nodeArray, (t) -> {
+ });
+ if (!result.isPresent() || !Arrays.equals(remoteRootHash, result.get().calculateHash())) {
+ logger.warn("Root hash does not match! Calculated is present: {}", result.isPresent());
+ return false;
+ }
+ logger.debug("Received chunk with correct trie.");
+ return true;
+ }
+
+ private static int findRoot(TrieDTO[] trieCollection, int start, int end) {
+ int max = start;
+ for (int i = start; i <= end; i++) {
+ if (getValue(trieCollection[i]) > getValue(trieCollection[max])) {
+ max = i;
+ }
+ }
+ return max;
+ }
+
+ private static TrieDTO fromTrieDTO(TrieDTO result,
+ Optional left,
+ Optional right) {
+ left.ifPresent((leftNode) -> {
+ try {
+ Keccak256 hash = new Keccak256(Keccak256Helper.keccak256(leftNode.toMessage()));
+ result.setLeftHash(hash.getBytes());
+ } catch (Throwable e) {
+ logger.error("Error recovering left node", e);
+ }
+ });
+ right.ifPresent((rightNode) -> {
+ Keccak256 hash = new Keccak256(Keccak256Helper.keccak256(rightNode.toMessage()));
+ result.setRightHash(hash.getBytes());
+ });
+ return result;
+ }
+
+ private static long getValue(TrieDTO trieCollection) {
+ return trieCollection.getChildrenSize().value;
+ }
+
+}
diff --git a/rskj-core/src/main/java/co/rsk/trie/TrieKeySlice.java b/rskj-core/src/main/java/co/rsk/trie/TrieKeySlice.java
index b92170285ac..48d4f55a11f 100644
--- a/rskj-core/src/main/java/co/rsk/trie/TrieKeySlice.java
+++ b/rskj-core/src/main/java/co/rsk/trie/TrieKeySlice.java
@@ -107,6 +107,9 @@ public TrieKeySlice leftPad(int paddingLength) {
}
public static TrieKeySlice fromKey(byte[] key) {
+ if (key == null) {
+ return TrieKeySlice.empty();
+ }
byte[] expandedKey = PathEncoder.decode(key, key.length * 8);
return new TrieKeySlice(expandedKey, 0, expandedKey.length);
}
diff --git a/rskj-core/src/main/java/co/rsk/trie/TrieStore.java b/rskj-core/src/main/java/co/rsk/trie/TrieStore.java
index 49e05075117..81b9687a4c4 100644
--- a/rskj-core/src/main/java/co/rsk/trie/TrieStore.java
+++ b/rskj-core/src/main/java/co/rsk/trie/TrieStore.java
@@ -30,8 +30,10 @@ public interface TrieStore {
* @return an optional containing the {@link Trie} with rootHash if found
*/
Optional retrieve(byte[] hash);
-
byte[] retrieveValue(byte[] hash);
void dispose();
+
+ Optional retrieveDTO(byte[] hash);
+ void saveDTO(TrieDTO trieDTO);
}
diff --git a/rskj-core/src/main/java/co/rsk/trie/TrieStoreImpl.java b/rskj-core/src/main/java/co/rsk/trie/TrieStoreImpl.java
index 07d58cd4fbf..b102ef0ff73 100644
--- a/rskj-core/src/main/java/co/rsk/trie/TrieStoreImpl.java
+++ b/rskj-core/src/main/java/co/rsk/trie/TrieStoreImpl.java
@@ -18,6 +18,8 @@
package co.rsk.trie;
+import co.rsk.crypto.Keccak256;
+import org.ethereum.crypto.Keccak256Helper;
import org.ethereum.datasource.DataSourceWithCache;
import org.ethereum.datasource.KeyValueDataSource;
import org.slf4j.Logger;
@@ -28,11 +30,11 @@
/**
* TrieStoreImpl store and retrieve Trie node by hash
- *
+ *
* It saves/retrieves the serialized form (byte array) of a Trie node
- *
+ *
* Internally, it uses a key value data source
- *
+ *
* Created by ajlopez on 08/01/2017.
*/
public class TrieStoreImpl implements TrieStore {
@@ -142,7 +144,7 @@ private void save(Trie trie, boolean isRootNode, int level, @Nullable TraceInfo
}
@Override
- public void flush(){
+ public void flush() {
this.store.flush();
}
@@ -164,6 +166,46 @@ public Optional retrieve(byte[] hash) {
return Optional.of(trie);
}
+ @Override
+ public Optional retrieveDTO(byte[] hash) {
+ byte[] message = this.store.get(hash);
+
+ if (message == null) {
+ return Optional.empty();
+ }
+
+ TrieDTO trie = TrieDTO.decodeFromMessage(message, this);
+ return Optional.of(trie);
+ }
+
+ @Override
+ public void saveDTO(TrieDTO trieDTO) {
+ byte[] message = trieDTO.toMessage();
+ byte[] messageHash = getValueHash(message);
+ this.store.put(messageHash, message);
+ if (trieDTO.isHasLongVal()) {
+ saveLongValue(trieDTO);
+ }
+ if (trieDTO.getLeftNode() != null && trieDTO.getLeftNode().isHasLongVal()) {
+ saveLongValue(trieDTO.getLeftNode());
+ }
+ if (trieDTO.getRightNode() != null && trieDTO.getRightNode().isHasLongVal()) {
+ saveLongValue(trieDTO.getRightNode());
+ }
+ }
+
+ private void saveLongValue(TrieDTO trieDTO) {
+ if (trieDTO.isHasLongVal()) {
+ byte[] value = trieDTO.getValue();
+ byte[] valueHash = getValueHash(value);
+ this.store.put(valueHash, value);
+ }
+ }
+
+ private static byte[] getValueHash(byte[] message) {
+ return new Keccak256(Keccak256Helper.keccak256(message)).getBytes();
+ }
+
@Override
public byte[] retrieveValue(byte[] hash) {
if (logger.isTraceEnabled()) {
diff --git a/rskj-core/src/main/java/org/ethereum/config/SystemProperties.java b/rskj-core/src/main/java/org/ethereum/config/SystemProperties.java
index 69493f096b8..c50e27a38b6 100644
--- a/rskj-core/src/main/java/org/ethereum/config/SystemProperties.java
+++ b/rskj-core/src/main/java/org/ethereum/config/SystemProperties.java
@@ -298,6 +298,14 @@ public NodeFilter trustedPeers() {
return ret;
}
+ public List getSnapBootNodes() {
+ if (!configFromFiles.hasPath("sync.snapshot.client.snapBootNodes")) {
+ return Collections.emptyList();
+ }
+ List list = configFromFiles.getObjectList("sync.snapshot.client.snapBootNodes");
+ return list.stream().map(this::parsePeer).collect(Collectors.toList());
+ }
+
public Integer peerChannelReadTimeout() {
return configFromFiles.getInt("peer.channel.read.timeout");
}
diff --git a/rskj-core/src/main/java/org/ethereum/core/TransactionPool.java b/rskj-core/src/main/java/org/ethereum/core/TransactionPool.java
index 7d2b45c6886..e50073462ee 100644
--- a/rskj-core/src/main/java/org/ethereum/core/TransactionPool.java
+++ b/rskj-core/src/main/java/org/ethereum/core/TransactionPool.java
@@ -62,6 +62,8 @@ public interface TransactionPool extends InternalService {
*/
void processBest(Block block);
+ void setBestBlock(Block block);
+
void removeTransactions(List txs);
/**
diff --git a/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java b/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java
index 2fe6f48e014..7e2df441f2d 100644
--- a/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java
+++ b/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java
@@ -39,6 +39,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
import java.io.*;
import java.math.BigInteger;
import java.util.*;
@@ -88,7 +89,7 @@ public synchronized Block getBestBlock() {
long maxLevel = index.getMaxNumber();
Block bestBlock = getChainBlockByNumber(maxLevel);
if (bestBlock != null) {
- return bestBlock;
+ return bestBlock;
}
// That scenario can happen
@@ -113,7 +114,7 @@ public byte[] getBlockHashByNumber(long blockNumber, byte[] branchBlockHash) {
throw new IllegalArgumentException(String.format("Requested block number > branch hash number: %d < %d",
blockNumber, branchBlock.getNumber()));
}
- while(branchBlock.getNumber() > blockNumber) {
+ while (branchBlock.getNumber() > blockNumber) {
branchBlock = getBlockByHash(branchBlock.getParentHash().getBytes());
}
return branchBlock.getHash().getBytes();
@@ -153,7 +154,7 @@ public Block getBlockAtDepthStartingAt(long depth, byte[] hash) {
return block;
}
- public boolean isBlockInMainChain(long blockNumber, Keccak256 blockHash){
+ public boolean isBlockInMainChain(long blockNumber, Keccak256 blockHash) {
List blockInfos = index.getBlocksByNumber(blockNumber);
if (blockInfos == null) {
return false;
@@ -241,7 +242,7 @@ public boolean isEmpty() {
}
@Override
- public synchronized Block getChainBlockByNumber(long number){
+ public synchronized Block getChainBlockByNumber(long number) {
List blockInfos = index.getBlocksByNumber(number);
for (BlockInfo blockInfo : blockInfos) {
@@ -294,14 +295,14 @@ public synchronized boolean isBlockExist(byte[] hash) {
}
@Override
- public synchronized BlockDifficulty getTotalDifficultyForHash(byte[] hash){
+ public synchronized BlockDifficulty getTotalDifficultyForHash(byte[] hash) {
Block block = this.getBlockByHash(hash);
if (block == null) {
return ZERO;
}
long level = block.getNumber();
- List blockInfos = index.getBlocksByNumber(level);
+ List blockInfos = index.getBlocksByNumber(level);
for (BlockInfo blockInfo : blockInfos) {
if (Arrays.equals(blockInfo.getHash().getBytes(), hash)) {
@@ -323,7 +324,7 @@ public long getMinNumber() {
}
@Override
- public synchronized List getListHashesEndWith(byte[] hash, long number){
+ public synchronized List getListHashesEndWith(byte[] hash, long number) {
List blocks = getListBlocksEndWith(hash, number);
List hashes = new ArrayList<>(blocks.size());
@@ -357,7 +358,7 @@ private synchronized List getListBlocksEndWith(byte[] hash, long qty) {
}
@Override
- public synchronized void reBranch(Block forkBlock){
+ public synchronized void reBranch(Block forkBlock) {
Block bestBlock = getBestBlock();
long maxLevel = Math.max(bestBlock.getNumber(), forkBlock.getNumber());
@@ -368,7 +369,7 @@ public synchronized void reBranch(Block forkBlock){
if (forkBlock.getNumber() > bestBlock.getNumber()) {
- while(currentLevel > bestBlock.getNumber()) {
+ while (currentLevel > bestBlock.getNumber()) {
List blocks = index.getBlocksByNumber(currentLevel);
BlockInfo blockInfo = getBlockInfoForHash(blocks, forkLine.getHash().getBytes());
if (blockInfo != null) {
@@ -383,10 +384,10 @@ public synchronized void reBranch(Block forkBlock){
}
Block bestLine = bestBlock;
- if (bestBlock.getNumber() > forkBlock.getNumber()){
+ if (bestBlock.getNumber() > forkBlock.getNumber()) {
- while(currentLevel > forkBlock.getNumber()) {
- List blocks = index.getBlocksByNumber(currentLevel);
+ while (currentLevel > forkBlock.getNumber()) {
+ List blocks = index.getBlocksByNumber(currentLevel);
BlockInfo blockInfo = getBlockInfoForHash(blocks, bestLine.getHash().getBytes());
if (blockInfo != null) {
blockInfo.setMainChain(false);
@@ -400,7 +401,7 @@ public synchronized void reBranch(Block forkBlock){
}
// 2. Loop back on each level until common block
- while( !bestLine.isEqual(forkLine) ) {
+ while (!bestLine.isEqual(forkLine)) {
List levelBlocks = index.getBlocksByNumber(currentLevel);
BlockInfo bestInfo = getBlockInfoForHash(levelBlocks, bestLine.getHash().getBytes());
@@ -433,7 +434,7 @@ public synchronized List getListHashesStartWith(long number, long maxBlo
int i;
for (i = 0; i < maxBlocks; ++i) {
- List blockInfos = index.getBlocksByNumber(number);
+ List blockInfos = index.getBlocksByNumber(number);
if (blockInfos == null) {
break;
}
@@ -452,8 +453,7 @@ public synchronized List getListHashesStartWith(long number, long maxBlo
}
-
- private static BlockInfo getBlockInfoForHash(List blocks, byte[] hash){
+ private static BlockInfo getBlockInfoForHash(List blocks, byte[] hash) {
if (blocks == null) {
return null;
}
@@ -468,16 +468,17 @@ private static BlockInfo getBlockInfoForHash(List blocks, byte[] hash
}
@Override
- public synchronized List getChainBlocksByNumber(long number){
+ @Nonnull
+ public synchronized List getChainBlocksByNumber(long number) {
List result = new ArrayList<>();
List blockInfos = index.getBlocksByNumber(number);
- if (blockInfos == null){
+ if (blockInfos == null) {
return result;
}
- for (BlockInfo blockInfo : blockInfos){
+ for (BlockInfo blockInfo : blockInfos) {
byte[] hash = blockInfo.getHash().getBytes();
Block block = getBlockByHash(hash);
@@ -516,19 +517,20 @@ public void rewind(long blockNumber) {
/**
* When a block is processed on remasc the contract needs to calculate all siblings that
* that should be rewarded when fees on this block are paid
+ *
* @param block the block is looked for siblings
* @return
*/
private Map> getSiblingsFromBlock(Block block) {
return block.getUncleList().stream()
.collect(
- Collectors.groupingBy(
- BlockHeader::getNumber,
- Collectors.mapping(
- header -> new Sibling(header, block.getCoinbase(), block.getNumber()),
- Collectors.toList()
+ Collectors.groupingBy(
+ BlockHeader::getNumber,
+ Collectors.mapping(
+ header -> new Sibling(header, block.getCoinbase(), block.getNumber()),
+ Collectors.toList()
+ )
)
- )
);
}
@@ -566,7 +568,7 @@ public void setMainChain(boolean mainChain) {
}
- public static final Serializer> BLOCK_INFO_SERIALIZER = new Serializer>(){
+ public static final Serializer> BLOCK_INFO_SERIALIZER = new Serializer>() {
@Override
public void serialize(DataOutput out, List value) throws IOException {
@@ -592,7 +594,7 @@ public List deserialize(DataInput in, int available) throws IOExcepti
ByteArrayInputStream bis = new ByteArrayInputStream(data, 0, data.length);
ObjectInputStream ois = new ObjectInputStream(bis);
- value = (ArrayList)ois.readObject();
+ value = (ArrayList) ois.readObject();
} catch (ClassNotFoundException e) {
logger.error("Class not found", e);
}
diff --git a/rskj-core/src/main/java/org/ethereum/net/NodeManager.java b/rskj-core/src/main/java/org/ethereum/net/NodeManager.java
index ebe67812573..3535353b1a3 100644
--- a/rskj-core/src/main/java/org/ethereum/net/NodeManager.java
+++ b/rskj-core/src/main/java/org/ethereum/net/NodeManager.java
@@ -19,8 +19,8 @@
package org.ethereum.net;
+import co.rsk.config.RskSystemProperties;
import co.rsk.net.discovery.PeerExplorer;
-import org.ethereum.config.SystemProperties;
import org.ethereum.net.rlpx.Node;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,7 +53,7 @@ public class NodeManager {
private boolean discoveryEnabled;
- public NodeManager(PeerExplorer peerExplorer, SystemProperties config) {
+ public NodeManager(PeerExplorer peerExplorer, RskSystemProperties config) {
this.peerExplorer = peerExplorer;
this.discoveryEnabled = config.isPeerDiscoveryEnabled();
@@ -62,6 +62,13 @@ public NodeManager(PeerExplorer peerExplorer, SystemProperties config) {
handler.getNodeStatistics().setPredefined(true);
createNodeHandler(node);
}
+ if(config.isClientSnapshotSyncEnabled()){
+ for (Node snapNode : config.getSnapBootNodes()) {
+ NodeHandler handler = new NodeHandler(snapNode);
+ handler.getNodeStatistics().setPredefined(true);
+ createNodeHandler(snapNode);
+ }
+ }
}
private synchronized NodeHandler getNodeHandler(Node n) {
diff --git a/rskj-core/src/main/java/org/ethereum/net/client/Capability.java b/rskj-core/src/main/java/org/ethereum/net/client/Capability.java
index 3ec133ee2dc..c8eb98b1669 100644
--- a/rskj-core/src/main/java/org/ethereum/net/client/Capability.java
+++ b/rskj-core/src/main/java/org/ethereum/net/client/Capability.java
@@ -28,6 +28,8 @@ public class Capability implements Comparable {
public static final String P2P = "p2p";
public static final String RSK = "rsk";
+ public static final String SNAP = "snap";
+ public static final byte SNAP_VERSION = (byte) 1;
private final String name;
private final byte version;
diff --git a/rskj-core/src/main/java/org/ethereum/net/client/ConfigCapabilitiesImpl.java b/rskj-core/src/main/java/org/ethereum/net/client/ConfigCapabilitiesImpl.java
index e9deda7d12f..eb20c0e2b64 100644
--- a/rskj-core/src/main/java/org/ethereum/net/client/ConfigCapabilitiesImpl.java
+++ b/rskj-core/src/main/java/org/ethereum/net/client/ConfigCapabilitiesImpl.java
@@ -19,7 +19,7 @@
package org.ethereum.net.client;
-import org.ethereum.config.SystemProperties;
+import co.rsk.config.RskSystemProperties;
import org.ethereum.net.eth.EthVersion;
import org.ethereum.net.p2p.HelloMessage;
@@ -29,6 +29,8 @@
import java.util.TreeSet;
import static org.ethereum.net.client.Capability.RSK;
+import static org.ethereum.net.client.Capability.SNAP;
+import static org.ethereum.net.client.Capability.SNAP_VERSION;
import static org.ethereum.net.eth.EthVersion.fromCode;
/**
@@ -36,24 +38,30 @@
*/
public class ConfigCapabilitiesImpl implements ConfigCapabilities{
- private final SystemProperties config;
+ private final RskSystemProperties config;
- private SortedSet allCaps = new TreeSet<>();
+ private SortedSet allCapabilities = new TreeSet<>();
- public ConfigCapabilitiesImpl(SystemProperties config) {
+ public ConfigCapabilitiesImpl(RskSystemProperties config) {
if (config.syncVersion() != null) {
EthVersion eth = fromCode(config.syncVersion());
if (eth != null) {
- allCaps.add(new Capability(RSK, eth.getCode()));
+ allCapabilities.add(new Capability(RSK, eth.getCode()));
}
} else {
for (EthVersion v : EthVersion.supported()) {
- allCaps.add(new Capability(RSK, v.getCode()));
+ allCapabilities.add(new Capability(RSK, v.getCode()));
}
}
+
+ if (config.isSnapshotSyncEnabled() && allCapabilities.stream().anyMatch(Capability::isRSK)) {
+ allCapabilities.add(new Capability(SNAP, SNAP_VERSION));
+ }
+
this.config = config;
}
+
/**
* Gets the capabilities listed in 'peer.capabilities' config property
* sorted by their names.
@@ -61,7 +69,7 @@ public ConfigCapabilitiesImpl(SystemProperties config) {
public List getConfigCapabilities() {
List ret = new ArrayList<>();
List caps = config.peerCapabilities();
- for (Capability capability : allCaps) {
+ for (Capability capability : allCapabilities) {
if (caps.contains(capability.getName())) {
ret.add(capability);
}
diff --git a/rskj-core/src/main/java/org/ethereum/net/rlpx/HandshakeHandler.java b/rskj-core/src/main/java/org/ethereum/net/rlpx/HandshakeHandler.java
index ea6db77e41c..36ae4391844 100644
--- a/rskj-core/src/main/java/org/ethereum/net/rlpx/HandshakeHandler.java
+++ b/rskj-core/src/main/java/org/ethereum/net/rlpx/HandshakeHandler.java
@@ -377,13 +377,15 @@ private void processHelloMessage(ChannelHandlerContext ctx, HelloMessage helloMe
List capInCommon = configCapabilities.getSupportedCapabilities(helloMessage);
channel.initMessageCodes(capInCommon);
for (Capability capability : capInCommon) {
- // It seems that the only supported capability is RSK, and everything else is ignored.
+ // RSK Capability is the condition needed to be able to finish the Handshake successfully.
if (Capability.RSK.equals(capability.getName())) {
publicRLPxHandshakeFinished(ctx, helloMessage, fromCode(capability.getVersion()));
return;
}
}
+ // RSK must be present. If RSK is not found, the handshake process does not complete.
+ // Other capabilities, such as SNAP, cannot be considered if RSK is not supported.
throw new RuntimeException("The remote peer didn't support the RSK capability");
}
diff --git a/rskj-core/src/main/java/org/ethereum/net/server/Channel.java b/rskj-core/src/main/java/org/ethereum/net/server/Channel.java
index a98f58df0ea..a79390d7dcd 100644
--- a/rskj-core/src/main/java/org/ethereum/net/server/Channel.java
+++ b/rskj-core/src/main/java/org/ethereum/net/server/Channel.java
@@ -25,6 +25,7 @@
import co.rsk.net.eth.RskWireProtocol;
import co.rsk.net.messages.Message;
import co.rsk.net.messages.MessageType;
+import com.google.common.annotations.VisibleForTesting;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import org.ethereum.net.MessageQueue;
@@ -50,6 +51,7 @@
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -75,6 +77,7 @@ public class Channel implements Peer {
private final PeerStatistics peerStats = new PeerStatistics();
private Stats stats;
+ private final boolean isSnapCapable;
public Channel(MessageQueue msgQueue,
MessageCodec messageCodec,
@@ -82,7 +85,8 @@ public Channel(MessageQueue msgQueue,
RskWireProtocol.Factory rskWireProtocolFactory,
Eth62MessageFactory eth62MessageFactory,
StaticMessages staticMessages,
- String remoteId) {
+ String remoteId,
+ List capabilities) {
this.msgQueue = msgQueue;
this.messageCodec = messageCodec;
this.nodeManager = nodeManager;
@@ -91,6 +95,19 @@ public Channel(MessageQueue msgQueue,
this.staticMessages = staticMessages;
this.isActive = remoteId != null && !remoteId.isEmpty();
this.stats = new Stats();
+ this.isSnapCapable = capabilities.stream()
+ .anyMatch(capability -> Capability.SNAP.equals(capability.getName()));
+ }
+
+ @VisibleForTesting
+ public Channel(MessageQueue msgQueue,
+ MessageCodec messageCodec,
+ NodeManager nodeManager,
+ RskWireProtocol.Factory rskWireProtocolFactory,
+ Eth62MessageFactory eth62MessageFactory,
+ StaticMessages staticMessages,
+ String remoteId) {
+ this(msgQueue, messageCodec, nodeManager, rskWireProtocolFactory, eth62MessageFactory, staticMessages, remoteId, new ArrayList<>());
}
public void sendHelloMessage(ChannelHandlerContext ctx, FrameCodec frameCodec, String nodeId,
@@ -263,6 +280,11 @@ public void imported(boolean best) {
stats.imported(best);
}
+ @Override
+ public boolean isSnapCapable() {
+ return isSnapCapable;
+ }
+
@Override
public String toString() {
return String.format("%s | %s", getPeerId(), inetSocketAddress);
diff --git a/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java b/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java
index 7d0f478f7ff..f38bd7eacf8 100644
--- a/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java
+++ b/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java
@@ -111,7 +111,7 @@ public void initChannel(NioSocketChannel ch) {
P2pHandler p2pHandler = new P2pHandler(ethereumListener, messageQueue, config.getPeerP2PPingInterval());
MessageCodec messageCodec = new MessageCodec(ethereumListener, config);
HandshakeHandler handshakeHandler = new HandshakeHandler(config, peerScoringManager, p2pHandler, messageCodec, configCapabilities);
- Channel channel = new Channel(messageQueue, messageCodec, nodeManager, rskWireProtocolFactory, eth62MessageFactory, staticMessages, remoteId);
+ Channel channel = new Channel(messageQueue, messageCodec, nodeManager, rskWireProtocolFactory, eth62MessageFactory, staticMessages, remoteId, configCapabilities.getConfigCapabilities());
ch.pipeline().addLast("readTimeoutHandler", new ReadTimeoutHandler(config.peerChannelReadTimeout(), TimeUnit.SECONDS));
ch.pipeline().addLast("handshakeHandler", handshakeHandler);
diff --git a/rskj-core/src/main/java/org/ethereum/net/server/Stats.java b/rskj-core/src/main/java/org/ethereum/net/server/Stats.java
index 0cf33879393..1cdcd9929fa 100644
--- a/rskj-core/src/main/java/org/ethereum/net/server/Stats.java
+++ b/rskj-core/src/main/java/org/ethereum/net/server/Stats.java
@@ -117,7 +117,6 @@ public double score(MessageType type) {
}
private double priority(MessageType type) {
-
switch (type) {
case TRANSACTIONS:
return 2;
@@ -151,8 +150,21 @@ private double priority(MessageType type) {
return 0.5;
case BLOCK_HEADERS_RESPONSE_MESSAGE:
return 5;
+ case SNAP_BLOCKS_REQUEST_MESSAGE:
+ return 1;
+ case SNAP_BLOCKS_RESPONSE_MESSAGE:
+ return 3;
+ case SNAP_STATUS_REQUEST_MESSAGE:
+ return 1;
+ case SNAP_STATUS_RESPONSE_MESSAGE:
+ return 3;
+ case SNAP_STATE_CHUNK_REQUEST_MESSAGE:
+ return 1;
+ case SNAP_STATE_CHUNK_RESPONSE_MESSAGE:
+ return 3;
+ default:
+ return 0.0;
}
- return 0.0;
}
public synchronized void imported(boolean best) {
if (best) {
diff --git a/rskj-core/src/main/resources/expected.conf b/rskj-core/src/main/resources/expected.conf
index 6dc5b16c5c5..236d60c62e1 100644
--- a/rskj-core/src/main/resources/expected.conf
+++ b/rskj-core/src/main/resources/expected.conf
@@ -273,6 +273,25 @@ sync = {
version =
waitForSync =
topBest =
+ snapshot = {
+ server = {
+ enabled =
+ }
+ client = {
+ enabled =
+ parallel =
+ chunkSize =
+ chunkRequestTimeout =
+ limit =
+ snapBootNodes = [
+ {
+ nodeId =
+ ip =
+ port =
+ }
+ ]
+ }
+ }
}
rpc = {
callGasCap =
diff --git a/rskj-core/src/main/resources/logback.xml b/rskj-core/src/main/resources/logback.xml
index e32d51130fb..cb777a4f3af 100644
--- a/rskj-core/src/main/resources/logback.xml
+++ b/rskj-core/src/main/resources/logback.xml
@@ -37,7 +37,7 @@
- ./logs/rsk.log
+ ${logging.dir:-./logs}/rsk.log
diff --git a/rskj-core/src/main/resources/reference.conf b/rskj-core/src/main/resources/reference.conf
index a7d71a2b4db..86fa6b74016 100644
--- a/rskj-core/src/main/resources/reference.conf
+++ b/rskj-core/src/main/resources/reference.conf
@@ -368,6 +368,28 @@ sync {
# X % of top best nodes will be considered when for random selection
topBest = 0
+
+ # (experimental, OFF by default for both client and server)
+ snapshot = {
+ server = {
+ # Server / snapshot sync enabled
+ enabled = false
+ }
+ client = {
+ # Client / snapshot sync enabled
+ enabled = false
+ # Server / chunk size
+ chunkSize = 50
+ # Request timeout (in seconds)
+ chunkRequestTimeout = 120
+ # Distance to the tip of the blockchain to start snapshot sync
+ limit = 1000000
+ # Parallel requests (true, false)
+ parallel = true
+ # list of SNAP-capable peers to connect to
+ snapBootNodes = []
+ }
+ }
}
rpc {
diff --git a/rskj-core/src/test/java/co/rsk/config/RskSystemPropertiesTest.java b/rskj-core/src/test/java/co/rsk/config/RskSystemPropertiesTest.java
index 8b82cded3fe..72dbbdfec59 100644
--- a/rskj-core/src/test/java/co/rsk/config/RskSystemPropertiesTest.java
+++ b/rskj-core/src/test/java/co/rsk/config/RskSystemPropertiesTest.java
@@ -18,16 +18,15 @@
package co.rsk.config;
-import co.rsk.cli.CliArgs;
import co.rsk.cli.RskCli;
import co.rsk.rpc.ModuleDescription;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigException;
import com.typesafe.config.ConfigFactory;
+import org.ethereum.net.rlpx.Node;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
-import org.mockito.Mock;
import java.util.List;
import java.util.Map;
@@ -45,8 +44,6 @@
class RskSystemPropertiesTest {
private final TestSystemProperties config = new TestSystemProperties();
- @Mock
- private CliArgs cliArgs;
@Test
void defaultValues() {
@@ -135,6 +132,80 @@ void testRpcModules() {
assertTrue(enabledModuleNames.stream().allMatch(k -> moduleNameEnabledMap.get(k).get(0).isEnabled()));
}
+
+ @Test
+ void rskCliSnapNodes_ShouldSetSnapBootNodes() {
+ RskCli rskCli = new RskCli();
+ String[] snapNodesArgs = {
+ "--snap-nodes=enode://b2a304b30b3ff90aabcb5e37fa3cc70511c9f5bf457d6d8bfb6f0905baf6d714b66a73fede2ea0671b3a4d1af2aed3379d7eb9340d775ae27800e0757dc1e502@3.94.45.146:50501",
+ "--snap-nodes=enode://b2a304b30b3ff90bbbcb5e37fa3cc70511c9f5bf457d6d8bfb6f0905baf6d714b66a73fede2ea0671b3a4d1af2aed3379d7eb9340d775ae27800e0757dc10502@3.94.45.146:50501"
+ };
+ rskCli.load(snapNodesArgs);
+
+ RskSystemProperties rskSystemProperties = new RskSystemProperties(
+ new ConfigLoader(
+ rskCli.getCliArgs()
+ )
+ );
+
+ Node expectedFirstSnapNode = new Node("enode://b2a304b30b3ff90aabcb5e37fa3cc70511c9f5bf457d6d8bfb6f0905baf6d714b66a73fede2ea0671b3a4d1af2aed3379d7eb9340d775ae27800e0757dc1e502@3.94.45.146:50501");
+ Node expectedSecondSnapNode = new Node("enode://b2a304b30b3ff90bbbcb5e37fa3cc70511c9f5bf457d6d8bfb6f0905baf6d714b66a73fede2ea0671b3a4d1af2aed3379d7eb9340d775ae27800e0757dc10502@3.94.45.146:50501");
+
+ Assertions.assertEquals(2, rskSystemProperties.getSnapBootNodes().size());
+ Assertions.assertEquals(expectedFirstSnapNode.getHexId(), rskSystemProperties.getSnapBootNodes().get(0).getHexId());
+ Assertions.assertEquals(expectedSecondSnapNode.getHexId(), rskSystemProperties.getSnapBootNodes().get(1).getHexId());
+ Assertions.assertEquals(expectedFirstSnapNode.getId(), rskSystemProperties.getSnapBootNodes().get(0).getId());
+ Assertions.assertEquals(expectedSecondSnapNode.getId(), rskSystemProperties.getSnapBootNodes().get(1).getId());
+ }
+
+ @Test
+ void rskCliSnapNodes_ShouldReturnZeroSnapBootNodesForInvalidNodeFormat() {
+ RskCli rskCli = new RskCli();
+ String[] snapNodesArgs = {
+ "--snap-nodes=http://www.google.es",
+ };
+
+ rskCli.load(snapNodesArgs);
+
+ Assertions.assertThrows(RuntimeException.class, () -> {
+ RskSystemProperties rskSystemProperties = new RskSystemProperties(
+ new ConfigLoader(rskCli.getCliArgs())
+ );
+
+ Assertions.assertEquals(0, rskSystemProperties.getSnapBootNodes().size());
+ });
+ }
+
+ @Test
+ void rskCliSyncMode_ShouldSetSyncMode() {
+ RskCli rskCli = new RskCli();
+ String[] snapNodesArgs = {"--sync-mode=snap"};
+ rskCli.load(snapNodesArgs);
+
+ RskSystemProperties rskSystemProperties = new RskSystemProperties(
+ new ConfigLoader(
+ rskCli.getCliArgs()
+ )
+ );
+
+ Assertions.assertTrue(rskSystemProperties.isClientSnapshotSyncEnabled());
+ }
+
+ @Test
+ void rskCliSyncMode_ShouldSetDefaultSyncMode() {
+ RskCli rskCli = new RskCli();
+ String[] snapNodesArgs = {"--sync-mode=full"};
+ rskCli.load(snapNodesArgs);
+
+ RskSystemProperties rskSystemProperties = new RskSystemProperties(
+ new ConfigLoader(
+ rskCli.getCliArgs()
+ )
+ );
+
+ Assertions.assertFalse(rskSystemProperties.isClientSnapshotSyncEnabled());
+ }
+
@Test
void testGetRpcModulesWithList() {
TestSystemProperties testSystemProperties = new TestSystemProperties(rawConfig ->
diff --git a/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerTest.java b/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerTest.java
index e4c88ab8149..09cbf6f5cca 100644
--- a/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerTest.java
+++ b/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerTest.java
@@ -43,18 +43,24 @@
import org.ethereum.core.*;
import org.ethereum.db.BlockStore;
import org.ethereum.listener.EthereumListener;
+import org.ethereum.net.NodeManager;
+import org.ethereum.net.server.Channel;
import org.ethereum.net.server.ChannelManager;
import org.ethereum.rpc.Simples.SimpleChannelManager;
+import org.ethereum.util.ByteUtil;
import org.ethereum.util.RskMockFactory;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.slf4j.Logger;
import javax.annotation.Nonnull;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.TimeUnit;
@@ -76,7 +82,7 @@ void processBlockMessageUsingProcessor() {
SimplePeer sender = new SimplePeer();
PeerScoringManager scoring = createPeerScoringManager();
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring, mock(StatusResolver.class));
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, scoring, mock(StatusResolver.class));
Block block = new BlockChainBuilder().ofSize(1, true).getBestBlock();
Message message = new BlockMessage(block);
@@ -108,7 +114,7 @@ void skipProcessGenesisBlock() {
SimplePeer sender = new SimplePeer();
PeerScoringManager scoring = createPeerScoringManager();
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring, mock(StatusResolver.class));
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, scoring, mock(StatusResolver.class));
Block block = new BlockGenerator().getGenesisBlock();
Message message = new BlockMessage(block);
@@ -130,7 +136,7 @@ void skipAdvancedBlock() {
PeerScoringManager scoring = createPeerScoringManager();
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
sbp.setBlockGap(100000);
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring, mock(StatusResolver.class));
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, scoring, mock(StatusResolver.class));
Block block = new BlockGenerator().createBlock(200000, 0);
Message message = new BlockMessage(block);
@@ -151,7 +157,7 @@ void postBlockMessageTwice() {
Peer sender = new SimplePeer();
PeerScoringManager scoring = createPeerScoringManager();
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, scoring,
mock(StatusResolver.class));
Block block = new BlockChainBuilder().ofSize(1, true).getBestBlock();
Message message = new BlockMessage(block);
@@ -171,7 +177,7 @@ void postBlockMessageTwice() {
@SuppressWarnings("squid:S2925") // Thread.sleep() used
void postBlockMessageUsingProcessor() throws InterruptedException {
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, null,
mock(StatusResolver.class));
Block block = new BlockChainBuilder().ofSize(1, true).getBestBlock();
Message message = new BlockMessage(block);
@@ -200,7 +206,7 @@ void postBlockMessageFromBannedMiner() {
RskAddress bannedMiner = block.getCoinbase();
doReturn(Collections.singletonList(bannedMiner.toHexString())).when(config).bannedMinerList();
- NodeMessageHandler nodeMessageHandler = new NodeMessageHandler(config, sbp, null, null, null, scoring,
+ NodeMessageHandler nodeMessageHandler = new NodeMessageHandler(config, sbp, null,null, null, null, scoring,
mock(StatusResolver.class));
nodeMessageHandler.postMessage(sender, message, null);
@@ -208,6 +214,105 @@ void postBlockMessageFromBannedMiner() {
Assertions.assertEquals(0, nodeMessageHandler.getMessageQueueSize());
}
+ @Test
+ void checkSnapMessagesOrderAndPriority() {
+ RskSystemProperties config = spy(this.config);
+ PeerScoringManager scoring = createPeerScoringManager();
+ SimpleBlockProcessor sbp = new SimpleBlockProcessor();
+ SyncProcessor syncProcessor = mock(SyncProcessor.class);
+ StatusResolver statusResolver = mock(StatusResolver.class);
+ Status status = mock(Status.class);
+ ChannelManager channelManager = mock(ChannelManager.class);
+
+ // Mock snap messages
+ Message snapStateChunkRequestMessage = Mockito.mock(SnapStateChunkRequestMessage.class);
+ Message snapStateChunkResponseMessage = Mockito.mock(SnapStateChunkResponseMessage.class);
+ Message snapStatusRequestMessage = Mockito.mock(SnapStatusRequestMessage.class);
+ Message snapStatusResponseMessage = Mockito.mock(SnapStatusResponseMessage.class);
+ Message snapBlocksRequestMessage = Mockito.mock(SnapBlocksRequestMessage.class);
+ Message snapBlocksResponseMessage = Mockito.mock(SnapBlocksResponseMessage.class);
+
+ Mockito.when(snapStateChunkRequestMessage.getMessageType()).thenReturn(MessageType.SNAP_STATE_CHUNK_REQUEST_MESSAGE);
+ Mockito.when(snapStateChunkResponseMessage.getMessageType()).thenReturn(MessageType.SNAP_STATE_CHUNK_RESPONSE_MESSAGE);
+ Mockito.when(snapStatusRequestMessage.getMessageType()).thenReturn(MessageType.SNAP_STATUS_REQUEST_MESSAGE);
+ Mockito.when(snapStatusResponseMessage.getMessageType()).thenReturn(MessageType.SNAP_STATUS_RESPONSE_MESSAGE);
+ Mockito.when(snapBlocksRequestMessage.getMessageType()).thenReturn(MessageType.SNAP_BLOCKS_REQUEST_MESSAGE);
+ Mockito.when(snapBlocksResponseMessage.getMessageType()).thenReturn(MessageType.SNAP_BLOCKS_RESPONSE_MESSAGE);
+
+ Mockito.when(status.getBestBlockNumber()).thenReturn(0L);
+ Mockito.when(status.getBestBlockHash()).thenReturn(ByteUtil.intToBytes(0));
+ Mockito.when(channelManager.broadcastStatus(any())).thenReturn(0);
+ Mockito.when(statusResolver.currentStatus()).thenReturn(status);
+
+ Channel sender = new Channel(null, null, mock(NodeManager.class), null, null, null, null);
+ InetAddress inetAddress = InetAddress.getLoopbackAddress();
+ InetSocketAddress inetSocketAddress = new InetSocketAddress(inetAddress, 500);
+ sender.setInetSocketAddress(inetSocketAddress);
+ sender.setNode(new NodeID(TestUtils.generatePeerId("peer")).getID());
+
+ RskAddress bannedMiner = new RskAddress("0000000000000000000000000000000000000023");
+ doReturn(Collections.singletonList(bannedMiner.toHexString())).when(config).bannedMinerList();
+
+ MessageCounter messageCounter = mock(MessageCounter.class);
+ Mockito.doThrow(new IllegalAccessError()).when(messageCounter).decrement(any());
+
+ NodeMessageHandler nodeMessageHandler = Mockito.spy( new NodeMessageHandler(
+ config,
+ sbp,
+ syncProcessor,
+ null,
+ channelManager,
+ null,
+ scoring,
+ statusResolver,
+ Mockito.mock(Thread.class),
+ messageCounter
+ ));
+
+ nodeMessageHandler.postMessage(sender, snapStateChunkRequestMessage, null);
+ nodeMessageHandler.postMessage(sender, snapStateChunkResponseMessage, null);
+ nodeMessageHandler.postMessage(sender, snapStatusRequestMessage, null);
+ nodeMessageHandler.postMessage(sender, snapStatusResponseMessage, null);
+ nodeMessageHandler.postMessage(sender, snapBlocksRequestMessage, null);
+ nodeMessageHandler.postMessage(sender, snapBlocksResponseMessage, null);
+
+ Assertions.assertEquals(6, nodeMessageHandler.getMessageQueueSize());
+
+ ArgumentCaptor snapMessagesCaptor = ArgumentCaptor.forClass(Message.class);
+
+ nodeMessageHandler.start();
+ // Snap responses scores = 300
+ // Snap requests scores = 100
+
+ nodeMessageHandler.run();
+ Mockito.verify(nodeMessageHandler, atLeastOnce()).processMessage(any(Peer.class), snapMessagesCaptor.capture());
+ Assertions.assertEquals(MessageType.SNAP_STATE_CHUNK_RESPONSE_MESSAGE, snapMessagesCaptor.getValue().getMessageType());
+
+ nodeMessageHandler.run();
+ Mockito.verify(nodeMessageHandler, atLeastOnce()).processMessage(any(Peer.class), snapMessagesCaptor.capture());
+ Assertions.assertEquals(MessageType.SNAP_STATUS_RESPONSE_MESSAGE, snapMessagesCaptor.getValue().getMessageType());
+
+ nodeMessageHandler.run();
+ Mockito.verify(nodeMessageHandler, atLeastOnce()).processMessage(any(Peer.class), snapMessagesCaptor.capture());
+ Assertions.assertEquals(MessageType.SNAP_BLOCKS_RESPONSE_MESSAGE, snapMessagesCaptor.getValue().getMessageType());
+
+ nodeMessageHandler.run();
+ Mockito.verify(nodeMessageHandler, atLeastOnce()).processMessage(any(Peer.class), snapMessagesCaptor.capture());
+ Assertions.assertEquals(MessageType.SNAP_STATE_CHUNK_REQUEST_MESSAGE, snapMessagesCaptor.getValue().getMessageType());
+
+ nodeMessageHandler.run();
+ Mockito.verify(nodeMessageHandler, atLeastOnce()).processMessage(any(Peer.class), snapMessagesCaptor.capture());
+ Assertions.assertEquals(MessageType.SNAP_BLOCKS_REQUEST_MESSAGE, snapMessagesCaptor.getValue().getMessageType());
+
+ nodeMessageHandler.run();
+ Mockito.verify(nodeMessageHandler, atLeastOnce()).processMessage(any(Peer.class), snapMessagesCaptor.capture());
+ Assertions.assertEquals(MessageType.SNAP_STATUS_REQUEST_MESSAGE, snapMessagesCaptor.getValue().getMessageType());
+
+ nodeMessageHandler.stop();
+
+ Assertions.assertEquals(0, nodeMessageHandler.getMessageQueueSize());
+ }
+
@Test
void postBlockMessageFromNonBannedMiner() {
RskSystemProperties config = spy(this.config);
@@ -223,7 +328,7 @@ void postBlockMessageFromNonBannedMiner() {
doReturn(Collections.singletonList(bannedMiner.toHexString())).when(config).bannedMinerList();
- NodeMessageHandler nodeMessageHandler = new NodeMessageHandler(config, sbp, null, null, null, scoring,
+ NodeMessageHandler nodeMessageHandler = new NodeMessageHandler(config, sbp, null,null, null, null, scoring,
mock(StatusResolver.class));
nodeMessageHandler.postMessage(sender, message, null);
@@ -237,7 +342,7 @@ public void processInvalidPoWMessageUsingProcessor() {
SimplePeer sender = new SimplePeer();
PeerScoringManager scoring = createPeerScoringManager();
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null,null, null, null, scoring,
mock(StatusResolver.class));
Block block = new BlockChainBuilder().ofSize(1, true).getBestBlock();
byte[] mergedMiningHeader = block.getBitcoinMergedMiningHeader();
@@ -264,7 +369,7 @@ void processMissingPoWBlockMessageUsingProcessor() {
SimplePeer sender = new SimplePeer();
PeerScoringManager scoring = createPeerScoringManager();
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring, mock(StatusResolver.class));
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null,null, null, null, scoring, mock(StatusResolver.class));
BlockGenerator blockGenerator = new BlockGenerator();
Block block = blockGenerator.getGenesisBlock();
@@ -292,7 +397,7 @@ void processMissingPoWBlockMessageUsingProcessor() {
@Test
void processFutureBlockMessageUsingProcessor() {
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null,null, null, null, null,
mock(StatusResolver.class));
Block block = new BlockGenerator().getGenesisBlock();
Message message = new BlockMessage(block);
@@ -313,7 +418,7 @@ void processStatusMessageUsingNodeBlockProcessor() {
BlockSyncService blockSyncService = new BlockSyncService(config, store, blockchain, nodeInformation, syncConfiguration, DummyBlockValidator.VALID_RESULT_INSTANCE);
final NodeBlockProcessor bp = new NodeBlockProcessor(store, blockchain, nodeInformation, blockSyncService, syncConfiguration);
final SimplePeer sender = new SimplePeer();
- final NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null,
+ final NodeMessageHandler handler = new NodeMessageHandler(config, bp, null,null, null, null, null,
mock(StatusResolver.class));
BlockGenerator blockGenerator = new BlockGenerator();
@@ -377,7 +482,7 @@ blockStore, mock(ConsensusValidationMainchainView.class),
null,
new PeersInformation(RskMockFactory.getChannelManager(), syncConfiguration, blockchain, RskMockFactory.getPeerScoringManager()), mock(Genesis.class), mock(EthereumListener.class));
final NodeMessageHandler handler = new NodeMessageHandler(config,
- bp, syncProcessor, null, null,
+ bp, syncProcessor, null, null, null,
null, mock(StatusResolver.class));
BlockGenerator blockGenerator = new BlockGenerator();
@@ -411,7 +516,7 @@ void processGetBlockMessageUsingBlockInStore() {
BlockSyncService blockSyncService = new BlockSyncService(config, store, blockchain, nodeInformation, syncConfiguration, DummyBlockValidator.VALID_RESULT_INSTANCE);
final NodeBlockProcessor bp = new NodeBlockProcessor(store, blockchain, nodeInformation, blockSyncService, syncConfiguration);
- final NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null,
+ final NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null,
null, mock(StatusResolver.class));
final SimplePeer sender = new SimplePeer();
@@ -446,7 +551,7 @@ void processGetBlockMessageUsingBlockInBlockchain() {
BlockSyncService blockSyncService = new BlockSyncService(config, store, blockchain, nodeInformation, syncConfiguration, DummyBlockValidator.VALID_RESULT_INSTANCE);
NodeBlockProcessor bp = new NodeBlockProcessor(store, blockchain, nodeInformation, blockSyncService, syncConfiguration);
- NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null,
+ NodeMessageHandler handler = new NodeMessageHandler(config, bp, null,null, null, null, null,
mock(StatusResolver.class));
SimplePeer sender = new SimplePeer();
@@ -478,7 +583,7 @@ void processGetBlockMessageUsingEmptyStore() {
BlockSyncService blockSyncService = new BlockSyncService(config, store, blockchain, nodeInformation, syncConfiguration, DummyBlockValidator.VALID_RESULT_INSTANCE);
final NodeBlockProcessor bp = new NodeBlockProcessor(store, blockchain, nodeInformation, blockSyncService, syncConfiguration);
- final NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null, mock(StatusResolver.class));
+ final NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null, null, mock(StatusResolver.class));
final SimplePeer sender = new SimplePeer();
@@ -502,7 +607,7 @@ void processBlockHeaderRequestMessageUsingBlockInStore() {
BlockSyncService blockSyncService = new BlockSyncService(config, store, blockchain, nodeInformation, syncConfiguration, DummyBlockValidator.VALID_RESULT_INSTANCE);
final NodeBlockProcessor bp = new NodeBlockProcessor(store, blockchain, nodeInformation, blockSyncService, syncConfiguration);
- final NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null,
+ final NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null, null,
mock(StatusResolver.class));
final SimplePeer sender = new SimplePeer();
@@ -537,7 +642,7 @@ void processBlockHeaderRequestMessageUsingBlockInBlockchain() {
BlockSyncService blockSyncService = new BlockSyncService(config, store, blockchain, nodeInformation, syncConfiguration, DummyBlockValidator.VALID_RESULT_INSTANCE);
NodeBlockProcessor bp = new NodeBlockProcessor(store, blockchain, nodeInformation, blockSyncService, syncConfiguration);
- NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null,
+ NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null, null,
mock(StatusResolver.class));
SimplePeer sender = new SimplePeer();
@@ -572,7 +677,7 @@ void processNewBlockHashesMessage() {
SyncConfiguration syncConfiguration = SyncConfiguration.IMMEDIATE_FOR_TESTING;
BlockSyncService blockSyncService = new BlockSyncService(config, store, blockchain, nodeInformation, syncConfiguration, DummyBlockValidator.VALID_RESULT_INSTANCE);
final NodeBlockProcessor bp = new NodeBlockProcessor(store, blockchain, nodeInformation, blockSyncService, syncConfiguration);
- final NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null,
+ final NodeMessageHandler handler = new NodeMessageHandler(config, bp, null, null, null, null, null,
mock(StatusResolver.class));
class TestCase {
@@ -679,7 +784,7 @@ void processNewBlockHashesMessageDoesNothingBecauseNodeIsSyncing() {
BlockProcessor blockProcessor = mock(BlockProcessor.class);
Mockito.when(blockProcessor.hasBetterBlockToSync()).thenReturn(true);
- final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, null, null, null,
+ final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, null, null, null, null,
mock(StatusResolver.class));
Message message = mock(Message.class);
@@ -699,7 +804,7 @@ void processTransactionsMessage() {
BlockProcessor blockProcessor = mock(BlockProcessor.class);
Mockito.when(blockProcessor.hasBetterBlockToSync()).thenReturn(false);
- final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, null, transactionGateway, scoring,
+ final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, null, null, transactionGateway, scoring,
mock(StatusResolver.class));
final SimplePeer sender = new SimplePeer(new NodeID(new byte[] {1}));
@@ -741,7 +846,7 @@ void processRejectedTransactionsMessage() {
BlockProcessor blockProcessor = mock(BlockProcessor.class);
Mockito.when(blockProcessor.hasBetterBlockToSync()).thenReturn(false);
- final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, channelManager, transactionGateway, scoring,
+ final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, null, channelManager, transactionGateway, scoring,
mock(StatusResolver.class));
final SimplePeer sender = new SimplePeer();
@@ -772,7 +877,7 @@ void processTooMuchGasTransactionMessage() {
BlockProcessor blockProcessor = mock(BlockProcessor.class);
Mockito.when(blockProcessor.hasBetterBlockToSync()).thenReturn(false);
- final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, channelManager, transactionGateway, scoring,
+ final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, null, channelManager, transactionGateway, scoring,
mock(StatusResolver.class));
final SimplePeer sender = new SimplePeer();
@@ -808,7 +913,7 @@ void processTransactionsMessageUsingTransactionPool() {
BlockProcessor blockProcessor = mock(BlockProcessor.class);
Mockito.when(blockProcessor.hasBetterBlockToSync()).thenReturn(false);
- final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, null, transactionGateway, RskMockFactory.getPeerScoringManager(),
+ final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, null, null, transactionGateway, RskMockFactory.getPeerScoringManager(),
mock(StatusResolver.class));
final SimplePeer sender = new SimplePeer(new NodeID(new byte[] {1}));
@@ -829,7 +934,7 @@ void processTransactionsMessageUsingTransactionPool() {
@Test
void processBlockByHashRequestMessageUsingProcessor() {
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, null,
mock(StatusResolver.class));
Block block = new BlockChainBuilder().ofSize(1, true).getBestBlock();
Message message = new BlockRequestMessage(100, block.getHash().getBytes());
@@ -844,7 +949,7 @@ void processBlockByHashRequestMessageUsingProcessor() {
void processBlockHeadersRequestMessageUsingProcessor() {
byte[] hash = TestUtils.generateBytes("sbp",32);
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, null,
mock(StatusResolver.class));
Message message = new BlockHeadersRequestMessage(100, hash, 50);
@@ -872,7 +977,7 @@ void fillMessageQueue_thenBlockNewMessages() {
BlockProcessor blockProcessor = mock(BlockProcessor.class);
Mockito.when(blockProcessor.hasBetterBlockToSync()).thenReturn(false);
- final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, null, transactionGateway, RskMockFactory.getPeerScoringManager(),
+ final NodeMessageHandler handler = new NodeMessageHandler(config, blockProcessor, null, null, null, transactionGateway, RskMockFactory.getPeerScoringManager(),
mock(StatusResolver.class));
final SimplePeer sender = new SimplePeer(new NodeID(new byte[] {1}));
@@ -977,7 +1082,7 @@ void whenPostMsgFromDiffSenders_shouldNotCountRepeatedMsgs() {
final SimplePeer sender2 = new SimplePeer(new NodeID(new byte[] {2}));
PeerScoringManager scoring = createPeerScoringManager();
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, scoring,
mock(StatusResolver.class));
Block block = new BlockChainBuilder().ofSize(1, true).getBestBlock();
Message message = new BlockMessage(block);
@@ -998,7 +1103,7 @@ void whenPostMsgFromSameSenders_shouldCountRepeatedMsgs() {
final SimplePeer sender2 = new SimplePeer(new NodeID(new byte[] {2}));
PeerScoringManager scoring = createPeerScoringManager();
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, scoring,
mock(StatusResolver.class));
Block block = new BlockChainBuilder().ofSize(1, true).getBestBlock();
Message message = new BlockMessage(block);
@@ -1020,7 +1125,7 @@ void whenPostMsg_shouldClearRcvMsgsCache() {
final SimplePeer sender2 = new SimplePeer(new NodeID(new byte[] {2}));
PeerScoringManager scoring = createPeerScoringManager();
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, scoring,
mock(StatusResolver.class));
Block block = new BlockChainBuilder().ofSize(1, true).getBestBlock();
Message message = new BlockMessage(block);
@@ -1042,7 +1147,7 @@ void whenAllowByMessageUniqueness_shouldReturnTrueForUniqueMsgs() {
final SimplePeer sender2 = new SimplePeer(new NodeID(new byte[] {2}));
PeerScoringManager scoring = mock(PeerScoringManager.class);
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, scoring,
mock(StatusResolver.class), 1);
Block block = new BlockChainBuilder().ofSize(1, true).getBestBlock();
@@ -1062,7 +1167,7 @@ void whenAllowByMessageUniqueness_shouldReturnTrueAfterCachedCleared() {
final SimplePeer sender2 = new SimplePeer(new NodeID(new byte[] {2}));
PeerScoringManager scoring = mock(PeerScoringManager.class);
SimpleBlockProcessor sbp = new SimpleBlockProcessor();
- NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, scoring,
+ NodeMessageHandler processor = new NodeMessageHandler(config, sbp, null, null, null, null, scoring,
mock(StatusResolver.class), 0);
Block block = new BlockChainBuilder().ofSize(1, true).getBestBlock();
diff --git a/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerUtil.java b/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerUtil.java
index dfbfe831b57..6dc630b72ff 100644
--- a/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerUtil.java
+++ b/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerUtil.java
@@ -41,7 +41,7 @@ DIFFICULTY_CALCULATOR, new PeersInformation(RskMockFactory.getChannelManager(),
mock(EthereumListener.class));
NodeBlockProcessor processor = new NodeBlockProcessor(store, blockchain, nodeInformation, blockSyncService, syncConfiguration);
- return new NodeMessageHandler(config, processor, syncProcessor, new SimpleChannelManager(), null, RskMockFactory.getPeerScoringManager(), mock(StatusResolver.class));
+ return new NodeMessageHandler(config, processor, syncProcessor, null, new SimpleChannelManager(), null, RskMockFactory.getPeerScoringManager(), mock(StatusResolver.class));
}
public static NodeMessageHandler createHandlerWithSyncProcessor(SyncConfiguration syncConfiguration, ChannelManager channelManager) {
@@ -73,6 +73,6 @@ blockValidationRule, new SyncBlockValidatorRule(new BlockUnclesHashValidationRul
mock(EthereumListener.class)
);
- return new NodeMessageHandler(config, processor, syncProcessor, channelManager, null, null, mock(StatusResolver.class));
+ return new NodeMessageHandler(config, processor, syncProcessor, null, channelManager, null, null, mock(StatusResolver.class));
}
}
diff --git a/rskj-core/src/test/java/co/rsk/net/OneAsyncNodeTest.java b/rskj-core/src/test/java/co/rsk/net/OneAsyncNodeTest.java
index 79e925118fc..1fbeeb87ef2 100644
--- a/rskj-core/src/test/java/co/rsk/net/OneAsyncNodeTest.java
+++ b/rskj-core/src/test/java/co/rsk/net/OneAsyncNodeTest.java
@@ -68,7 +68,7 @@ blockchain, mock(org.ethereum.db.BlockStore.class), mock(ConsensusValidationMain
mock(Genesis.class),
mock(EthereumListener.class)
);
- NodeMessageHandler handler = new NodeMessageHandler(config, processor, syncProcessor, channelManager, null, RskMockFactory.getPeerScoringManager(), mock(StatusResolver.class));
+ NodeMessageHandler handler = new NodeMessageHandler(config, processor, syncProcessor, null, channelManager, null, RskMockFactory.getPeerScoringManager(), mock(StatusResolver.class));
return new SimpleAsyncNode(handler, blockchain, syncProcessor, channelManager);
}
diff --git a/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java b/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java
new file mode 100644
index 00000000000..0a6b126bb2d
--- /dev/null
+++ b/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java
@@ -0,0 +1,389 @@
+/*
+ * This file is part of RskJ
+ * Copyright (C) 2024 RSK Labs Ltd.
+ * (derived from ethereumJ library, Copyright (c) 2016 )
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ */
+package co.rsk.net;
+
+import co.rsk.core.BlockDifficulty;
+import co.rsk.net.messages.*;
+import co.rsk.net.sync.SnapSyncState;
+import co.rsk.net.sync.SnapshotPeersInformation;
+import co.rsk.net.sync.SyncMessageHandler;
+import co.rsk.test.builders.BlockChainBuilder;
+import co.rsk.trie.TrieStore;
+import org.ethereum.core.Block;
+import org.ethereum.core.Blockchain;
+import org.ethereum.core.TransactionPool;
+import org.ethereum.db.BlockStore;
+import org.ethereum.util.RLP;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.*;
+
+public class SnapshotProcessorTest {
+ public static final int TEST_CHUNK_SIZE = 200;
+ private static final long THREAD_JOIN_TIMEOUT = 10_000; // 10 secs
+
+ private Blockchain blockchain;
+ private TransactionPool transactionPool;
+ private BlockStore blockStore;
+ private TrieStore trieStore;
+ private Peer peer;
+ private final SnapshotPeersInformation peersInformation = mock(SnapshotPeersInformation.class);
+ private final SnapSyncState snapSyncState = mock(SnapSyncState.class);
+ private final SyncMessageHandler.Listener listener = mock(SyncMessageHandler.Listener.class);
+ private SnapshotProcessor underTest;
+
+ @BeforeEach
+ void setUp() throws UnknownHostException {
+ peer = mockedPeer();
+ when(peersInformation.getBestSnapPeerCandidates()).thenReturn(Collections.singletonList(peer));
+ }
+
+ @AfterEach
+ void tearDown() {
+ if (underTest != null) {
+ underTest.stop();
+ }
+ }
+
+ @Test
+ void givenStartSyncingIsCalled_thenSnapStatusStartToBeRequestedFromPeer() {
+ //given
+ initializeBlockchainWithAmountOfBlocks(10);
+ underTest = new SnapshotProcessor(
+ blockchain,
+ trieStore,
+ peersInformation,
+ blockStore,
+ transactionPool,
+ TEST_CHUNK_SIZE,
+ false);
+ //when
+ underTest.startSyncing();
+ //then
+ verify(peer).sendMessage(any(SnapStatusRequestMessage.class));
+ }
+
+ @Test
+ void givenSnapStatusResponseCalled_thenSnapChunkRequestsAreMade() {
+ //given
+ List blocks = new ArrayList<>();
+ List