Coverage Summary for Class: DownloadingBodiesSyncState (co.rsk.net.sync)

Class Method, % Line, %
DownloadingBodiesSyncState 0% (0/34) 0% (0/160)
DownloadingBodiesSyncState$PendingBodyResponse 0% (0/2) 0% (0/4)
Total 0% (0/36) 0% (0/164)


1 package co.rsk.net.sync; 2  3 import co.rsk.crypto.Keccak256; 4 import co.rsk.net.BlockSyncService; 5 import co.rsk.net.NodeID; 6 import co.rsk.net.Peer; 7 import co.rsk.net.messages.BodyResponseMessage; 8 import co.rsk.scoring.EventType; 9 import co.rsk.validators.SyncBlockValidatorRule; 10 import com.google.common.annotations.VisibleForTesting; 11 import org.ethereum.core.*; 12 import org.ethereum.util.ByteUtil; 13 import org.slf4j.Logger; 14 import org.slf4j.LoggerFactory; 15  16 import java.time.Duration; 17 import java.util.*; 18 import java.util.stream.Collectors; 19  20 public class DownloadingBodiesSyncState extends BaseSyncState { 21  22  private static final Logger logger = LoggerFactory.getLogger("syncprocessor"); 23  24  private final PeersInformation peersInformation; 25  private final Blockchain blockchain; 26  private final BlockFactory blockFactory; 27  28  // responses on wait 29  private final Map<Long, PendingBodyResponse> pendingBodyResponses; 30  31  // messages on wait from a peer 32  private final Map<Peer, Long> messagesByPeers; 33  // chunks currently being downloaded 34  private final Map<Peer, Integer> chunksBeingDownloaded; 35  // segments currently being downloaded (many nodes can be downloading same segment) 36  private final Map<Peer, Integer> segmentsBeingDownloaded; 37  38  // headers waiting to be completed by bodies divided by chunks 39  private final List<Deque<BlockHeader>> pendingHeaders; 40  41  // a skeleton from each suitable peer 42  private final Map<Peer, List<BlockIdentifier>> skeletons; 43  44  // segment a peer belongs to 45  private final Map<Peer, Integer> segmentByNode; 46  47  // time elapse registered for each active peer 48  private final Map<Peer, Duration> timeElapsedByPeer; 49  50  // chunks divided by segments 51  private final List<Deque<Integer>> chunksBySegment; 52  53  // peers that can be used to download blocks 54  private final List<Peer> suitablePeers; 55  // maximum time waiting for a peer to answer 56  private final Duration limit; 57  private final SyncBlockValidatorRule blockValidationRule; 58  private final BlockSyncService blockSyncService; 59  60  public DownloadingBodiesSyncState(SyncConfiguration syncConfiguration, 61  SyncEventsHandler syncEventsHandler, 62  PeersInformation peersInformation, 63  Blockchain blockchain, 64  BlockFactory blockFactory, 65  BlockSyncService blockSyncService, 66  SyncBlockValidatorRule blockValidationRule, 67  List<Deque<BlockHeader>> pendingHeaders, 68  Map<Peer, List<BlockIdentifier>> skeletons) { 69  70  super(syncEventsHandler, syncConfiguration); 71  this.peersInformation = peersInformation; 72  this.blockchain = blockchain; 73  this.blockFactory = blockFactory; 74  this.limit = syncConfiguration.getTimeoutWaitingRequest(); 75  this.blockSyncService = blockSyncService; 76  this.blockValidationRule = blockValidationRule; 77  this.pendingBodyResponses = new HashMap<>(); 78  this.pendingHeaders = pendingHeaders; 79  this.skeletons = skeletons; 80  this.segmentByNode = new HashMap<>(); 81  this.chunksBySegment = new ArrayList<>(); 82  this.chunksBeingDownloaded = new HashMap<>(); 83  this.segmentsBeingDownloaded = new HashMap<>(); 84  this.timeElapsedByPeer = new HashMap<>(); 85  this.messagesByPeers = new HashMap<>(); 86  87  initializeSegments(); 88  this.suitablePeers = new ArrayList<>(segmentByNode.keySet()); 89  } 90  91  @Override 92  public void newBody(BodyResponseMessage message, Peer peer) { 93  NodeID peerId = peer.getPeerNodeID(); 94  long requestId = message.getId(); 95  if (!isExpectedBody(requestId, peerId)) { 96  handleUnexpectedBody(peer); 97  return; 98  } 99  100  // we already checked that this message was expected 101  BlockHeader header = pendingBodyResponses.remove(requestId).header; 102  Block block; 103  try { 104  block = blockFactory.newBlock(header, message.getTransactions(), message.getUncles()); 105  block.seal(); 106  } catch (IllegalArgumentException ex) { 107  handleInvalidMessage(peer, header); 108  return; 109  } 110  111  if (!blockValidationRule.isValid(block)) { 112  handleInvalidMessage(peer, header); 113  return; 114  } 115  116  // handle block 117  // this is a controled place where we ask for blocks, we never should look for missing hashes 118  if (blockSyncService.processBlock(block, peer, true).isInvalidBlock()){ 119  handleInvalidBlock(peer, header); 120  return; 121  } 122  // updates peer downloading information 123  tryRequestNextBody(peer); 124  // check if this was the last block to download 125  verifyDownloadIsFinished(); 126  } 127  128  private void verifyDownloadIsFinished() { 129  // all headers have been requested and there is not any chunk still in process 130  if (chunksBeingDownloaded.isEmpty() && 131  pendingHeaders.stream().allMatch(Collection::isEmpty)) { 132  // Finished syncing 133  logger.info("Completed syncing phase"); 134  syncEventsHandler.stopSyncing(); 135  136  } 137  } 138  139  private void tryRequestNextBody(Peer peer) { 140  updateHeadersAndChunks(peer, chunksBeingDownloaded.get(peer)) 141  .ifPresent(blockHeader -> tryRequestBody(peer, blockHeader)); 142  } 143  144  private void handleInvalidBlock(Peer peer, BlockHeader header) { 145  peersInformation.reportEventWithLog( 146  "Invalid block received from node {} {} {}", 147  peer.getPeerNodeID(), EventType.INVALID_BLOCK, 148  peer.getPeerNodeID(), header.getNumber(), header.getPrintableHash()); 149  150  clearPeerInfo(peer); 151  if (suitablePeers.isEmpty()){ 152  syncEventsHandler.stopSyncing(); 153  return; 154  } 155  messagesByPeers.remove(peer); 156  resetChunkAndHeader(peer, header); 157  startDownloading(getInactivePeers()); 158  } 159  160  private void handleInvalidMessage(Peer peer, BlockHeader header) { 161  peersInformation.reportEventWithLog( 162  "Invalid body received from node {} {} {}", 163  peer.getPeerNodeID(), EventType.INVALID_MESSAGE, 164  peer.getPeerNodeID(), header.getNumber(), header.getPrintableHash()); 165  166  clearPeerInfo(peer); 167  if (suitablePeers.isEmpty()){ 168  syncEventsHandler.stopSyncing(); 169  return; 170  } 171  messagesByPeers.remove(peer); 172  resetChunkAndHeader(peer, header); 173  startDownloading(getInactivePeers()); 174  } 175  176  private void handleUnexpectedBody(Peer peer) { 177  peersInformation.reportEventWithLog( 178  "Unexpected body received from node {}", 179  peer.getPeerNodeID(), EventType.UNEXPECTED_MESSAGE, peer.getPeerNodeID()); 180  181  clearPeerInfo(peer); 182  if (suitablePeers.isEmpty()) { 183  syncEventsHandler.stopSyncing(); 184  return; 185  } 186  // if this peer has another different message pending then its restored to the stack 187  Long messageId = messagesByPeers.remove(peer); 188  if (messageId != null) { 189  resetChunkAndHeader(peer, pendingBodyResponses.remove(messageId).header); 190  } 191  startDownloading(getInactivePeers()); 192  } 193  194  private void resetChunkAndHeader(Peer peer, BlockHeader header) { 195  int chunkNumber = chunksBeingDownloaded.remove(peer); 196  pendingHeaders.get(chunkNumber).addLast(header); 197  int segmentNumber = segmentsBeingDownloaded.remove(peer); 198  chunksBySegment.get(segmentNumber).push(chunkNumber); 199  } 200  201  private void clearPeerInfo(Peer peer) { 202  suitablePeers.remove(peer); 203  timeElapsedByPeer.remove(peer); 204  } 205  206  private Optional<BlockHeader> updateHeadersAndChunks(Peer peer, Integer currentChunk) { 207  Deque<BlockHeader> headers = pendingHeaders.get(currentChunk); 208  BlockHeader header = headers.poll(); 209  while (header != null) { 210  // we double check if the header was not downloaded or obtained by another way 211  if (!isKnownBlock(header.getHash())) { 212  return Optional.of(header); 213  } 214  header = headers.poll(); 215  } 216  217  Optional<BlockHeader> blockHeader = tryFindBlockHeader(peer); 218  if (!blockHeader.isPresent()){ 219  chunksBeingDownloaded.remove(peer); 220  segmentsBeingDownloaded.remove(peer); 221  messagesByPeers.remove(peer); 222  } 223  224  return blockHeader; 225  } 226  227  private boolean isKnownBlock(Keccak256 hash) { 228  return blockchain.getBlockByHash(hash.getBytes()) != null; 229  } 230  231  private Optional<BlockHeader> tryFindBlockHeader(Peer peer) { 232  // we start from the last chunk that can be downloaded 233  for (int segmentNumber = segmentByNode.get(peer); segmentNumber >= 0; segmentNumber--){ 234  Deque<Integer> chunks = chunksBySegment.get(segmentNumber); 235  // if the segment stack is empty then continue to next segment 236  if (!chunks.isEmpty()) { 237  int chunkNumber = chunks.pollLast(); 238  Deque<BlockHeader> headers = pendingHeaders.get(chunkNumber); 239  BlockHeader header = headers.poll(); 240  while (header != null) { 241  // we double check if the header was not downloaded or obtained by another way 242  if (!isBlockKnown(header.getHash())) { 243  chunksBeingDownloaded.put(peer, chunkNumber); 244  segmentsBeingDownloaded.put(peer, segmentNumber); 245  return Optional.of(header); 246  } 247  header = headers.poll(); 248  } 249  } 250  } 251  return Optional.empty(); 252  } 253  254  private boolean isBlockKnown(Keccak256 hash) { 255  return blockchain.getBlockByHash(hash.getBytes()) != null; 256  } 257  258  @Override 259  public void onEnter() { 260  startDownloading(suitablePeers); 261  } 262  263  private void startDownloading(List<Peer> peers) { 264  peers.forEach(p -> tryFindBlockHeader(p).ifPresent(header -> tryRequestBody(p, header))); 265  } 266  267  @Override 268  public void tick(Duration duration) { 269  // first we update all the nodes that are expected to be working 270  List<Peer> updatedNodes = timeElapsedByPeer.keySet().stream() 271  .filter(chunksBeingDownloaded::containsKey) 272  .collect(Collectors.toList()); 273  274  updatedNodes.forEach(k -> timeElapsedByPeer.put(k, timeElapsedByPeer.get(k).plus(duration))); 275  276  // we get the nodes that got beyond timeout limit and remove them 277  updatedNodes.stream() 278  .filter(k -> timeElapsedByPeer.get(k).compareTo(limit) >= 0) 279  .forEach(this::handleTimeoutMessage); 280  281  if (suitablePeers.isEmpty()){ 282  syncEventsHandler.stopSyncing(); 283  return; 284  } 285  286  startDownloading(getInactivePeers()); 287  288  if (chunksBeingDownloaded.isEmpty()){ 289  syncEventsHandler.stopSyncing(); 290  } 291  } 292  293  private void handleTimeoutMessage(Peer peer) { 294  peersInformation.reportEventWithLog("Timeout waiting body from node {}", 295  peer.getPeerNodeID(), EventType.TIMEOUT_MESSAGE, peer); 296  Long messageId = messagesByPeers.remove(peer); 297  BlockHeader header = pendingBodyResponses.remove(messageId).header; 298  clearPeerInfo(peer); 299  resetChunkAndHeader(peer, header); 300  } 301  302  private List<Peer> getInactivePeers() { 303  return suitablePeers.stream() 304  .filter(p -> !chunksBeingDownloaded.containsKey(p)) 305  .collect(Collectors.toList()); 306  } 307  308  /** 309  * This method finds with the skeletons from each node, the segments we can divide chunks. 310  * Each chunk belongs to a single segment, and each node associated to a segment can answer 311  * for each block inside the chunks belonging to a segment. 312  * Also each node on a superior segment can answer for every block on lower segments. 313  * The idea is to find the "min common chunks" between nodes to find when a new segment starts 314  */ 315  private void initializeSegments() { 316  Deque<Integer> segmentChunks = new ArrayDeque<>(); 317  Integer segmentNumber = 0; 318  Integer chunkNumber = 0; 319  List<Peer> nodes = getAvailableNodesIDSFor(chunkNumber); 320  List<Peer> prevNodes = nodes; 321  segmentChunks.push(chunkNumber); 322  chunkNumber++; 323  324  for (; chunkNumber < pendingHeaders.size(); chunkNumber++){ 325  nodes = getAvailableNodesIDSFor(chunkNumber); 326  if (prevNodes.size() != nodes.size()){ 327  final List<Peer> filteringNodes = nodes; 328  List<Peer> insertedNodes = prevNodes.stream() 329  .filter(k -> !filteringNodes.contains(k)).collect(Collectors.toList()); 330  insertSegment(segmentChunks, insertedNodes, segmentNumber); 331  segmentNumber++; 332  prevNodes = nodes; 333  segmentChunks = new ArrayDeque<>(); 334  } 335  segmentChunks.push(chunkNumber); 336  } 337  338  // last segment should be added always 339  insertSegment(segmentChunks, nodes, segmentNumber); 340  } 341  342  private List<Peer> getAvailableNodesIDSFor(Integer chunkNumber) { 343  return skeletons.entrySet().stream() 344  .filter(e -> e.getValue().size() > chunkNumber + 1) 345  .filter(e -> ByteUtil.fastEquals( 346  // the hash of the start of next chunk 347  e.getValue().get(chunkNumber + 1).getHash(), 348  // the first header of chunk 349  pendingHeaders.get(chunkNumber).getLast().getHash().getBytes())) 350  .map(Map.Entry::getKey) 351  .collect(Collectors.toList()); 352  } 353  354  private void insertSegment(Deque<Integer> segmentChunks, List<Peer> nodes, Integer segmentNumber) { 355  chunksBySegment.add(segmentChunks); 356  nodes.forEach(peer -> segmentByNode.put(peer, segmentNumber)); 357  } 358  359  private void tryRequestBody(Peer peer, BlockHeader header){ 360  long messageId = syncEventsHandler.sendBodyRequest(peer, header); 361  pendingBodyResponses.put(messageId, new PendingBodyResponse(peer.getPeerNodeID(), header)); 362  timeElapsedByPeer.put(peer, Duration.ZERO); 363  messagesByPeers.put(peer, messageId); 364  } 365  366  private boolean isExpectedBody(long requestId, NodeID peerId) { 367  PendingBodyResponse expected = pendingBodyResponses.get(requestId); 368  return expected != null && expected.nodeID.equals(peerId); 369  } 370  371  @Override 372  public boolean isSyncing() { 373  return true; 374  } 375  376  @VisibleForTesting 377  public void expectBodyResponseFor(long requestId, NodeID nodeID, BlockHeader header) { 378  pendingBodyResponses.put(requestId, new PendingBodyResponse(nodeID, header)); 379  } 380  381  private static class PendingBodyResponse { 382  private NodeID nodeID; 383  private BlockHeader header; 384  385  PendingBodyResponse(NodeID nodeID, BlockHeader header) { 386  this.nodeID = nodeID; 387  this.header = header; 388  } 389  } 390 }