Coverage Summary for Class: IndexedBlockStore (org.ethereum.db)

Class Method, % Line, %
IndexedBlockStore 41.9% (13/31) 48.7% (116/238)
IndexedBlockStore$1 33.3% (1/3) 5% (1/20)
IndexedBlockStore$BlockInfo 100% (7/7) 100% (10/10)
Total 51.2% (21/41) 47.4% (127/268)


1 /* 2  * This file is part of RskJ 3  * Copyright (C) 2017 RSK Labs Ltd. 4  * (derived from ethereumJ library, Copyright (c) 2016 <ether.camp>) 5  * 6  * This program is free software: you can redistribute it and/or modify 7  * it under the terms of the GNU Lesser General Public License as published by 8  * the Free Software Foundation, either version 3 of the License, or 9  * (at your option) any later version. 10  * 11  * This program is distributed in the hope that it will be useful, 12  * but WITHOUT ANY WARRANTY; without even the implied warranty of 13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14  * GNU Lesser General Public License for more details. 15  * 16  * You should have received a copy of the GNU Lesser General Public License 17  * along with this program. If not, see <http://www.gnu.org/licenses/>. 18  */ 19  20 package org.ethereum.db; 21  22 import co.rsk.core.BlockDifficulty; 23 import co.rsk.crypto.Keccak256; 24 import co.rsk.db.BlocksIndex; 25 import co.rsk.metrics.profilers.Metric; 26 import co.rsk.metrics.profilers.Profiler; 27 import co.rsk.metrics.profilers.ProfilerFactory; 28 import co.rsk.net.BlockCache; 29 import co.rsk.remasc.Sibling; 30 import co.rsk.util.MaxSizeHashMap; 31 import com.google.common.annotations.VisibleForTesting; 32 import org.ethereum.core.Block; 33 import org.ethereum.core.BlockFactory; 34 import org.ethereum.core.BlockHeader; 35 import org.ethereum.core.Bloom; 36 import org.ethereum.datasource.KeyValueDataSource; 37 import org.mapdb.DataIO; 38 import org.mapdb.Serializer; 39 import org.slf4j.Logger; 40 import org.slf4j.LoggerFactory; 41  42 import java.io.*; 43 import java.math.BigInteger; 44 import java.util.*; 45 import java.util.stream.Collectors; 46  47 import static co.rsk.core.BlockDifficulty.ZERO; 48  49 public class IndexedBlockStore implements BlockStore { 50  51  private static final Logger logger = LoggerFactory.getLogger("general"); 52  private static final Profiler profiler = ProfilerFactory.getInstance(); 53  54  private final BlockCache blockCache; 55  private final MaxSizeHashMap<Keccak256, Map<Long, List<Sibling>>> remascCache; 56  57  private final BlocksIndex index; 58  private final KeyValueDataSource blocks; 59  private final BlockFactory blockFactory; 60  61  public IndexedBlockStore( 62  BlockFactory blockFactory, 63  KeyValueDataSource blocks, 64  BlocksIndex index) { 65  this.index = index; 66  this.blocks = blocks; 67  this.blockFactory = blockFactory; 68  //TODO(lsebrie): move these maps creation outside blockstore, 69  // remascCache should be an external component and not be inside blockstore 70  this.blockCache = new BlockCache(5000); 71  this.remascCache = new MaxSizeHashMap<>(50000, true); 72  } 73  74  @Override 75  public synchronized void removeBlock(Block block) { 76  this.blockCache.removeBlock(block); 77  this.remascCache.remove(block.getHash()); 78  this.blocks.delete(block.getHash().getBytes()); 79  80  List<BlockInfo> binfos = this.index.getBlocksByNumber(block.getNumber()); 81  82  if (binfos == null) { 83  return; 84  } 85  86  List<BlockInfo> toremove = new ArrayList<>(); 87  88  for (BlockInfo binfo : binfos) { 89  if (binfo.getHash().equals(block.getHash())) { 90  toremove.add(binfo); 91  } 92  } 93  94  binfos.removeAll(toremove); 95  } 96  97  @Override 98  public synchronized Block getBestBlock() { 99  if (index.isEmpty()) { 100  return null; 101  } 102  103  long maxLevel = index.getMaxNumber(); 104  Block bestBlock = getChainBlockByNumber(maxLevel); 105  if (bestBlock != null) { 106  return bestBlock; 107  } 108  109  // That scenario can happen 110  // if there is a fork branch that is 111  // higher than main branch but has 112  // less TD than the main branch TD 113  while (bestBlock == null && maxLevel >= 0) { 114  --maxLevel; 115  bestBlock = getChainBlockByNumber(maxLevel); 116  } 117  118  return bestBlock; 119  } 120  121  @Override 122  public byte[] getBlockHashByNumber(long blockNumber, byte[] branchBlockHash) { 123  Block branchBlock = getBlockByHash(branchBlockHash); 124  if (branchBlock.getNumber() < blockNumber) { 125  throw new IllegalArgumentException(String.format("Requested block number > branch hash number: %d < %d", 126  blockNumber, branchBlock.getNumber())); 127  } 128  while(branchBlock.getNumber() > blockNumber) { 129  branchBlock = getBlockByHash(branchBlock.getParentHash().getBytes()); 130  } 131  return branchBlock.getHash().getBytes(); 132  } 133  134  @Override 135  // This method is an optimized way to traverse a branch in search for a block at a given depth. Starting at a given 136  // block (by hash) it tries to find the first block that is part of the best chain, when it finds one we now that 137  // we can jump to the block that is at the remaining depth. If not block is found then it continues traversing the 138  // branch from parent to parent. The search is limited by the maximum depth received as parameter. 139  // This method either needs to traverse the parent chain or if a block in the parent chain is part of the best chain 140  // then it can skip the traversal by going directly to the block at the remaining depth. 141  public Block getBlockAtDepthStartingAt(long depth, byte[] hash) { 142  Block start = this.getBlockByHash(hash); 143  144  if (start != null && depth == 0) { 145  return start; 146  } 147  148  if (start == null || start.getNumber() <= depth) { 149  return null; 150  } 151  152  Block block = start; 153  154  for (long i = 0; i < depth; i++) { 155  if (isBlockInMainChain(block.getNumber(), block.getHash())) { 156  return getChainBlockByNumber(start.getNumber() - depth); 157  } 158  159  block = this.getBlockByHash(block.getParentHash().getBytes()); 160  } 161  162  return block; 163  } 164  165  public boolean isBlockInMainChain(long blockNumber, Keccak256 blockHash){ 166  List<BlockInfo> blockInfos = index.getBlocksByNumber(blockNumber); 167  if (blockInfos == null) { 168  return false; 169  } 170  171  for (BlockInfo blockInfo : blockInfos) { 172  if (blockInfo.isMainChain() && blockHash.equals(blockInfo.getHash())) { 173  return true; 174  } 175  } 176  177  return false; 178  } 179  180  @Override 181  public synchronized void flush() { 182  Metric metric = profiler.start(Profiler.PROFILING_TYPE.DB_WRITE); 183  index.flush(); 184  profiler.stop(metric); 185  } 186  187  public void close() { 188  this.index.close(); 189  } 190  191  @Override 192  public Bloom bloomByBlockNumber(long blockNumber) { 193  return new Bloom(getChainBlockByNumber(blockNumber).getLogBloom()); 194  } 195  196  @Override 197  public synchronized void saveBlock(Block block, BlockDifficulty cummDifficulty, boolean mainChain) { 198  List<BlockInfo> blockInfos = index.getBlocksByNumber(block.getNumber()); 199  200  BlockInfo blockInfo = null; 201  for (BlockInfo bi : blockInfos) { 202  if (bi.getHash().equals(block.getHash())) { 203  blockInfo = bi; 204  } else if (mainChain) { 205  bi.setMainChain(false); 206  } 207  } 208  if (blockInfo == null) { 209  blockInfo = new BlockInfo(); 210  blockInfos.add(blockInfo); 211  } 212  213  blockInfo.setCummDifficulty(cummDifficulty); 214  blockInfo.setHash(block.getHash().getBytes()); 215  blockInfo.setMainChain(mainChain); 216  217  if (blocks.get(block.getHash().getBytes()) == null) { 218  blocks.put(block.getHash().getBytes(), block.getEncoded()); 219  } 220  221  index.putBlocks(block.getNumber(), blockInfos); 222  blockCache.addBlock(block); 223  remascCache.put(block.getHash(), getSiblingsFromBlock(block)); 224  } 225  226  @Override 227  public synchronized List<BlockInformation> getBlocksInformationByNumber(long number) { 228  List<BlockInformation> result = new ArrayList<>(); 229  230  List<BlockInfo> blockInfos = index.getBlocksByNumber(number); 231  232  for (BlockInfo blockInfo : blockInfos) { 233  byte[] hash = blockInfo.getHash().copy().getBytes(); 234  BlockDifficulty totalDifficulty = blockInfo.getCummDifficulty(); 235  boolean isInBlockChain = blockInfo.isMainChain(); 236  237  result.add(new BlockInformation(hash, totalDifficulty, isInBlockChain)); 238  } 239  240  return result; 241  } 242  243  @Override 244  public boolean isEmpty() { 245  return index.isEmpty(); 246  } 247  248  @Override 249  public synchronized Block getChainBlockByNumber(long number){ 250  List<BlockInfo> blockInfos = index.getBlocksByNumber(number); 251  252  for (BlockInfo blockInfo : blockInfos) { 253  if (blockInfo.isMainChain()) { 254  255  byte[] hash = blockInfo.getHash().getBytes(); 256  return getBlockByHash(hash); 257  } 258  } 259  260  return null; 261  } 262  263  @Override 264  public synchronized Block getBlockByHash(byte[] hash) { 265  266  Block block = getBlock(hash); 267  if (block == null) { 268  return null; 269  } 270  271  blockCache.addBlock(block); 272  remascCache.put(block.getHash(), getSiblingsFromBlock(block)); 273  return block; 274  } 275  276  private synchronized Block getBlock(byte[] hash) { 277  Block block = this.blockCache.getBlockByHash(hash); 278  279  if (block != null) { 280  return block; 281  } 282  283  byte[] blockRlp = blocks.get(hash); 284  if (blockRlp == null) { 285  return null; 286  } 287  288  return blockFactory.decodeBlock(blockRlp); 289  } 290  291  @Override 292  public synchronized Map<Long, List<Sibling>> getSiblingsFromBlockByHash(Keccak256 hash) { 293  return this.remascCache.computeIfAbsent(hash, key -> getSiblingsFromBlock(getBlock(key.getBytes()))); 294  } 295  296  @Override 297  public synchronized boolean isBlockExist(byte[] hash) { 298  return getBlockByHash(hash) != null; 299  } 300  301  @Override 302  public synchronized BlockDifficulty getTotalDifficultyForHash(byte[] hash){ 303  Block block = this.getBlockByHash(hash); 304  if (block == null) { 305  return ZERO; 306  } 307  308  long level = block.getNumber(); 309  List<BlockInfo> blockInfos = index.getBlocksByNumber(level); 310  311  for (BlockInfo blockInfo : blockInfos) { 312  if (Arrays.equals(blockInfo.getHash().getBytes(), hash)) { 313  return blockInfo.getCummDifficulty(); 314  } 315  } 316  317  return ZERO; 318  } 319  320  @Override 321  public long getMaxNumber() { 322  return index.getMaxNumber(); 323  } 324  325  @Override 326  public long getMinNumber() { 327  return index.getMinNumber(); 328  } 329  330  @Override 331  public synchronized List<byte[]> getListHashesEndWith(byte[] hash, long number){ 332  333  List<Block> blocks = getListBlocksEndWith(hash, number); 334  List<byte[]> hashes = new ArrayList<>(blocks.size()); 335  336  for (Block b : blocks) { 337  hashes.add(b.getHash().getBytes()); 338  } 339  340  return hashes; 341  } 342  343  private synchronized List<Block> getListBlocksEndWith(byte[] hash, long qty) { 344  Block block = getBlockByHash(hash); 345  346  if (block == null) { 347  return new ArrayList<>(); 348  } 349  350  List<Block> blocks = new ArrayList<>((int) qty); 351  352  for (int i = 0; i < qty; ++i) { 353  354  blocks.add(block); 355  block = getBlockByHash(hash); 356  if (block == null) { 357  break; 358  } 359  } 360  361  return blocks; 362  } 363  364  @Override 365  public synchronized void reBranch(Block forkBlock){ 366  367  Block bestBlock = getBestBlock(); 368  long maxLevel = Math.max(bestBlock.getNumber(), forkBlock.getNumber()); 369  370  // 1. First ensure that you are on the save level 371  long currentLevel = maxLevel; 372  Block forkLine = forkBlock; 373  374  if (forkBlock.getNumber() > bestBlock.getNumber()) { 375  376  while(currentLevel > bestBlock.getNumber()) { 377  List<BlockInfo> blocks = index.getBlocksByNumber(currentLevel); 378  BlockInfo blockInfo = getBlockInfoForHash(blocks, forkLine.getHash().getBytes()); 379  if (blockInfo != null) { 380  blockInfo.setMainChain(true); 381  if (index.contains(currentLevel)) { 382  index.putBlocks(currentLevel, blocks); 383  } 384  } 385  forkLine = getBlockByHash(forkLine.getParentHash().getBytes()); 386  --currentLevel; 387  } 388  } 389  390  Block bestLine = bestBlock; 391  if (bestBlock.getNumber() > forkBlock.getNumber()){ 392  393  while(currentLevel > forkBlock.getNumber()) { 394  List<BlockInfo> blocks = index.getBlocksByNumber(currentLevel); 395  BlockInfo blockInfo = getBlockInfoForHash(blocks, bestLine.getHash().getBytes()); 396  if (blockInfo != null) { 397  blockInfo.setMainChain(false); 398  if (index.contains(currentLevel)) { 399  index.putBlocks(currentLevel, blocks); 400  } 401  } 402  bestLine = getBlockByHash(bestLine.getParentHash().getBytes()); 403  --currentLevel; 404  } 405  } 406  407  // 2. Loop back on each level until common block 408  while( !bestLine.isEqual(forkLine) ) { 409  410  List<BlockInfo> levelBlocks = index.getBlocksByNumber(currentLevel); 411  BlockInfo bestInfo = getBlockInfoForHash(levelBlocks, bestLine.getHash().getBytes()); 412  if (bestInfo != null) { 413  bestInfo.setMainChain(false); 414  if (index.contains(currentLevel)) { 415  index.putBlocks(currentLevel, levelBlocks); 416  } 417  } 418  419  BlockInfo forkInfo = getBlockInfoForHash(levelBlocks, forkLine.getHash().getBytes()); 420  if (forkInfo != null) { 421  forkInfo.setMainChain(true); 422  if (index.contains(currentLevel)) { 423  index.putBlocks(currentLevel, levelBlocks); 424  } 425  } 426  427  bestLine = getBlockByHash(bestLine.getParentHash().getBytes()); 428  forkLine = getBlockByHash(forkLine.getParentHash().getBytes()); 429  430  --currentLevel; 431  } 432  } 433  434  @VisibleForTesting 435  public synchronized List<byte[]> getListHashesStartWith(long number, long maxBlocks) { 436  437  List<byte[]> result = new ArrayList<>(); 438  439  int i; 440  for (i = 0; i < maxBlocks; ++i) { 441  List<BlockInfo> blockInfos = index.getBlocksByNumber(number); 442  if (blockInfos == null) { 443  break; 444  } 445  446  for (BlockInfo blockInfo : blockInfos) { 447  if (blockInfo.isMainChain()) { 448  result.add(blockInfo.getHash().getBytes()); 449  break; 450  } 451  } 452  453  ++number; 454  } 455  456  return result; 457  } 458  459  460  461  private static BlockInfo getBlockInfoForHash(List<BlockInfo> blocks, byte[] hash){ 462  if (blocks == null) { 463  return null; 464  } 465  466  for (BlockInfo blockInfo : blocks) { 467  if (Arrays.equals(hash, blockInfo.getHash().getBytes())) { 468  return blockInfo; 469  } 470  } 471  472  return null; 473  } 474  475  @Override 476  public synchronized List<Block> getChainBlocksByNumber(long number){ 477  List<Block> result = new ArrayList<>(); 478  479  List<BlockInfo> blockInfos = index.getBlocksByNumber(number); 480  481  if (blockInfos == null){ 482  return result; 483  } 484  485  for (BlockInfo blockInfo : blockInfos){ 486  487  byte[] hash = blockInfo.getHash().getBytes(); 488  Block block = getBlockByHash(hash); 489  490  // TODO(mc) investigate and fix this, probably a cache invalidation problem 491  if (block != null) { 492  result.add(block); 493  } 494  } 495  496  return result; 497  } 498  499  /** 500  * Deletes from disk storage all blocks with number strictly larger than blockNumber. 501  * Note that this doesn't clean the caches, making it unsuitable for using after initialization. 502  */ 503  @Override 504  public void rewind(long blockNumber) { 505  if (index.isEmpty()) { 506  return; 507  } 508  509  long maxNumber = getMaxNumber(); 510  for (long i = maxNumber; i > blockNumber; i--) { 511  List<BlockInfo> blockInfos = index.removeLast(); 512  513  for (BlockInfo blockInfo : blockInfos) { 514  this.blocks.delete(blockInfo.getHash().getBytes()); 515  } 516  } 517  518  flush(); 519  blocks.flush(); 520  } 521  522  /** 523  * When a block is processed on remasc the contract needs to calculate all siblings that 524  * that should be rewarded when fees on this block are paid 525  * @param block the block is looked for siblings 526  * @return 527  */ 528  private Map<Long, List<Sibling>> getSiblingsFromBlock(Block block) { 529  return block.getUncleList().stream() 530  .collect( 531  Collectors.groupingBy( 532  BlockHeader::getNumber, 533  Collectors.mapping( 534  header -> new Sibling(header, block.getCoinbase(), block.getNumber()), 535  Collectors.toList() 536  ) 537  ) 538  ); 539  } 540  541  542  public static class BlockInfo implements Serializable { 543  private static final long serialVersionUID = 5906746360128478753L; 544  545  private byte[] hash; 546  private BigInteger cummDifficulty; 547  private boolean mainChain; 548  549  public Keccak256 getHash() { 550  return new Keccak256(hash); 551  } 552  553  public void setHash(byte[] hash) { 554  this.hash = hash; 555  } 556  557  public BlockDifficulty getCummDifficulty() { 558  return new BlockDifficulty(cummDifficulty); 559  } 560  561  public void setCummDifficulty(BlockDifficulty cummDifficulty) { 562  this.cummDifficulty = cummDifficulty.asBigInteger(); 563  } 564  565  public boolean isMainChain() { 566  return mainChain; 567  } 568  569  public void setMainChain(boolean mainChain) { 570  this.mainChain = mainChain; 571  } 572  } 573  574  575  public static final Serializer<List<BlockInfo>> BLOCK_INFO_SERIALIZER = new Serializer<List<BlockInfo>>(){ 576  577  @Override 578  public void serialize(DataOutput out, List<BlockInfo> value) throws IOException { 579  ArrayList<BlockInfo> valueToSerialize = new ArrayList<>(value); 580  581  ByteArrayOutputStream bos = new ByteArrayOutputStream(); 582  ObjectOutputStream oos = new ObjectOutputStream(bos); 583  oos.writeObject(valueToSerialize); 584  585  byte[] data = bos.toByteArray(); 586  DataIO.packInt(out, data.length); 587  out.write(data); 588  } 589  590  @Override 591  public List<BlockInfo> deserialize(DataInput in, int available) throws IOException { 592  593  ArrayList<BlockInfo> value = null; 594  try { 595  int size = DataIO.unpackInt(in); 596  byte[] data = new byte[size]; 597  in.readFully(data); 598  599  ByteArrayInputStream bis = new ByteArrayInputStream(data, 0, data.length); 600  ObjectInputStream ois = new ObjectInputStream(bis); 601  value = (ArrayList<BlockInfo>)ois.readObject(); 602  } catch (ClassNotFoundException e) { 603  logger.error("Class not found", e); 604  } 605  606  return value; 607  } 608  }; 609 }