Coverage Summary for Class: IndexedBlockStore (org.ethereum.db)
Class |
Method, %
|
Line, %
|
IndexedBlockStore |
41.9%
(13/31)
|
48.7%
(116/238)
|
IndexedBlockStore$1 |
33.3%
(1/3)
|
5%
(1/20)
|
IndexedBlockStore$BlockInfo |
100%
(7/7)
|
100%
(10/10)
|
Total |
51.2%
(21/41)
|
47.4%
(127/268)
|
1 /*
2 * This file is part of RskJ
3 * Copyright (C) 2017 RSK Labs Ltd.
4 * (derived from ethereumJ library, Copyright (c) 2016 <ether.camp>)
5 *
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU Lesser General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20 package org.ethereum.db;
21
22 import co.rsk.core.BlockDifficulty;
23 import co.rsk.crypto.Keccak256;
24 import co.rsk.db.BlocksIndex;
25 import co.rsk.metrics.profilers.Metric;
26 import co.rsk.metrics.profilers.Profiler;
27 import co.rsk.metrics.profilers.ProfilerFactory;
28 import co.rsk.net.BlockCache;
29 import co.rsk.remasc.Sibling;
30 import co.rsk.util.MaxSizeHashMap;
31 import com.google.common.annotations.VisibleForTesting;
32 import org.ethereum.core.Block;
33 import org.ethereum.core.BlockFactory;
34 import org.ethereum.core.BlockHeader;
35 import org.ethereum.core.Bloom;
36 import org.ethereum.datasource.KeyValueDataSource;
37 import org.mapdb.DataIO;
38 import org.mapdb.Serializer;
39 import org.slf4j.Logger;
40 import org.slf4j.LoggerFactory;
41
42 import java.io.*;
43 import java.math.BigInteger;
44 import java.util.*;
45 import java.util.stream.Collectors;
46
47 import static co.rsk.core.BlockDifficulty.ZERO;
48
49 public class IndexedBlockStore implements BlockStore {
50
51 private static final Logger logger = LoggerFactory.getLogger("general");
52 private static final Profiler profiler = ProfilerFactory.getInstance();
53
54 private final BlockCache blockCache;
55 private final MaxSizeHashMap<Keccak256, Map<Long, List<Sibling>>> remascCache;
56
57 private final BlocksIndex index;
58 private final KeyValueDataSource blocks;
59 private final BlockFactory blockFactory;
60
61 public IndexedBlockStore(
62 BlockFactory blockFactory,
63 KeyValueDataSource blocks,
64 BlocksIndex index) {
65 this.index = index;
66 this.blocks = blocks;
67 this.blockFactory = blockFactory;
68 //TODO(lsebrie): move these maps creation outside blockstore,
69 // remascCache should be an external component and not be inside blockstore
70 this.blockCache = new BlockCache(5000);
71 this.remascCache = new MaxSizeHashMap<>(50000, true);
72 }
73
74 @Override
75 public synchronized void removeBlock(Block block) {
76 this.blockCache.removeBlock(block);
77 this.remascCache.remove(block.getHash());
78 this.blocks.delete(block.getHash().getBytes());
79
80 List<BlockInfo> binfos = this.index.getBlocksByNumber(block.getNumber());
81
82 if (binfos == null) {
83 return;
84 }
85
86 List<BlockInfo> toremove = new ArrayList<>();
87
88 for (BlockInfo binfo : binfos) {
89 if (binfo.getHash().equals(block.getHash())) {
90 toremove.add(binfo);
91 }
92 }
93
94 binfos.removeAll(toremove);
95 }
96
97 @Override
98 public synchronized Block getBestBlock() {
99 if (index.isEmpty()) {
100 return null;
101 }
102
103 long maxLevel = index.getMaxNumber();
104 Block bestBlock = getChainBlockByNumber(maxLevel);
105 if (bestBlock != null) {
106 return bestBlock;
107 }
108
109 // That scenario can happen
110 // if there is a fork branch that is
111 // higher than main branch but has
112 // less TD than the main branch TD
113 while (bestBlock == null && maxLevel >= 0) {
114 --maxLevel;
115 bestBlock = getChainBlockByNumber(maxLevel);
116 }
117
118 return bestBlock;
119 }
120
121 @Override
122 public byte[] getBlockHashByNumber(long blockNumber, byte[] branchBlockHash) {
123 Block branchBlock = getBlockByHash(branchBlockHash);
124 if (branchBlock.getNumber() < blockNumber) {
125 throw new IllegalArgumentException(String.format("Requested block number > branch hash number: %d < %d",
126 blockNumber, branchBlock.getNumber()));
127 }
128 while(branchBlock.getNumber() > blockNumber) {
129 branchBlock = getBlockByHash(branchBlock.getParentHash().getBytes());
130 }
131 return branchBlock.getHash().getBytes();
132 }
133
134 @Override
135 // This method is an optimized way to traverse a branch in search for a block at a given depth. Starting at a given
136 // block (by hash) it tries to find the first block that is part of the best chain, when it finds one we now that
137 // we can jump to the block that is at the remaining depth. If not block is found then it continues traversing the
138 // branch from parent to parent. The search is limited by the maximum depth received as parameter.
139 // This method either needs to traverse the parent chain or if a block in the parent chain is part of the best chain
140 // then it can skip the traversal by going directly to the block at the remaining depth.
141 public Block getBlockAtDepthStartingAt(long depth, byte[] hash) {
142 Block start = this.getBlockByHash(hash);
143
144 if (start != null && depth == 0) {
145 return start;
146 }
147
148 if (start == null || start.getNumber() <= depth) {
149 return null;
150 }
151
152 Block block = start;
153
154 for (long i = 0; i < depth; i++) {
155 if (isBlockInMainChain(block.getNumber(), block.getHash())) {
156 return getChainBlockByNumber(start.getNumber() - depth);
157 }
158
159 block = this.getBlockByHash(block.getParentHash().getBytes());
160 }
161
162 return block;
163 }
164
165 public boolean isBlockInMainChain(long blockNumber, Keccak256 blockHash){
166 List<BlockInfo> blockInfos = index.getBlocksByNumber(blockNumber);
167 if (blockInfos == null) {
168 return false;
169 }
170
171 for (BlockInfo blockInfo : blockInfos) {
172 if (blockInfo.isMainChain() && blockHash.equals(blockInfo.getHash())) {
173 return true;
174 }
175 }
176
177 return false;
178 }
179
180 @Override
181 public synchronized void flush() {
182 Metric metric = profiler.start(Profiler.PROFILING_TYPE.DB_WRITE);
183 index.flush();
184 profiler.stop(metric);
185 }
186
187 public void close() {
188 this.index.close();
189 }
190
191 @Override
192 public Bloom bloomByBlockNumber(long blockNumber) {
193 return new Bloom(getChainBlockByNumber(blockNumber).getLogBloom());
194 }
195
196 @Override
197 public synchronized void saveBlock(Block block, BlockDifficulty cummDifficulty, boolean mainChain) {
198 List<BlockInfo> blockInfos = index.getBlocksByNumber(block.getNumber());
199
200 BlockInfo blockInfo = null;
201 for (BlockInfo bi : blockInfos) {
202 if (bi.getHash().equals(block.getHash())) {
203 blockInfo = bi;
204 } else if (mainChain) {
205 bi.setMainChain(false);
206 }
207 }
208 if (blockInfo == null) {
209 blockInfo = new BlockInfo();
210 blockInfos.add(blockInfo);
211 }
212
213 blockInfo.setCummDifficulty(cummDifficulty);
214 blockInfo.setHash(block.getHash().getBytes());
215 blockInfo.setMainChain(mainChain);
216
217 if (blocks.get(block.getHash().getBytes()) == null) {
218 blocks.put(block.getHash().getBytes(), block.getEncoded());
219 }
220
221 index.putBlocks(block.getNumber(), blockInfos);
222 blockCache.addBlock(block);
223 remascCache.put(block.getHash(), getSiblingsFromBlock(block));
224 }
225
226 @Override
227 public synchronized List<BlockInformation> getBlocksInformationByNumber(long number) {
228 List<BlockInformation> result = new ArrayList<>();
229
230 List<BlockInfo> blockInfos = index.getBlocksByNumber(number);
231
232 for (BlockInfo blockInfo : blockInfos) {
233 byte[] hash = blockInfo.getHash().copy().getBytes();
234 BlockDifficulty totalDifficulty = blockInfo.getCummDifficulty();
235 boolean isInBlockChain = blockInfo.isMainChain();
236
237 result.add(new BlockInformation(hash, totalDifficulty, isInBlockChain));
238 }
239
240 return result;
241 }
242
243 @Override
244 public boolean isEmpty() {
245 return index.isEmpty();
246 }
247
248 @Override
249 public synchronized Block getChainBlockByNumber(long number){
250 List<BlockInfo> blockInfos = index.getBlocksByNumber(number);
251
252 for (BlockInfo blockInfo : blockInfos) {
253 if (blockInfo.isMainChain()) {
254
255 byte[] hash = blockInfo.getHash().getBytes();
256 return getBlockByHash(hash);
257 }
258 }
259
260 return null;
261 }
262
263 @Override
264 public synchronized Block getBlockByHash(byte[] hash) {
265
266 Block block = getBlock(hash);
267 if (block == null) {
268 return null;
269 }
270
271 blockCache.addBlock(block);
272 remascCache.put(block.getHash(), getSiblingsFromBlock(block));
273 return block;
274 }
275
276 private synchronized Block getBlock(byte[] hash) {
277 Block block = this.blockCache.getBlockByHash(hash);
278
279 if (block != null) {
280 return block;
281 }
282
283 byte[] blockRlp = blocks.get(hash);
284 if (blockRlp == null) {
285 return null;
286 }
287
288 return blockFactory.decodeBlock(blockRlp);
289 }
290
291 @Override
292 public synchronized Map<Long, List<Sibling>> getSiblingsFromBlockByHash(Keccak256 hash) {
293 return this.remascCache.computeIfAbsent(hash, key -> getSiblingsFromBlock(getBlock(key.getBytes())));
294 }
295
296 @Override
297 public synchronized boolean isBlockExist(byte[] hash) {
298 return getBlockByHash(hash) != null;
299 }
300
301 @Override
302 public synchronized BlockDifficulty getTotalDifficultyForHash(byte[] hash){
303 Block block = this.getBlockByHash(hash);
304 if (block == null) {
305 return ZERO;
306 }
307
308 long level = block.getNumber();
309 List<BlockInfo> blockInfos = index.getBlocksByNumber(level);
310
311 for (BlockInfo blockInfo : blockInfos) {
312 if (Arrays.equals(blockInfo.getHash().getBytes(), hash)) {
313 return blockInfo.getCummDifficulty();
314 }
315 }
316
317 return ZERO;
318 }
319
320 @Override
321 public long getMaxNumber() {
322 return index.getMaxNumber();
323 }
324
325 @Override
326 public long getMinNumber() {
327 return index.getMinNumber();
328 }
329
330 @Override
331 public synchronized List<byte[]> getListHashesEndWith(byte[] hash, long number){
332
333 List<Block> blocks = getListBlocksEndWith(hash, number);
334 List<byte[]> hashes = new ArrayList<>(blocks.size());
335
336 for (Block b : blocks) {
337 hashes.add(b.getHash().getBytes());
338 }
339
340 return hashes;
341 }
342
343 private synchronized List<Block> getListBlocksEndWith(byte[] hash, long qty) {
344 Block block = getBlockByHash(hash);
345
346 if (block == null) {
347 return new ArrayList<>();
348 }
349
350 List<Block> blocks = new ArrayList<>((int) qty);
351
352 for (int i = 0; i < qty; ++i) {
353
354 blocks.add(block);
355 block = getBlockByHash(hash);
356 if (block == null) {
357 break;
358 }
359 }
360
361 return blocks;
362 }
363
364 @Override
365 public synchronized void reBranch(Block forkBlock){
366
367 Block bestBlock = getBestBlock();
368 long maxLevel = Math.max(bestBlock.getNumber(), forkBlock.getNumber());
369
370 // 1. First ensure that you are on the save level
371 long currentLevel = maxLevel;
372 Block forkLine = forkBlock;
373
374 if (forkBlock.getNumber() > bestBlock.getNumber()) {
375
376 while(currentLevel > bestBlock.getNumber()) {
377 List<BlockInfo> blocks = index.getBlocksByNumber(currentLevel);
378 BlockInfo blockInfo = getBlockInfoForHash(blocks, forkLine.getHash().getBytes());
379 if (blockInfo != null) {
380 blockInfo.setMainChain(true);
381 if (index.contains(currentLevel)) {
382 index.putBlocks(currentLevel, blocks);
383 }
384 }
385 forkLine = getBlockByHash(forkLine.getParentHash().getBytes());
386 --currentLevel;
387 }
388 }
389
390 Block bestLine = bestBlock;
391 if (bestBlock.getNumber() > forkBlock.getNumber()){
392
393 while(currentLevel > forkBlock.getNumber()) {
394 List<BlockInfo> blocks = index.getBlocksByNumber(currentLevel);
395 BlockInfo blockInfo = getBlockInfoForHash(blocks, bestLine.getHash().getBytes());
396 if (blockInfo != null) {
397 blockInfo.setMainChain(false);
398 if (index.contains(currentLevel)) {
399 index.putBlocks(currentLevel, blocks);
400 }
401 }
402 bestLine = getBlockByHash(bestLine.getParentHash().getBytes());
403 --currentLevel;
404 }
405 }
406
407 // 2. Loop back on each level until common block
408 while( !bestLine.isEqual(forkLine) ) {
409
410 List<BlockInfo> levelBlocks = index.getBlocksByNumber(currentLevel);
411 BlockInfo bestInfo = getBlockInfoForHash(levelBlocks, bestLine.getHash().getBytes());
412 if (bestInfo != null) {
413 bestInfo.setMainChain(false);
414 if (index.contains(currentLevel)) {
415 index.putBlocks(currentLevel, levelBlocks);
416 }
417 }
418
419 BlockInfo forkInfo = getBlockInfoForHash(levelBlocks, forkLine.getHash().getBytes());
420 if (forkInfo != null) {
421 forkInfo.setMainChain(true);
422 if (index.contains(currentLevel)) {
423 index.putBlocks(currentLevel, levelBlocks);
424 }
425 }
426
427 bestLine = getBlockByHash(bestLine.getParentHash().getBytes());
428 forkLine = getBlockByHash(forkLine.getParentHash().getBytes());
429
430 --currentLevel;
431 }
432 }
433
434 @VisibleForTesting
435 public synchronized List<byte[]> getListHashesStartWith(long number, long maxBlocks) {
436
437 List<byte[]> result = new ArrayList<>();
438
439 int i;
440 for (i = 0; i < maxBlocks; ++i) {
441 List<BlockInfo> blockInfos = index.getBlocksByNumber(number);
442 if (blockInfos == null) {
443 break;
444 }
445
446 for (BlockInfo blockInfo : blockInfos) {
447 if (blockInfo.isMainChain()) {
448 result.add(blockInfo.getHash().getBytes());
449 break;
450 }
451 }
452
453 ++number;
454 }
455
456 return result;
457 }
458
459
460
461 private static BlockInfo getBlockInfoForHash(List<BlockInfo> blocks, byte[] hash){
462 if (blocks == null) {
463 return null;
464 }
465
466 for (BlockInfo blockInfo : blocks) {
467 if (Arrays.equals(hash, blockInfo.getHash().getBytes())) {
468 return blockInfo;
469 }
470 }
471
472 return null;
473 }
474
475 @Override
476 public synchronized List<Block> getChainBlocksByNumber(long number){
477 List<Block> result = new ArrayList<>();
478
479 List<BlockInfo> blockInfos = index.getBlocksByNumber(number);
480
481 if (blockInfos == null){
482 return result;
483 }
484
485 for (BlockInfo blockInfo : blockInfos){
486
487 byte[] hash = blockInfo.getHash().getBytes();
488 Block block = getBlockByHash(hash);
489
490 // TODO(mc) investigate and fix this, probably a cache invalidation problem
491 if (block != null) {
492 result.add(block);
493 }
494 }
495
496 return result;
497 }
498
499 /**
500 * Deletes from disk storage all blocks with number strictly larger than blockNumber.
501 * Note that this doesn't clean the caches, making it unsuitable for using after initialization.
502 */
503 @Override
504 public void rewind(long blockNumber) {
505 if (index.isEmpty()) {
506 return;
507 }
508
509 long maxNumber = getMaxNumber();
510 for (long i = maxNumber; i > blockNumber; i--) {
511 List<BlockInfo> blockInfos = index.removeLast();
512
513 for (BlockInfo blockInfo : blockInfos) {
514 this.blocks.delete(blockInfo.getHash().getBytes());
515 }
516 }
517
518 flush();
519 blocks.flush();
520 }
521
522 /**
523 * When a block is processed on remasc the contract needs to calculate all siblings that
524 * that should be rewarded when fees on this block are paid
525 * @param block the block is looked for siblings
526 * @return
527 */
528 private Map<Long, List<Sibling>> getSiblingsFromBlock(Block block) {
529 return block.getUncleList().stream()
530 .collect(
531 Collectors.groupingBy(
532 BlockHeader::getNumber,
533 Collectors.mapping(
534 header -> new Sibling(header, block.getCoinbase(), block.getNumber()),
535 Collectors.toList()
536 )
537 )
538 );
539 }
540
541
542 public static class BlockInfo implements Serializable {
543 private static final long serialVersionUID = 5906746360128478753L;
544
545 private byte[] hash;
546 private BigInteger cummDifficulty;
547 private boolean mainChain;
548
549 public Keccak256 getHash() {
550 return new Keccak256(hash);
551 }
552
553 public void setHash(byte[] hash) {
554 this.hash = hash;
555 }
556
557 public BlockDifficulty getCummDifficulty() {
558 return new BlockDifficulty(cummDifficulty);
559 }
560
561 public void setCummDifficulty(BlockDifficulty cummDifficulty) {
562 this.cummDifficulty = cummDifficulty.asBigInteger();
563 }
564
565 public boolean isMainChain() {
566 return mainChain;
567 }
568
569 public void setMainChain(boolean mainChain) {
570 this.mainChain = mainChain;
571 }
572 }
573
574
575 public static final Serializer<List<BlockInfo>> BLOCK_INFO_SERIALIZER = new Serializer<List<BlockInfo>>(){
576
577 @Override
578 public void serialize(DataOutput out, List<BlockInfo> value) throws IOException {
579 ArrayList<BlockInfo> valueToSerialize = new ArrayList<>(value);
580
581 ByteArrayOutputStream bos = new ByteArrayOutputStream();
582 ObjectOutputStream oos = new ObjectOutputStream(bos);
583 oos.writeObject(valueToSerialize);
584
585 byte[] data = bos.toByteArray();
586 DataIO.packInt(out, data.length);
587 out.write(data);
588 }
589
590 @Override
591 public List<BlockInfo> deserialize(DataInput in, int available) throws IOException {
592
593 ArrayList<BlockInfo> value = null;
594 try {
595 int size = DataIO.unpackInt(in);
596 byte[] data = new byte[size];
597 in.readFully(data);
598
599 ByteArrayInputStream bis = new ByteArrayInputStream(data, 0, data.length);
600 ObjectInputStream ois = new ObjectInputStream(bis);
601 value = (ArrayList<BlockInfo>)ois.readObject();
602 } catch (ClassNotFoundException e) {
603 logger.error("Class not found", e);
604 }
605
606 return value;
607 }
608 };
609 }