Skip to content

Commit 0982653

Browse files
pslaskipslaski
committed
etcm-73 made part of the blockchain transactional
1 parent db81a4d commit 0982653

File tree

67 files changed

+877
-671
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+877
-671
lines changed

‎src/benchmark/scala/io/iohk/ethereum/mpt/MerklePatriciaTreeSpeedSpec.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
package io.iohk.ethereum.mpt
22

3-
import io.iohk.ethereum.{ObjectGenerators, crypto}
43
import io.iohk.ethereum.db.dataSource.EphemDataSource
54
import io.iohk.ethereum.db.storage.{ArchiveNodeStorage, MptStorage, NodeStorage, SerializingMptStorage}
65
import io.iohk.ethereum.mpt.MerklePatriciaTrie.defaultByteArraySerializable
76
import io.iohk.ethereum.utils.Logger
7+
import io.iohk.ethereum.{ObjectGenerators, crypto}
8+
import org.bouncycastle.util.encoders.Hex
89
import org.scalatest.FunSuite
910
import org.scalatest.prop.PropertyChecks
10-
import org.bouncycastle.util.encoders.Hex
1111

1212
class MerklePatriciaTreeSpeedSpec extends FunSuite
1313
with PropertyChecks

‎src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala

+7-5
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import io.iohk.ethereum.consensus.ethash.EthashConsensus
66
import io.iohk.ethereum.consensus.ethash.validators.EthashValidators
77
import io.iohk.ethereum.consensus.{ConsensusConfig, FullConsensusConfig, TestConsensus, ethash}
88
import io.iohk.ethereum.db.components.Storages.PruningModeComponent
9-
import io.iohk.ethereum.db.components.{SharedEphemDataSources, Storages}
9+
import io.iohk.ethereum.db.components.{EphemDataSourceComponent, Storages}
1010
import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode}
1111
import io.iohk.ethereum.domain.Block.BlockDec
1212
import io.iohk.ethereum.domain._
@@ -36,7 +36,7 @@ object ScenarioSetup {
3636

3737

3838
def getBlockchain: BlockchainImpl = {
39-
val storagesInstance = new SharedEphemDataSources with Pruning with Storages.DefaultStorages
39+
val storagesInstance = new EphemDataSourceComponent with Pruning with Storages.DefaultStorages
4040
BlockchainImpl(storagesInstance.storages)
4141
}
4242
}
@@ -75,9 +75,11 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
7575
Block(scenario.genesisBlockHeader.toBlockHeader, BlockBody(Nil, Nil))
7676
}
7777

78-
blockchain.save(genesisBlock)
79-
blockchain.save(genesisBlock.header.hash, Nil)
80-
blockchain.save(genesisBlock.header.hash, genesisBlock.header.difficulty)
78+
blockchain.storeBlock(genesisBlock)
79+
.and(blockchain.storeReceipts(genesisBlock.header.hash, Nil))
80+
.and(blockchain.storeTotalDifficulty(genesisBlock.header.hash, genesisBlock.header.difficulty))
81+
.commit()
82+
8183
genesisBlock
8284
}
8385

‎src/it/scala/io/iohk/ethereum/db/DataSourceIntegrationTestBehavior.scala

+60-37
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ import java.nio.file.Files
55

66
import akka.util.ByteString
77
import io.iohk.ethereum.ObjectGenerators
8-
import io.iohk.ethereum.db.dataSource.DataSource
8+
import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceUpdate}
9+
import io.iohk.ethereum.db.dataSource.DataSource.{Key, Namespace, Value}
910
import org.scalatest.FlatSpec
1011
import org.scalatest.prop.PropertyChecks
1112

@@ -34,9 +35,16 @@ trait DataSourceIntegrationTestBehavior
3435
}
3536
}
3637

37-
def updateInSeparateCalls(dataSource: DataSource, toUpsert: Seq[(ByteString, ByteString)]): DataSource = {
38-
toUpsert.foldLeft(dataSource) { case (recDB, keyValuePair) =>
39-
recDB.update(OtherNamespace, Seq(), Seq(keyValuePair))
38+
def prepareUpdate(
39+
namespace: Namespace = OtherNamespace,
40+
toRemove: Seq[Key] = Nil,
41+
toUpsert: Seq[(Key, Value)] = Nil
42+
): Seq[DataSourceUpdate] =
43+
Seq(DataSourceUpdate(namespace, toRemove, toUpsert))
44+
45+
def updateInSeparateCalls(dataSource: DataSource, toUpsert: Seq[(ByteString, ByteString)]): Unit = {
46+
toUpsert.foreach { keyValuePair =>
47+
dataSource.update(prepareUpdate(toUpsert = Seq(keyValuePair)))
4048
}
4149
}
4250

@@ -46,8 +54,9 @@ trait DataSourceIntegrationTestBehavior
4654
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
4755
withDir { path =>
4856
val keyList = unFilteredKeyList.take(KeyNumberLimit)
49-
val db = updateInSeparateCalls(
50-
dataSource = createDataSource(path),
57+
val db = createDataSource(path)
58+
updateInSeparateCalls(
59+
dataSource = db,
5160
toUpsert = keyList.zip(keyList)
5261
)
5362
keyList.foreach { key => assert(db.get(OtherNamespace, key).contains(key)) }
@@ -61,7 +70,8 @@ trait DataSourceIntegrationTestBehavior
6170
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
6271
withDir { path =>
6372
val keyList = unFilteredKeyList.take(KeyNumberLimit)
64-
val db = createDataSource(path).update(OtherNamespace, Seq(), keyList.zip(keyList))
73+
val db = createDataSource(path)
74+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
6575

6676
keyList.foreach { key => assert(db.get(OtherNamespace, key).contains(key)) }
6777

@@ -74,16 +84,18 @@ trait DataSourceIntegrationTestBehavior
7484
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
7585
withDir { path =>
7686
val keyList = unFilteredKeyList.take(KeyNumberLimit)
77-
val db = createDataSource(path).update(OtherNamespace, Seq(), keyList.zip(keyList))
87+
val db = createDataSource(path)
88+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
7889

7990
val keyListWithExtraByte = keyList.map(1.toByte +: _)
80-
val dbAfterUpdate = updateInSeparateCalls(db, keyList.zip(keyListWithExtraByte))
91+
updateInSeparateCalls(db, keyList.zip(keyListWithExtraByte))
8192

82-
keyList.zip(keyListWithExtraByte).foreach { case (key, value) =>
83-
assert(dbAfterUpdate.get(OtherNamespace, key).contains(value))
93+
keyList.zip(keyListWithExtraByte).foreach {
94+
case (key, value) =>
95+
assert(db.get(OtherNamespace, key).contains(value))
8496
}
8597

86-
dbAfterUpdate.destroy()
98+
db.destroy()
8799
}
88100
}
89101
}
@@ -92,16 +104,18 @@ trait DataSourceIntegrationTestBehavior
92104
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
93105
withDir { path =>
94106
val keyList = unFilteredKeyList.take(KeyNumberLimit)
95-
val db = createDataSource(path).update(OtherNamespace, Seq(), keyList.zip(keyList))
107+
val db = createDataSource(path)
108+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
96109

97110
val keyListWithExtraByte = keyList.map(1.toByte +: _)
98-
val dbAfterUpdate = db.update(OtherNamespace, Seq(), keyList.zip(keyListWithExtraByte))
111+
db.update(prepareUpdate(toUpsert = keyList.zip(keyListWithExtraByte)))
99112

100-
keyList.zip(keyListWithExtraByte).foreach { case (key, value) =>
101-
assert(dbAfterUpdate.get(OtherNamespace, key).contains(value))
113+
keyList.zip(keyListWithExtraByte).foreach {
114+
case (key, value) =>
115+
assert(db.get(OtherNamespace, key).contains(value))
102116
}
103117

104-
dbAfterUpdate.destroy()
118+
db.destroy()
105119
}
106120
}
107121
}
@@ -110,8 +124,9 @@ trait DataSourceIntegrationTestBehavior
110124
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
111125
withDir { path =>
112126
val keyList = unFilteredKeyList.take(KeyNumberLimit)
113-
val db = createDataSource(path).update(namespace = OtherNamespace, toRemove = Seq(), toUpsert = keyList.zip(keyList))
114-
.clear
127+
val db = createDataSource(path)
128+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
129+
db.clear()
115130

116131
keyList.foreach { key => assert(db.get(OtherNamespace, key).isEmpty) }
117132

@@ -124,7 +139,8 @@ trait DataSourceIntegrationTestBehavior
124139
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
125140
withDir { path =>
126141
val keyList = unFilteredKeyList.take(KeyNumberLimit)
127-
val db = createDataSource(path).update(namespace = OtherNamespace, toRemove = Seq(), toUpsert = keyList.zip(keyList))
142+
val db = createDataSource(path)
143+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
128144
db.close()
129145

130146
val dbAfterClose = createDataSource(path)
@@ -139,10 +155,11 @@ trait DataSourceIntegrationTestBehavior
139155
withDir { path =>
140156
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
141157
val keyList = unFilteredKeyList.take(KeyNumberLimit)
142-
val db = createDataSource(path).update(namespace = OtherNamespace, toRemove = Seq(), toUpsert = keyList.zip(keyList))
158+
val db = createDataSource(path)
159+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
143160
db.destroy()
144161

145-
assert(!new File(path).exists())
162+
assert(!new File("/tmp/iodbDestroy").exists())
146163

147164
val dbAfterDestroy = createDataSource(path)
148165
keyList.foreach { key => assert(dbAfterDestroy.get(OtherNamespace, key).isEmpty) }
@@ -160,15 +177,15 @@ trait DataSourceIntegrationTestBehavior
160177
val db = createDataSource(path)
161178

162179
val valList1 = keyList.map(1.toByte +: _)
163-
db.update(OtherNamespace, Seq(), keyList.zip(valList1))
180+
db.update(prepareUpdate(namespace = OtherNamespace, toUpsert = keyList.zip(valList1)))
164181

165182
val valList2 = keyList.map(2.toByte +: _)
166-
db.update(OtherNamespace2, Seq(), keyList.zip(valList2))
183+
db.update(prepareUpdate(namespace = OtherNamespace2, toUpsert = keyList.zip(valList2)))
167184

168-
keyList.zip(valList1).foreach { case (key, value) =>
169-
assert(db.get(OtherNamespace, key).contains(value))
185+
keyList.zip(valList1).foreach {
186+
case (key, value) =>
187+
assert(db.get(OtherNamespace, key).contains(value))
170188
}
171-
172189
keyList.zip(valList2).foreach { case (key, value) =>
173190
assert(db.get(OtherNamespace2, key).contains(value))
174191
}
@@ -186,25 +203,31 @@ trait DataSourceIntegrationTestBehavior
186203
val db = createDataSource(path)
187204

188205
val valList1 = keyList.map(1.toByte +: _)
189-
db.update(OtherNamespace, Seq(), keyList.zip(valList1))
206+
db.update(prepareUpdate(namespace = OtherNamespace, toUpsert = keyList.zip(valList1)))
190207

191208
val valList2 = keyList.map(2.toByte +: _)
192-
db.update(OtherNamespace2, Seq(), keyList.zip(valList2))
209+
db.update(prepareUpdate(namespace = OtherNamespace2, toUpsert = keyList.zip(valList2)))
193210

194211
//Removal of keys from the OtherNamespace namespace
195-
db.update(OtherNamespace, keyList, Nil)
212+
db.update(prepareUpdate(namespace = OtherNamespace, toRemove = keyList))
196213

197-
keyList.foreach { key => assert(db.get(OtherNamespace, key).isEmpty) }
198-
keyList.zip(valList2).foreach { case (key, value) =>
199-
assert(db.get(OtherNamespace2, key).contains(value))
214+
keyList.foreach { key =>
215+
assert(db.get(OtherNamespace, key).isEmpty)
216+
}
217+
keyList.zip(valList2).foreach {
218+
case (key, value) =>
219+
assert(db.get(OtherNamespace2, key).contains(value))
200220
}
201221

202222
//Removal of keys from the OtherNamespace2 namespace
203-
db.update(OtherNamespace2, keyList, Nil)
204-
205-
keyList.foreach { key => assert(db.get(OtherNamespace, key).isEmpty) }
206-
keyList.foreach { key => assert(db.get(OtherNamespace2, key).isEmpty) }
223+
db.update(prepareUpdate(namespace = OtherNamespace2, toRemove = keyList))
207224

225+
keyList.foreach { key =>
226+
assert(db.get(OtherNamespace, key).isEmpty)
227+
}
228+
keyList.foreach { key =>
229+
assert(db.get(OtherNamespace2, key).isEmpty)
230+
}
208231
db.destroy()
209232
}
210233
}

‎src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala

+8-7
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import akka.actor.ActorSystem
44
import akka.util.ByteString
55
import com.typesafe.config.ConfigFactory
66
import io.iohk.ethereum.db.components.Storages.PruningModeComponent
7-
import io.iohk.ethereum.db.components.{SharedRocksDbDataSources, Storages}
7+
import io.iohk.ethereum.db.components.{RocksDbDataSourceComponent, Storages}
88
import io.iohk.ethereum.db.storage.{AppStateStorage, StateStorage}
99
import io.iohk.ethereum.db.storage.NodeStorage.{NodeEncoded, NodeHash}
1010
import io.iohk.ethereum.db.storage.TransactionMappingStorage.TransactionLocation
@@ -22,6 +22,7 @@ import io.iohk.ethereum.nodebuilder.{AuthHandshakerBuilder, NodeKeyBuilder, Secu
2222
import io.iohk.ethereum.utils.{Config, NodeStatus, ServerStatus}
2323
import java.util.concurrent.atomic.AtomicReference
2424

25+
import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate
2526
import org.bouncycastle.util.encoders.Hex
2627

2728
import scala.concurrent.duration._
@@ -59,7 +60,7 @@ object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder wit
5960
trait PruningConfig extends PruningModeComponent {
6061
override val pruningMode: PruningMode = ArchivePruning
6162
}
62-
val storagesInstance = new SharedRocksDbDataSources with PruningConfig with Storages.DefaultStorages
63+
val storagesInstance = new RocksDbDataSourceComponent with PruningConfig with Storages.DefaultStorages
6364

6465
val blockchain: Blockchain = new BlockchainMock(genesisHash)
6566

@@ -114,15 +115,15 @@ object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder wit
114115

115116
override def getMptNodeByHash(hash: ByteString): Option[MptNode] = ???
116117

117-
override def save(blockHeader: BlockHeader): Unit = ???
118+
override def storeBlockHeader(blockHeader: BlockHeader): DataSourceBatchUpdate = ???
118119

119-
override def save(blockHash: ByteString, blockBody: BlockBody): Unit = ???
120+
override def storeBlockBody(blockHash: ByteString, blockBody: BlockBody): DataSourceBatchUpdate = ???
120121

121-
override def save(blockHash: ByteString, receipts: Seq[Receipt]): Unit = ???
122+
override def storeReceipts(blockHash: ByteString, receipts: Seq[Receipt]): DataSourceBatchUpdate = ???
122123

123-
override def save(hash: ByteString, evmCode: ByteString): Unit = ???
124+
override def storeEvmCode(hash: ByteString, evmCode: ByteString): DataSourceBatchUpdate = ???
124125

125-
override def save(blockhash: ByteString, totalDifficulty: BigInt): Unit = ???
126+
override def storeTotalDifficulty(blockhash: ByteString, totalDifficulty: BigInt): DataSourceBatchUpdate = ???
126127

127128
override def saveNode(nodeHash: NodeHash, nodeEncoded: NodeEncoded, blockNumber: BigInt): Unit = ???
128129

‎src/it/scala/io/iohk/ethereum/txExecTest/util/FixtureProvider.scala

+19-15
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package io.iohk.ethereum.txExecTest.util
33
import java.io.Closeable
44

55
import akka.util.ByteString
6-
import io.iohk.ethereum.db.dataSource.EphemDataSource
76
import io.iohk.ethereum.db.storage._
87
import io.iohk.ethereum.domain._
98
import io.iohk.ethereum.domain.BlockHeader._
@@ -12,6 +11,7 @@ import io.iohk.ethereum.network.p2p.messages.PV63._
1211
import MptNodeEncoders._
1312
import ReceiptImplicits._
1413
import io.iohk.ethereum.db.cache.{AppCaches, LruCache}
14+
import io.iohk.ethereum.db.components.EphemDataSourceComponent
1515
import io.iohk.ethereum.db.storage.NodeStorage.NodeHash
1616
import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode}
1717
import io.iohk.ethereum.mpt.{BranchNode, ExtensionNode, HashNode, LeafNode, MptNode}
@@ -37,19 +37,19 @@ object FixtureProvider {
3737
// scalastyle:off
3838
def prepareStorages(blockNumber: BigInt, fixtures: Fixture): BlockchainStorages = {
3939

40-
val storages: BlockchainStorages = new BlockchainStorages with AppCaches {
40+
val storages: BlockchainStorages = new BlockchainStorages with AppCaches with EphemDataSourceComponent {
4141

42-
override val receiptStorage: ReceiptStorage = new ReceiptStorage(EphemDataSource())
43-
override val evmCodeStorage: EvmCodeStorage = new EvmCodeStorage(EphemDataSource())
44-
override val blockHeadersStorage: BlockHeadersStorage = new BlockHeadersStorage(EphemDataSource())
45-
override val blockNumberMappingStorage: BlockNumberMappingStorage = new BlockNumberMappingStorage(EphemDataSource())
46-
override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(EphemDataSource())
47-
override val totalDifficultyStorage: TotalDifficultyStorage = new TotalDifficultyStorage(EphemDataSource())
48-
override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(EphemDataSource())
49-
override val nodeStorage: NodeStorage = new NodeStorage(EphemDataSource())
42+
override val receiptStorage: ReceiptStorage = new ReceiptStorage(dataSource)
43+
override val evmCodeStorage: EvmCodeStorage = new EvmCodeStorage(dataSource)
44+
override val blockHeadersStorage: BlockHeadersStorage = new BlockHeadersStorage(dataSource)
45+
override val blockNumberMappingStorage: BlockNumberMappingStorage = new BlockNumberMappingStorage(dataSource)
46+
override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(dataSource)
47+
override val totalDifficultyStorage: TotalDifficultyStorage = new TotalDifficultyStorage(dataSource)
48+
override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(dataSource)
49+
override val nodeStorage: NodeStorage = new NodeStorage(dataSource)
5050
override val cachedNodeStorage: CachedNodeStorage = new CachedNodeStorage(nodeStorage, caches.nodeCache)
5151
override val pruningMode: PruningMode = ArchivePruning
52-
override val appStateStorage: AppStateStorage = new AppStateStorage(EphemDataSource())
52+
override val appStateStorage: AppStateStorage = new AppStateStorage(dataSource)
5353
override val stateStorage: StateStorage =
5454
StateStorage(
5555
pruningMode,
@@ -63,10 +63,14 @@ object FixtureProvider {
6363
val blockchain = BlockchainImpl(storages)
6464

6565
blocksToInclude.foreach { case (_, block) =>
66+
val receiptsUpdates = fixtures.receipts.get(block.header.hash)
67+
.map(r => storages.receiptStorage.put(block.header.hash, r))
68+
.getOrElse(storages.receiptStorage.emptyBatchUpdate)
6669
storages.blockBodiesStorage.put(block.header.hash, fixtures.blockBodies(block.header.hash))
67-
storages.blockHeadersStorage.put(block.header.hash, fixtures.blockHeaders(block.header.hash))
68-
storages.blockNumberMappingStorage.put(block.header.number, block.header.hash)
69-
fixtures.receipts.get(block.header.hash).foreach(r => storages.receiptStorage.put(block.header.hash, r))
70+
.and(storages.blockHeadersStorage.put(block.header.hash, fixtures.blockHeaders(block.header.hash)))
71+
.and(storages.blockNumberMappingStorage.put(block.header.number, block.header.hash))
72+
.and(receiptsUpdates)
73+
.commit()
7074

7175
def traverse(nodeHash: ByteString): Unit = fixtures.stateMpt.get(nodeHash).orElse(fixtures.contractMpts.get(nodeHash)) match {
7276
case Some(m: BranchNode) =>
@@ -85,7 +89,7 @@ object FixtureProvider {
8589
storages.stateStorage.saveNode(ByteString(m.hash), m.toBytes, block.header.number)
8690
Try(m.value.toArray[Byte].toAccount).toOption.foreach { account =>
8791
if (account.codeHash != DumpChainActor.emptyEvm) {
88-
storages.evmCodeStorage.put(account.codeHash, fixtures.evmCode(account.codeHash))
92+
storages.evmCodeStorage.put(account.codeHash, fixtures.evmCode(account.codeHash)).commit()
8993
}
9094
if (account.storageRoot != DumpChainActor.emptyStorage) {
9195
traverse(account.storageRoot)

0 commit comments

Comments
 (0)