Skip to content

Commit 01484e1

Browse files
pslaskipslaski
committed
etcm-73 made part of the blockchain transactional
1 parent 4684082 commit 01484e1

File tree

66 files changed

+873
-705
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+873
-705
lines changed

‎src/benchmark/scala/io/iohk/ethereum/mpt/MerklePatriciaTreeSpeedSpec.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
package io.iohk.ethereum.mpt
22

3-
import io.iohk.ethereum.{ObjectGenerators, crypto}
43
import io.iohk.ethereum.db.dataSource.EphemDataSource
54
import io.iohk.ethereum.db.storage.{ArchiveNodeStorage, MptStorage, NodeStorage, SerializingMptStorage}
65
import io.iohk.ethereum.mpt.MerklePatriciaTrie.defaultByteArraySerializable
76
import io.iohk.ethereum.utils.Logger
8-
import org.scalatest.FunSuite
7+
import io.iohk.ethereum.{ObjectGenerators, crypto}
98
import org.bouncycastle.util.encoders.Hex
9+
import org.scalatest.FunSuite
1010
import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
1111

1212
class MerklePatriciaTreeSpeedSpec

‎src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala

+7-5
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import io.iohk.ethereum.consensus.ethash.EthashConsensus
66
import io.iohk.ethereum.consensus.ethash.validators.ValidatorsExecutor
77
import io.iohk.ethereum.consensus.{ConsensusConfig, FullConsensusConfig, TestConsensus, ethash}
88
import io.iohk.ethereum.db.components.Storages.PruningModeComponent
9-
import io.iohk.ethereum.db.components.{SharedEphemDataSources, Storages}
9+
import io.iohk.ethereum.db.components.{EphemDataSourceComponent, Storages}
1010
import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode}
1111
import io.iohk.ethereum.domain.Block.BlockDec
1212
import io.iohk.ethereum.domain._
@@ -36,7 +36,7 @@ object ScenarioSetup {
3636

3737

3838
def getBlockchain: BlockchainImpl = {
39-
val storagesInstance = new SharedEphemDataSources with Pruning with Storages.DefaultStorages
39+
val storagesInstance = new EphemDataSourceComponent with Pruning with Storages.DefaultStorages
4040
BlockchainImpl(storagesInstance.storages)
4141
}
4242
}
@@ -75,9 +75,11 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
7575
Block(scenario.genesisBlockHeader.toBlockHeader, BlockBody(Nil, Nil))
7676
}
7777

78-
blockchain.save(genesisBlock)
79-
blockchain.save(genesisBlock.header.hash, Nil)
80-
blockchain.save(genesisBlock.header.hash, genesisBlock.header.difficulty)
78+
blockchain.storeBlock(genesisBlock)
79+
.and(blockchain.storeReceipts(genesisBlock.header.hash, Nil))
80+
.and(blockchain.storeTotalDifficulty(genesisBlock.header.hash, genesisBlock.header.difficulty))
81+
.commit()
82+
8183
genesisBlock
8284
}
8385

‎src/it/scala/io/iohk/ethereum/db/DataSourceIntegrationTestBehavior.scala

+52-65
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ import java.io.File
44
import java.nio.file.Files
55
import akka.util.ByteString
66
import io.iohk.ethereum.ObjectGenerators
7-
import io.iohk.ethereum.db.dataSource.DataSource
7+
import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceUpdate}
8+
import io.iohk.ethereum.db.dataSource.DataSource.{Key, Namespace, Value}
89
import org.scalatest.FlatSpec
910
import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
1011

@@ -30,12 +31,19 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
3031
}
3132
}
3233

34+
def prepareUpdate(
35+
namespace: Namespace = OtherNamespace,
36+
toRemove: Seq[Key] = Nil,
37+
toUpsert: Seq[(Key, Value)] = Nil
38+
): Seq[DataSourceUpdate] =
39+
Seq(DataSourceUpdate(namespace, toRemove, toUpsert))
40+
3341
def updateInSeparateCalls(
34-
dataSource: DataSource,
35-
toUpsert: Seq[(ByteString, ByteString)]
36-
): DataSource = {
37-
toUpsert.foldLeft(dataSource) { case (recDB, keyValuePair) =>
38-
recDB.update(OtherNamespace, Seq(), Seq(keyValuePair))
42+
dataSource: DataSource,
43+
toUpsert: Seq[(ByteString, ByteString)]
44+
): Unit = {
45+
toUpsert.foreach { keyValuePair =>
46+
dataSource.update(prepareUpdate(toUpsert = Seq(keyValuePair)))
3947
}
4048
}
4149

@@ -45,8 +53,9 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
4553
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
4654
withDir { path =>
4755
val keyList = unFilteredKeyList.take(KeyNumberLimit)
48-
val db = updateInSeparateCalls(
49-
dataSource = createDataSource(path),
56+
val db = createDataSource(path)
57+
updateInSeparateCalls(
58+
dataSource = db,
5059
toUpsert = keyList.zip(keyList)
5160
)
5261
keyList.foreach { key =>
@@ -62,11 +71,8 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
6271
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
6372
withDir { path =>
6473
val keyList = unFilteredKeyList.take(KeyNumberLimit)
65-
val db = createDataSource(path).update(
66-
OtherNamespace,
67-
Seq(),
68-
keyList.zip(keyList)
69-
)
74+
val db = createDataSource(path)
75+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
7076

7177
keyList.foreach { key =>
7278
assert(db.get(OtherNamespace, key).contains(key))
@@ -81,21 +87,18 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
8187
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
8288
withDir { path =>
8389
val keyList = unFilteredKeyList.take(KeyNumberLimit)
84-
val db = createDataSource(path).update(
85-
OtherNamespace,
86-
Seq(),
87-
keyList.zip(keyList)
88-
)
90+
val db = createDataSource(path)
91+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
8992

9093
val keyListWithExtraByte = keyList.map(1.toByte +: _)
91-
val dbAfterUpdate =
92-
updateInSeparateCalls(db, keyList.zip(keyListWithExtraByte))
94+
updateInSeparateCalls(db, keyList.zip(keyListWithExtraByte))
9395

94-
keyList.zip(keyListWithExtraByte).foreach { case (key, value) =>
95-
assert(dbAfterUpdate.get(OtherNamespace, key).contains(value))
96+
keyList.zip(keyListWithExtraByte).foreach {
97+
case (key, value) =>
98+
assert(db.get(OtherNamespace, key).contains(value))
9699
}
97100

98-
dbAfterUpdate.destroy()
101+
db.destroy()
99102
}
100103
}
101104
}
@@ -104,24 +107,18 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
104107
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
105108
withDir { path =>
106109
val keyList = unFilteredKeyList.take(KeyNumberLimit)
107-
val db = createDataSource(path).update(
108-
OtherNamespace,
109-
Seq(),
110-
keyList.zip(keyList)
111-
)
110+
val db = createDataSource(path)
111+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
112112

113113
val keyListWithExtraByte = keyList.map(1.toByte +: _)
114-
val dbAfterUpdate = db.update(
115-
OtherNamespace,
116-
Seq(),
117-
keyList.zip(keyListWithExtraByte)
118-
)
114+
db.update(prepareUpdate(toUpsert = keyList.zip(keyListWithExtraByte)))
119115

120-
keyList.zip(keyListWithExtraByte).foreach { case (key, value) =>
121-
assert(dbAfterUpdate.get(OtherNamespace, key).contains(value))
116+
keyList.zip(keyListWithExtraByte).foreach {
117+
case (key, value) =>
118+
assert(db.get(OtherNamespace, key).contains(value))
122119
}
123120

124-
dbAfterUpdate.destroy()
121+
db.destroy()
125122
}
126123
}
127124
}
@@ -131,12 +128,8 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
131128
withDir { path =>
132129
val keyList = unFilteredKeyList.take(KeyNumberLimit)
133130
val db = createDataSource(path)
134-
.update(
135-
namespace = OtherNamespace,
136-
toRemove = Seq(),
137-
toUpsert = keyList.zip(keyList)
138-
)
139-
.clear
131+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
132+
db.clear()
140133

141134
keyList.foreach { key =>
142135
assert(db.get(OtherNamespace, key).isEmpty)
@@ -151,11 +144,8 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
151144
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
152145
withDir { path =>
153146
val keyList = unFilteredKeyList.take(KeyNumberLimit)
154-
val db = createDataSource(path).update(
155-
namespace = OtherNamespace,
156-
toRemove = Seq(),
157-
toUpsert = keyList.zip(keyList)
158-
)
147+
val db = createDataSource(path)
148+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
159149
db.close()
160150

161151
val dbAfterClose = createDataSource(path)
@@ -172,14 +162,11 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
172162
withDir { path =>
173163
forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] =>
174164
val keyList = unFilteredKeyList.take(KeyNumberLimit)
175-
val db = createDataSource(path).update(
176-
namespace = OtherNamespace,
177-
toRemove = Seq(),
178-
toUpsert = keyList.zip(keyList)
179-
)
165+
val db = createDataSource(path)
166+
db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
180167
db.destroy()
181168

182-
assert(!new File(path).exists())
169+
assert(!new File("/tmp/iodbDestroy").exists())
183170

184171
val dbAfterDestroy = createDataSource(path)
185172
keyList.foreach { key =>
@@ -199,15 +186,15 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
199186
val db = createDataSource(path)
200187

201188
val valList1 = keyList.map(1.toByte +: _)
202-
db.update(OtherNamespace, Seq(), keyList.zip(valList1))
189+
db.update(prepareUpdate(namespace = OtherNamespace, toUpsert = keyList.zip(valList1)))
203190

204191
val valList2 = keyList.map(2.toByte +: _)
205-
db.update(OtherNamespace2, Seq(), keyList.zip(valList2))
192+
db.update(prepareUpdate(namespace = OtherNamespace2, toUpsert = keyList.zip(valList2)))
206193

207-
keyList.zip(valList1).foreach { case (key, value) =>
208-
assert(db.get(OtherNamespace, key).contains(value))
194+
keyList.zip(valList1).foreach {
195+
case (key, value) =>
196+
assert(db.get(OtherNamespace, key).contains(value))
209197
}
210-
211198
keyList.zip(valList2).foreach { case (key, value) =>
212199
assert(db.get(OtherNamespace2, key).contains(value))
213200
}
@@ -225,31 +212,31 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
225212
val db = createDataSource(path)
226213

227214
val valList1 = keyList.map(1.toByte +: _)
228-
db.update(OtherNamespace, Seq(), keyList.zip(valList1))
215+
db.update(prepareUpdate(namespace = OtherNamespace, toUpsert = keyList.zip(valList1)))
229216

230217
val valList2 = keyList.map(2.toByte +: _)
231-
db.update(OtherNamespace2, Seq(), keyList.zip(valList2))
218+
db.update(prepareUpdate(namespace = OtherNamespace2, toUpsert = keyList.zip(valList2)))
232219

233220
//Removal of keys from the OtherNamespace namespace
234-
db.update(OtherNamespace, keyList, Nil)
221+
db.update(prepareUpdate(namespace = OtherNamespace, toRemove = keyList))
235222

236223
keyList.foreach { key =>
237224
assert(db.get(OtherNamespace, key).isEmpty)
238225
}
239-
keyList.zip(valList2).foreach { case (key, value) =>
240-
assert(db.get(OtherNamespace2, key).contains(value))
226+
keyList.zip(valList2).foreach {
227+
case (key, value) =>
228+
assert(db.get(OtherNamespace2, key).contains(value))
241229
}
242230

243231
//Removal of keys from the OtherNamespace2 namespace
244-
db.update(OtherNamespace2, keyList, Nil)
232+
db.update(prepareUpdate(namespace = OtherNamespace2, toRemove = keyList))
245233

246234
keyList.foreach { key =>
247235
assert(db.get(OtherNamespace, key).isEmpty)
248236
}
249237
keyList.foreach { key =>
250238
assert(db.get(OtherNamespace2, key).isEmpty)
251239
}
252-
253240
db.destroy()
254241
}
255242
}

‎src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala

+8-7
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import akka.actor.ActorSystem
44
import akka.util.ByteString
55
import com.typesafe.config.ConfigFactory
66
import io.iohk.ethereum.db.components.Storages.PruningModeComponent
7-
import io.iohk.ethereum.db.components.{SharedRocksDbDataSources, Storages}
7+
import io.iohk.ethereum.db.components.{RocksDbDataSourceComponent, Storages}
88
import io.iohk.ethereum.db.storage.{AppStateStorage, StateStorage}
99
import io.iohk.ethereum.db.storage.NodeStorage.{NodeEncoded, NodeHash}
1010
import io.iohk.ethereum.db.storage.TransactionMappingStorage.TransactionLocation
@@ -22,6 +22,7 @@ import io.iohk.ethereum.nodebuilder.{AuthHandshakerBuilder, NodeKeyBuilder, Secu
2222
import io.iohk.ethereum.utils.{Config, NodeStatus, ServerStatus}
2323
import java.util.concurrent.atomic.AtomicReference
2424

25+
import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate
2526
import org.bouncycastle.util.encoders.Hex
2627

2728
import scala.concurrent.duration._
@@ -59,7 +60,7 @@ object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder wit
5960
trait PruningConfig extends PruningModeComponent {
6061
override val pruningMode: PruningMode = ArchivePruning
6162
}
62-
val storagesInstance = new SharedRocksDbDataSources with PruningConfig with Storages.DefaultStorages
63+
val storagesInstance = new RocksDbDataSourceComponent with PruningConfig with Storages.DefaultStorages
6364

6465
val blockchain: Blockchain = new BlockchainMock(genesisHash)
6566

@@ -114,15 +115,15 @@ object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder wit
114115

115116
override def getMptNodeByHash(hash: ByteString): Option[MptNode] = ???
116117

117-
override def save(blockHeader: BlockHeader): Unit = ???
118+
override def storeBlockHeader(blockHeader: BlockHeader): DataSourceBatchUpdate = ???
118119

119-
override def save(blockHash: ByteString, blockBody: BlockBody): Unit = ???
120+
override def storeBlockBody(blockHash: ByteString, blockBody: BlockBody): DataSourceBatchUpdate = ???
120121

121-
override def save(blockHash: ByteString, receipts: Seq[Receipt]): Unit = ???
122+
override def storeReceipts(blockHash: ByteString, receipts: Seq[Receipt]): DataSourceBatchUpdate = ???
122123

123-
override def save(hash: ByteString, evmCode: ByteString): Unit = ???
124+
override def storeEvmCode(hash: ByteString, evmCode: ByteString): DataSourceBatchUpdate = ???
124125

125-
override def save(blockhash: ByteString, totalDifficulty: BigInt): Unit = ???
126+
override def storeTotalDifficulty(blockhash: ByteString, totalDifficulty: BigInt): DataSourceBatchUpdate = ???
126127

127128
override def saveNode(nodeHash: NodeHash, nodeEncoded: NodeEncoded, blockNumber: BigInt): Unit = ???
128129

‎src/it/scala/io/iohk/ethereum/txExecTest/util/FixtureProvider.scala

+19-15
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package io.iohk.ethereum.txExecTest.util
33
import java.io.Closeable
44

55
import akka.util.ByteString
6-
import io.iohk.ethereum.db.dataSource.EphemDataSource
76
import io.iohk.ethereum.db.storage._
87
import io.iohk.ethereum.domain._
98
import io.iohk.ethereum.domain.BlockHeader._
@@ -12,6 +11,7 @@ import io.iohk.ethereum.network.p2p.messages.PV63._
1211
import MptNodeEncoders._
1312
import ReceiptImplicits._
1413
import io.iohk.ethereum.db.cache.{AppCaches, LruCache}
14+
import io.iohk.ethereum.db.components.EphemDataSourceComponent
1515
import io.iohk.ethereum.db.storage.NodeStorage.NodeHash
1616
import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode}
1717
import io.iohk.ethereum.mpt.{BranchNode, ExtensionNode, HashNode, LeafNode, MptNode}
@@ -37,19 +37,19 @@ object FixtureProvider {
3737
// scalastyle:off
3838
def prepareStorages(blockNumber: BigInt, fixtures: Fixture): BlockchainStorages = {
3939

40-
val storages: BlockchainStorages = new BlockchainStorages with AppCaches {
40+
val storages: BlockchainStorages = new BlockchainStorages with AppCaches with EphemDataSourceComponent {
4141

42-
override val receiptStorage: ReceiptStorage = new ReceiptStorage(EphemDataSource())
43-
override val evmCodeStorage: EvmCodeStorage = new EvmCodeStorage(EphemDataSource())
44-
override val blockHeadersStorage: BlockHeadersStorage = new BlockHeadersStorage(EphemDataSource())
45-
override val blockNumberMappingStorage: BlockNumberMappingStorage = new BlockNumberMappingStorage(EphemDataSource())
46-
override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(EphemDataSource())
47-
override val totalDifficultyStorage: TotalDifficultyStorage = new TotalDifficultyStorage(EphemDataSource())
48-
override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(EphemDataSource())
49-
override val nodeStorage: NodeStorage = new NodeStorage(EphemDataSource())
42+
override val receiptStorage: ReceiptStorage = new ReceiptStorage(dataSource)
43+
override val evmCodeStorage: EvmCodeStorage = new EvmCodeStorage(dataSource)
44+
override val blockHeadersStorage: BlockHeadersStorage = new BlockHeadersStorage(dataSource)
45+
override val blockNumberMappingStorage: BlockNumberMappingStorage = new BlockNumberMappingStorage(dataSource)
46+
override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(dataSource)
47+
override val totalDifficultyStorage: TotalDifficultyStorage = new TotalDifficultyStorage(dataSource)
48+
override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(dataSource)
49+
override val nodeStorage: NodeStorage = new NodeStorage(dataSource)
5050
override val cachedNodeStorage: CachedNodeStorage = new CachedNodeStorage(nodeStorage, caches.nodeCache)
5151
override val pruningMode: PruningMode = ArchivePruning
52-
override val appStateStorage: AppStateStorage = new AppStateStorage(EphemDataSource())
52+
override val appStateStorage: AppStateStorage = new AppStateStorage(dataSource)
5353
override val stateStorage: StateStorage =
5454
StateStorage(
5555
pruningMode,
@@ -63,10 +63,14 @@ object FixtureProvider {
6363
val blockchain = BlockchainImpl(storages)
6464

6565
blocksToInclude.foreach { case (_, block) =>
66+
val receiptsUpdates = fixtures.receipts.get(block.header.hash)
67+
.map(r => storages.receiptStorage.put(block.header.hash, r))
68+
.getOrElse(storages.receiptStorage.emptyBatchUpdate)
6669
storages.blockBodiesStorage.put(block.header.hash, fixtures.blockBodies(block.header.hash))
67-
storages.blockHeadersStorage.put(block.header.hash, fixtures.blockHeaders(block.header.hash))
68-
storages.blockNumberMappingStorage.put(block.header.number, block.header.hash)
69-
fixtures.receipts.get(block.header.hash).foreach(r => storages.receiptStorage.put(block.header.hash, r))
70+
.and(storages.blockHeadersStorage.put(block.header.hash, fixtures.blockHeaders(block.header.hash)))
71+
.and(storages.blockNumberMappingStorage.put(block.header.number, block.header.hash))
72+
.and(receiptsUpdates)
73+
.commit()
7074

7175
def traverse(nodeHash: ByteString): Unit = fixtures.stateMpt.get(nodeHash).orElse(fixtures.contractMpts.get(nodeHash)) match {
7276
case Some(m: BranchNode) =>
@@ -85,7 +89,7 @@ object FixtureProvider {
8589
storages.stateStorage.saveNode(ByteString(m.hash), m.toBytes, block.header.number)
8690
Try(m.value.toArray[Byte].toAccount).toOption.foreach { account =>
8791
if (account.codeHash != DumpChainActor.emptyEvm) {
88-
storages.evmCodeStorage.put(account.codeHash, fixtures.evmCode(account.codeHash))
92+
storages.evmCodeStorage.put(account.codeHash, fixtures.evmCode(account.codeHash)).commit()
8993
}
9094
if (account.storageRoot != DumpChainActor.emptyStorage) {
9195
traverse(account.storageRoot)

0 commit comments

Comments
 (0)