Merge remote-tracking branch 'origin/develop' into fix/fix-block-fetcher-bug

Author: KonradStaniec

Diff for: .scalafmt.conf

@@ -0,0 +1,3 @@
+version = "2.3.0"
+align = none
+maxColumn = 120

Diff for: build.sbt

@@ -4,7 +4,8 @@ val commonSettings = Seq(
   name := "mantis",
   version := "3.0",
   scalaVersion := "2.12.12",
-  testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, "-l", "EthashMinerSpec") // miner tests disabled by default
+  testOptions in Test += Tests
+    .Argument(TestFrameworks.ScalaTest, "-l", "EthashMinerSpec") // miner tests disabled by default
 )
 
 // Resolver for rocksDb
@@ -15,7 +16,7 @@ val dep = {
   val akkaHttpVersion = "10.2.0"
   val circeVersion = "0.9.3"
   val rocksDb = "5.9.2"
-  
+
   Seq(
     "com.typesafe.akka" %% "akka-actor" % akkaVersion,
     "com.typesafe.akka" %% "akka-slf4j" % akkaVersion,
@@ -43,15 +44,20 @@ val dep = {
     "org.scala-sbt.ipcsocket" % "ipcsocket" % "1.0.0",
     "org.bouncycastle" % "bcprov-jdk15on" % "1.59",
     "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0",
-    "org.typelevel" %% "mouse" % "0.18",
+    "org.typelevel" %% "mouse" % "0.23",
+    "org.typelevel" %% "cats-core" % "2.0.0",
+    "org.typelevel" %% "cats-effect" % "2.0.0",
     "com.twitter" %% "util-collection" % "18.5.0",
     "com.google.guava" % "guava" % "28.0-jre",
-
+    "io.monix" %% "monix" % "3.1.0",
+    "com.beachape" %% "enumeratum" % "1.5.13",
+    "com.beachape" %% "enumeratum-cats" % "1.5.15",
+    "com.beachape" %% "enumeratum-scalacheck" % "1.5.16" % Test,
     // mallet deps
     "org.jline" % "jline" % "3.1.2",
     "net.java.dev.jna" % "jna" % "4.5.1",
+    "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.5",
     "com.github.scopt" %% "scopt" % "3.7.0",
-
     // Metrics (https://github.com/DataDog/java-dogstatsd-client)
     "com.datadoghq" % "java-dogstatsd-client" % "2.5",
     "org.xerial.snappy" % "snappy-java" % "1.1.7.2",
@@ -71,19 +77,20 @@ val Snappy = config("snappy") extend Test
 
 val Rpc = config("rpcTest") extend Test
 
-val root = project.in(file("."))
-    .configs(Integration, Benchmark, Evm, Ets, Snappy, Rpc)
-    .settings(commonSettings: _*)
-    .settings(
-      libraryDependencies ++= dep,
-      executableScriptName := name.value
-    )
-    .settings(inConfig(Integration)(Defaults.testSettings) : _*)
-    .settings(inConfig(Benchmark)(Defaults.testSettings) : _*)
-    .settings(inConfig(Evm)(Defaults.testSettings) : _*)
-    .settings(inConfig(Ets)(Defaults.testSettings) : _*)
-    .settings(inConfig(Snappy)(Defaults.testSettings) : _*)
-    .settings(inConfig(Rpc)(Defaults.testSettings) : _*)
+val root = project
+  .in(file("."))
+  .configs(Integration, Benchmark, Evm, Ets, Snappy, Rpc)
+  .settings(commonSettings: _*)
+  .settings(
+    libraryDependencies ++= dep,
+    executableScriptName := name.value
+  )
+  .settings(inConfig(Integration)(Defaults.testSettings): _*)
+  .settings(inConfig(Benchmark)(Defaults.testSettings): _*)
+  .settings(inConfig(Evm)(Defaults.testSettings): _*)
+  .settings(inConfig(Ets)(Defaults.testSettings): _*)
+  .settings(inConfig(Snappy)(Defaults.testSettings): _*)
+  .settings(inConfig(Rpc)(Defaults.testSettings): _*)
 
 scalacOptions := Seq(
   "-unchecked",
@@ -93,13 +100,16 @@ scalacOptions := Seq(
   "-Xlint:unsound-match",
   "-Ywarn-inaccessible",
   "-Ywarn-unused-import",
-  "-encoding", "utf-8"
+  "-encoding",
+  "utf-8"
 )
 
-scalacOptions in (Compile, console) ~= (_.filterNot(Set(
-  "-Ywarn-unused-import",
-  "-Xfatal-warnings"
-)))
+scalacOptions in (Compile, console) ~= (_.filterNot(
+  Set(
+    "-Ywarn-unused-import",
+    "-Xfatal-warnings"
+  )
+))
 
 parallelExecution in Test := false
 
@@ -117,7 +127,7 @@ unmanagedResourceDirectories in Compile += baseDirectory.value / "src" / "main"
 (sourceDirectory in Evm) := baseDirectory.value / "src" / "evmTest"
 
 (scalastyleConfig in Test) := baseDirectory.value / "scalastyle-test-config.xml"
-scalastyleSources in Test ++= {(unmanagedSourceDirectories in Integration).value}
+scalastyleSources in Test ++= { (unmanagedSourceDirectories in Integration).value }
 
 mainClass in Compile := Some("io.iohk.ethereum.App")
 

Diff for: insomnia_workspace.json

@@ -1,9 +1,105 @@
 {
   "_type": "export",
   "__export_format": 4,
-  "__export_date": "2020-09-11T12:07:41.513Z",
-  "__export_source": "insomnia.desktop.app:v2020.2.1",
+  "__export_date": "2020-09-16T09:48:00.794Z",
+  "__export_source": "insomnia.desktop.app:v2020.3.3",
   "resources": [
+    {
+      "_id": "req_cd0078ce4a034ebdbdf7dc9e20e78a29",
+      "authentication": {},
+      "body": {
+        "mimeType": "application/json",
+        "text": "{\n\t\"jsonrpc\": \"2.0\",\n  \"method\": \"qa_mineBlocks\", \n  \"params\": [1, true],\n  \"id\": 1\n}"
+      },
+      "created": 1600249397450,
+      "description": "",
+      "headers": [
+        {
+          "id": "pair_9f4d6a9dde554cd384487e04fa3b21aa",
+          "name": "Content-Type",
+          "value": "application/json"
+        },
+        {
+          "id": "pair_088edc31f5e04f20a16b465a673871bb",
+          "name": "Cache-Control",
+          "value": "no-cache"
+        }
+      ],
+      "isPrivate": false,
+      "metaSortKey": -1552939150156.9297,
+      "method": "POST",
+      "modified": 1600249644998,
+      "name": "qa_mineBlocks",
+      "parameters": [],
+      "parentId": "fld_2b54cbb84e244284b3ef752c5f805376",
+      "settingDisableRenderRequestBody": false,
+      "settingEncodeUrl": true,
+      "settingFollowRedirects": "global",
+      "settingRebuildPath": true,
+      "settingSendCookies": true,
+      "settingStoreCookies": true,
+      "url": "{{ node_url }}",
+      "_type": "request"
+    },
+    {
+      "_id": "fld_2b54cbb84e244284b3ef752c5f805376",
+      "created": 1600249374160,
+      "description": "",
+      "environment": {},
+      "environmentPropertyOrder": null,
+      "metaSortKey": -1600249374160,
+      "modified": 1600249374160,
+      "name": "QA",
+      "parentId": "wrk_097d43914a4d4aea8b6f73f647921182",
+      "_type": "request_group"
+    },
+    {
+      "_id": "wrk_097d43914a4d4aea8b6f73f647921182",
+      "created": 1552662762769,
+      "description": "",
+      "modified": 1599825617921,
+      "name": "Mantis",
+      "parentId": null,
+      "scope": null,
+      "_type": "workspace"
+    },
+    {
+      "_id": "req_6197fefa1e1448a89f30712ec12295f8",
+      "authentication": {},
+      "body": {
+        "mimeType": "application/json",
+        "text": "{\n\t\"jsonrpc\": \"2.0\",\n  \"method\": \"qa_getPendingTransactions\", \n  \"params\": [],\n  \"id\": 1\n}"
+      },
+      "created": 1600249491560,
+      "description": "",
+      "headers": [
+        {
+          "id": "pair_9f4d6a9dde554cd384487e04fa3b21aa",
+          "name": "Content-Type",
+          "value": "application/json"
+        },
+        {
+          "id": "pair_088edc31f5e04f20a16b465a673871bb",
+          "name": "Cache-Control",
+          "value": "no-cache"
+        }
+      ],
+      "isPrivate": false,
+      "metaSortKey": -1552939150156.832,
+      "method": "POST",
+      "modified": 1600249571669,
+      "name": "qa_getPendingTransactions",
+      "parameters": [],
+      "parentId": "fld_2b54cbb84e244284b3ef752c5f805376",
+      "settingDisableRenderRequestBody": false,
+      "settingEncodeUrl": true,
+      "settingFollowRedirects": "global",
+      "settingRebuildPath": true,
+      "settingSendCookies": true,
+      "settingStoreCookies": true,
+      "url": "{{ node_url }}",
+      "_type": "request"
+    },
     {
       "_id": "req_dd6e5c718f974407bb79fe3c953b7106",
       "authentication": {},
@@ -53,16 +149,6 @@
       "parentId": "wrk_097d43914a4d4aea8b6f73f647921182",
       "_type": "request_group"
     },
-    {
-      "_id": "wrk_097d43914a4d4aea8b6f73f647921182",
-      "created": 1552662762769,
-      "description": "",
-      "modified": 1599825617921,
-      "name": "Mantis",
-      "parentId": null,
-      "scope": null,
-      "_type": "workspace"
-    },
     {
       "_id": "req_105cb28c6f774eadbf099a28617a6d1f",
       "authentication": {},
@@ -695,8 +781,14 @@
       "_id": "env_2e1a8d7b5f3649308cb3c524e84a3f33",
       "color": null,
       "created": 1552663140073,
-      "data": { "node_url": "http://127.0.0.1:8546" },
-      "dataPropertyOrder": { "&": ["node_url"] },
+      "data": {
+        "node_url": "http://127.0.0.1:8546"
+      },
+      "dataPropertyOrder": {
+        "&": [
+          "node_url"
+        ]
+      },
       "isPrivate": false,
       "metaSortKey": 1552663140073,
       "modified": 1599825641645,
@@ -705,4 +797,4 @@
       "_type": "environment"
     }
   ]
-}
+}

Diff for: src/main/resources/application.conf

@@ -185,8 +185,8 @@ mantis {
       }
 
       # Enabled JSON-RPC APIs over the JSON-RPC endpoint
-      # Available choices are: eth, web3, net, personal, test, daedalus, iele
-      apis = "eth,web3,net,personal,daedalus,debug"
+      # Available choices are: eth, web3, net, personal, test, daedalus, iele, qa
+      apis = "eth,web3,net,personal,daedalus,debug,qa"
 
       # Maximum number of blocks for daedalus_getAccountTransactions
       account-transactions-max-blocks = 50000
@@ -534,6 +534,25 @@ mantis {
     # Iff true, any errors during metrics client operations will be logged.
     log-errors = true
   }
+
+  async {
+    dispatchers {
+      block-forger {
+        type = Dispatcher
+        executor = "fork-join-executor"
+
+        fork-join-executor {
+          parallelism-min = 2
+          parallelism-factor = 2.0
+          parallelism-max = 8
+
+          task-peeking-mode = "FIFO"
+        }
+
+        throughput = 5
+      }
+    }
+  }
 }
 
 akka {

Diff for: src/main/scala/io/iohk/ethereum/consensus/Consensus.scala

@@ -1,10 +1,12 @@
 package io.iohk.ethereum.consensus
 
 import io.iohk.ethereum.consensus.blocks.{BlockGenerator, TestBlockGenerator}
+import io.iohk.ethereum.consensus.ethash.{MinerProtocol, MinerResponse}
 import io.iohk.ethereum.consensus.validators.Validators
 import io.iohk.ethereum.ledger.BlockPreparator
 import io.iohk.ethereum.ledger.Ledger.VMImpl
 import io.iohk.ethereum.nodebuilder.Node
+import scala.concurrent.Future
 
 /**
  * Abstraction for a consensus protocol implementation.
@@ -53,6 +55,11 @@ trait Consensus {
    * This is called internally when the node terminates.
    */
   def stopProtocol(): Unit
+
+  /**
+    * Sends msg to the internal miner
+    */
+  def sendMiner(msg: MinerProtocol): Future[MinerResponse]
 }
 
 /**

Diff for: src/main/scala/io/iohk/ethereum/consensus/ConsensusBuilder.scala

@@ -38,7 +38,7 @@ trait StdConsensusBuilder extends ConsensusBuilder {
 
     val consensus =
       config.protocol match {
-        case Protocol.Ethash ⇒ buildEthashConsensus()
+        case Protocol.Ethash | Protocol.MockedPow ⇒ buildEthashConsensus()
       }
     log.info(s"Using '${protocol.name}' consensus [${consensus.getClass.getName}]")
 

Diff for: src/main/scala/io/iohk/ethereum/consensus/Protocol.scala

@@ -17,17 +17,22 @@ object Protocol {
   object Names {
     // This is the standard Ethereum PoW consensus protocol.
     final val Ethash = "ethash"
+
+    final val MockedPow = "mocked"
   }
 
   sealed abstract class ProtocolImpl(val name: String) extends Protocol
 
   /** The standard Ethereum PoW consensus protocol. */
   case object Ethash extends ProtocolImpl(Names.Ethash)
 
+  /** Mocked pow consensus algorithm used for tests etc. */
+  case object MockedPow extends ProtocolImpl(Names.MockedPow)
+
 
   /** All the known protocols. If a protocol is not put here, then it cannot be used to run Mantis. */
   final val KnownProtocols = Set(
-    Ethash
+    Ethash, MockedPow
   )
 
   final val KnownProtocolNames = KnownProtocols.map(_.name)