[ETCM-143] Redefine block header structure and encoding

src/ets/scala/io/iohk/ethereum/ets/blockchain/Scenario.scala

@@ -55,8 +55,8 @@ case class BlockHeaderDef(
 ) {
 
   def toBlockHeader: BlockHeader =
-    BlockHeader(parentHash, uncleHash, coinbase, stateRoot, transactionsTrie, receiptTrie, bloom, difficulty, number,
-      gasLimit, gasUsed, timestamp, extraData, mixHash, nonce, None
+    BlockHeader(parentHash, uncleHash, coinbase, stateRoot, transactionsTrie, receiptTrie, bloom,
+      difficulty, number, gasLimit, gasUsed, timestamp, extraData, mixHash, nonce
     )
 }
 

src/ets/scala/io/iohk/ethereum/ets/vm/ScenarioBuilder.scala

@@ -51,8 +51,7 @@ object ScenarioBuilder {
       env.currentTimestamp,
       bEmpty,
       bEmpty,
-      bEmpty,
-      None
+      bEmpty
     )
 
   def prepareWorld(accounts: Map[Address, AccountState], blockNumber: BigInt, exec: Exec): MockWorldState = {

src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainActor.scala

@@ -7,7 +7,7 @@ import akka.actor.{Actor, ActorRef, _}
 import akka.util.ByteString
 import io.iohk.ethereum.crypto.kec256
 import io.iohk.ethereum.domain.{BlockBody, BlockHeader, Receipt}
-import io.iohk.ethereum.domain.BlockHeader._
+import io.iohk.ethereum.domain.BlockHeaderImplicits._
 import io.iohk.ethereum.network.{Peer, PeerManagerActor}
 import io.iohk.ethereum.network.PeerActor.SendMessage
 import io.iohk.ethereum.network.PeerManagerActor.{GetPeers, Peers}

src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala

@@ -26,6 +26,7 @@ import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate
 import org.bouncycastle.util.encoders.Hex
 
 import scala.concurrent.duration._
+import io.iohk.ethereum.domain.BlockHeader.HeaderExtraFields.HefEmpty
 
 object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder with AuthHandshakerBuilder {
     val conf = ConfigFactory.load("txExecTest/chainDump.conf")
@@ -103,7 +104,7 @@ object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder wit
   class BlockchainMock(genesisHash: ByteString) extends Blockchain {
 
     class FakeHeader() extends BlockHeader(ByteString.empty, ByteString.empty, ByteString.empty, ByteString.empty,
-      ByteString.empty, ByteString.empty, ByteString.empty, 0, 0, 0, 0, 0, ByteString.empty, ByteString.empty, ByteString.empty, None) {
+      ByteString.empty, ByteString.empty, ByteString.empty, 0, 0, 0, 0, 0, ByteString.empty, ByteString.empty, ByteString.empty, HefEmpty) {
       override lazy val hash: ByteString = genesisHash
     }
 

src/it/scala/io/iohk/ethereum/txExecTest/util/FixtureProvider.scala

@@ -5,7 +5,7 @@ import java.io.Closeable
 import akka.util.ByteString
 import io.iohk.ethereum.db.storage._
 import io.iohk.ethereum.domain._
-import io.iohk.ethereum.domain.BlockHeader._
+import io.iohk.ethereum.domain.BlockHeaderImplicits._
 import io.iohk.ethereum.domain.BlockBody._
 import io.iohk.ethereum.network.p2p.messages.PV63._
 import MptNodeEncoders._

src/main/scala/io/iohk/ethereum/blockchain/data/GenesisDataLoader.scala

@@ -134,8 +134,7 @@ class GenesisDataLoader(
       unixTimestamp = BigInt(genesisData.timestamp.replace("0x", ""), 16).toLong,
       extraData = genesisData.extraData,
       mixHash = genesisData.mixHash.getOrElse(zeros(hashLength)),
-      nonce = genesisData.nonce,
-      treasuryOptOut = None
+      nonce = genesisData.nonce
     )
 
   private def zeros(length: Int) =

src/main/scala/io/iohk/ethereum/consensus/blocks/BlockGeneratorSkeleton.scala

@@ -11,6 +11,7 @@ import io.iohk.ethereum.crypto.kec256
 import io.iohk.ethereum.db.dataSource.EphemDataSource
 import io.iohk.ethereum.db.storage.StateStorage
 import io.iohk.ethereum.domain._
+import io.iohk.ethereum.domain.BlockHeader.HeaderExtraFields._
 import io.iohk.ethereum.consensus.ethash.blocks.OmmersSeqEnc
 import io.iohk.ethereum.ledger.Ledger.{BlockPreparationResult, BlockResult}
 import io.iohk.ethereum.ledger.{BlockPreparator, BloomFilter}
@@ -51,7 +52,11 @@ abstract class BlockGeneratorSkeleton(
     blockTimestamp: Long,
     x: Ommers
   ): BlockHeader = {
-    val optOut = if(blockNumber >= blockchainConfig.ecip1098BlockNumber) Some(consensusConfig.treasuryOptOut) else None
+    val extraFields =
+      if(blockNumber >= blockchainConfig.ecip1098BlockNumber)
+        HefPostEcip1098(consensusConfig.treasuryOptOut)
+      else
+        HefEmpty
 
     BlockHeader(
       parentHash = parent.header.hash,
@@ -70,7 +75,7 @@ abstract class BlockGeneratorSkeleton(
       extraData = blockchainConfig.daoForkConfig.flatMap(daoForkConfig => daoForkConfig.getExtraData(blockNumber)).getOrElse(headerExtraData),
       mixHash = ByteString.empty,
       nonce = ByteString.empty,
-      treasuryOptOut = optOut
+      extraFields = extraFields
     )
   }
 

src/main/scala/io/iohk/ethereum/consensus/ethash/blocks/package.scala

@@ -2,6 +2,7 @@ package io.iohk.ethereum.consensus.ethash
 
 import io.iohk.ethereum.consensus.ethash.validators.OmmersValidator.OmmersError
 import io.iohk.ethereum.domain.BlockHeader
+import io.iohk.ethereum.domain.BlockHeaderImplicits._
 import io.iohk.ethereum.ledger.BlockPreparationError
 import io.iohk.ethereum.rlp.{RLPEncodeable, RLPList, RLPSerializable}
 

src/main/scala/io/iohk/ethereum/consensus/validators/BlockHeaderValidator.scala

@@ -3,6 +3,7 @@ package validators
 
 import io.iohk.ethereum.crypto.ECDSASignature
 import io.iohk.ethereum.domain.BlockHeader
+import io.iohk.ethereum.domain.BlockHeader.HeaderExtraFields
 
 /**
  * Validates a [[io.iohk.ethereum.domain.BlockHeader BlockHeader]].
@@ -33,13 +34,13 @@ object BlockHeaderError {
   case object HeaderGasLimitError extends BlockHeaderError
   case object HeaderNumberError extends BlockHeaderError
   case object HeaderPoWError extends BlockHeaderError
-  case class HeaderOptOutError(ecip1098Activated: Boolean, optOutDefined: Boolean) extends BlockHeaderError
+  case class HeaderExtraFieldsError(extraFields: HeaderExtraFields, ecip1097Activated: Boolean, ecip1098Activated: Boolean) extends BlockHeaderError
   case class HeaderWrongNumberOfCheckpointSignatures(sigCount: Int) extends BlockHeaderError
   case class HeaderInvalidCheckpointSignatures(invalidSignaturesWithPublics: Seq[(ECDSASignature, Option[String])])
     extends BlockHeaderError
-  case object HeaderCheckpointTooEarly extends BlockHeaderError
   case class HeaderFieldNotEmptyError(msg: String) extends BlockHeaderError
   case class HeaderNotMatchParentError(msg: String) extends BlockHeaderError
+  case object CheckpointHeaderTreasuryOptOutError extends BlockHeaderError
 
   case class HeaderUnexpectedError(msg: String) extends BlockHeaderError
 }

src/main/scala/io/iohk/ethereum/consensus/validators/BlockHeaderValidatorSkeleton.scala

@@ -4,6 +4,7 @@ import io.iohk.ethereum.consensus.GetBlockHeaderByHash
 import io.iohk.ethereum.consensus.difficulty.DifficultyCalculator
 import io.iohk.ethereum.consensus.validators.BlockHeaderError._
 import io.iohk.ethereum.domain.BlockHeader
+import io.iohk.ethereum.domain.BlockHeader.HeaderExtraFields.{HefEmpty, HefPostEcip1097, HefPostEcip1098}
 import io.iohk.ethereum.utils.{BlockchainConfig, DaoForkConfig}
 
 /**
@@ -77,7 +78,7 @@ abstract class BlockHeaderValidatorSkeleton(blockchainConfig: BlockchainConfig)
       _ <- validateGasUsed(blockHeader)
       _ <- validateGasLimit(blockHeader, parentHeader)
       _ <- validateNumber(blockHeader, parentHeader)
-      _ <- validateOptOut(blockHeader)
+      _ <- validateExtraFields(blockHeader)
       _ <- validateEvenMore(blockHeader, parentHeader)
     } yield BlockHeaderValid
   }
@@ -95,6 +96,7 @@ abstract class BlockHeaderValidatorSkeleton(blockchainConfig: BlockchainConfig)
     for {
       _ <- blockWithCheckpointHeaderValidator.validate(blockHeader, parentHeader)
       _ <- validateNumber(blockHeader, parentHeader)
+      _ <- validateExtraFields(blockHeader)
     } yield BlockHeaderValid
   }
 
@@ -197,18 +199,23 @@ abstract class BlockHeaderValidatorSkeleton(blockchainConfig: BlockchainConfig)
     else Left(HeaderNumberError)
 
   /**
-    * Validates [[io.iohk.ethereum.domain.BlockHeader.treasuryOptOut]] is only defined if ECIP1098 is enabled at the block's number
+    * Validates [[io.iohk.ethereum.domain.BlockHeader.extraFields]] match the ECIP1097 and ECIP1098 enabling configuration
     *
     * @param blockHeader BlockHeader to validate.
-    * @return BlockHeader if valid, an [[HeaderOptOutError]] otherwise
+    * @return BlockHeader if valid, an [[HeaderExtraFieldsError]] otherwise
     */
-  private def validateOptOut(blockHeader: BlockHeader): Either[BlockHeaderError, BlockHeaderValid] = {
-    val isEcip1098Activated = blockHeader.number >= blockchainConfig.ecip1098BlockNumber
-    val isOptOutDefined = blockHeader.treasuryOptOut.isDefined
-
-    if (isEcip1098Activated && isOptOutDefined) Right(BlockHeaderValid)
-    else if (!isEcip1098Activated && !isOptOutDefined) Right(BlockHeaderValid)
-    else Left(HeaderOptOutError(isEcip1098Activated, isOptOutDefined))
+  private def validateExtraFields(blockHeader: BlockHeader): Either[BlockHeaderError, BlockHeaderValid] = {
+    val isECIP1098Activated = blockHeader.number >= blockchainConfig.ecip1098BlockNumber
+    val isECIP1097Activated = blockHeader.number >= blockchainConfig.ecip1097BlockNumber
+
+    blockHeader.extraFields match {
+      case HefPostEcip1097(_, _) if isECIP1097Activated && isECIP1098Activated => Right(BlockHeaderValid)
+      case HefPostEcip1098(_) if !isECIP1097Activated && isECIP1098Activated => Right(BlockHeaderValid)
+      case HefEmpty if !isECIP1097Activated && !isECIP1098Activated => Right(BlockHeaderValid)
+      case _ =>
+        val error = HeaderExtraFieldsError(blockHeader.extraFields, isECIP1097Activated, isECIP1098Activated)
+        Left(error)
+    }
   }
 
 }

src/main/scala/io/iohk/ethereum/consensus/validators/BlockWithCheckpointHeaderValidator.scala

@@ -13,12 +13,12 @@ class BlockWithCheckpointHeaderValidator(blockchainConfig: BlockchainConfig) {
 
   def validate(blockHeader: BlockHeader, parentHeader: BlockHeader): Either[BlockHeaderError, BlockHeaderValid] = {
     for {
-      _ <- validateECIP1097Number(blockHeader)
       _ <- validateCheckpointSignatures(blockHeader, parentHeader)
       _ <- validateEmptyFields(blockHeader)
       _ <- validateFieldsCopiedFromParent(blockHeader, parentHeader)
       _ <- validateGasUsed(blockHeader)
       _ <- validateTimestamp(blockHeader, parentHeader)
+      _ <- validateTreasuryOptOut(blockHeader)
     } yield BlockHeaderValid
   }
 
@@ -95,8 +95,6 @@ class BlockWithCheckpointHeaderValidator(blockchainConfig: BlockchainConfig) {
       notEmptyFieldError("logsBloom")
     else if (blockHeader.extraData.nonEmpty)
       notEmptyFieldError("extraData")
-    else if (blockHeader.treasuryOptOut.isDefined)
-      notEmptyFieldError("treasuryOptOut")
     else if (blockHeader.nonce.nonEmpty)
       notEmptyFieldError("nonce")
     else if (blockHeader.mixHash.nonEmpty)
@@ -155,14 +153,8 @@ class BlockWithCheckpointHeaderValidator(blockchainConfig: BlockchainConfig) {
     if (blockHeader.unixTimestamp == parentHeader.unixTimestamp + 1) Right(BlockHeaderValid)
     else Left(HeaderTimestampError)
 
-  /**
-    * Validates [[io.iohk.ethereum.domain.BlockHeader.checkpoint]] is only defined if ECIP1097 is enabled at the block's number
-    *
-    * @param blockHeader BlockHeader to validate.
-    * @return BlockHeader if valid, an [[HeaderCheckpointTooEarly]] otherwise
-    */
-  private def validateECIP1097Number(blockHeader: BlockHeader): Either[BlockHeaderError, BlockHeaderValid] = {
-    if (blockHeader.number >= blockchainConfig.ecip1097BlockNumber) Right(BlockHeaderValid)
-    else Left(HeaderCheckpointTooEarly)
-  }
+  private def validateTreasuryOptOut(blockHeader: BlockHeader): Either[BlockHeaderError, BlockHeaderValid] =
+    if (blockHeader.treasuryOptOut.contains(false)) Right(BlockHeaderValid)
+    else Left(CheckpointHeaderTreasuryOptOutError)
+
 }

src/main/scala/io/iohk/ethereum/db/storage/BlockBodiesStorage.scala

@@ -4,11 +4,11 @@ import java.nio.ByteBuffer
 
 import akka.util.ByteString
 import boopickle.Default.{Pickle, Unpickle}
-import io.iohk.ethereum.crypto.ECDSASignature
 import io.iohk.ethereum.db.dataSource.DataSource
 import io.iohk.ethereum.db.storage.BlockBodiesStorage.BlockBodyHash
-import io.iohk.ethereum.domain.{Address, BlockBody, BlockHeader, Checkpoint, SignedTransaction, Transaction}
+import io.iohk.ethereum.domain.BlockBody
 import io.iohk.ethereum.utils.ByteUtils.compactPickledBytes
+import io.iohk.ethereum.utils.Picklers._
 
 /**
   * This class is used to store the BlockBody, by using:
@@ -30,20 +30,4 @@ class BlockBodiesStorage(val dataSource: DataSource) extends TransactionalKeyVal
 
 object BlockBodiesStorage {
   type BlockBodyHash = ByteString
-
-  import boopickle.DefaultBasic._
-
-  implicit val byteStringPickler: Pickler[ByteString] = transformPickler[ByteString, Array[Byte]](ByteString(_))(_.toArray[Byte])
-  implicit val addressPickler: Pickler[Address] =
-    transformPickler[Address, ByteString](bytes => Address(bytes))(address => address.bytes)
-  implicit val transactionPickler: Pickler[Transaction] = generatePickler[Transaction]
-  implicit val ecdsaSignaturePickler: Pickler[ECDSASignature] = generatePickler[ECDSASignature]
-  implicit val checkpointPickler: Pickler[Checkpoint] = generatePickler[Checkpoint]
-  implicit val signedTransactionPickler: Pickler[SignedTransaction] = transformPickler[SignedTransaction, (Transaction, ECDSASignature)]
-  { case (tx, signature) => new SignedTransaction(tx, signature) }{ stx => (stx.tx, stx.signature)}
-
-  implicit val blockHeaderPickler: Pickler[BlockHeader] = generatePickler[BlockHeader]
-  implicit val blockBodyPickler: Pickler[BlockBody] = transformPickler[BlockBody, (Seq[SignedTransaction], Seq[BlockHeader])]
-  {case (stx, nodes) => BlockBody(stx, nodes) }{ blockBody => (blockBody.transactionList, blockBody.uncleNodesList) }
-
 }

src/main/scala/io/iohk/ethereum/db/storage/BlockHeadersStorage.scala

@@ -4,11 +4,12 @@ import java.nio.ByteBuffer
 
 import akka.util.ByteString
 import boopickle.Default.{Pickle, Unpickle}
-import io.iohk.ethereum.crypto.ECDSASignature
+import boopickle.DefaultBasic._
 import io.iohk.ethereum.db.dataSource.DataSource
 import io.iohk.ethereum.db.storage.BlockHeadersStorage.BlockHeaderHash
-import io.iohk.ethereum.domain.{BlockHeader, Checkpoint}
+import io.iohk.ethereum.domain.BlockHeader
 import io.iohk.ethereum.utils.ByteUtils.compactPickledBytes
+import io.iohk.ethereum.utils.Picklers._
 
 /**
   * This class is used to store the BlockHeader, by using:
@@ -32,11 +33,4 @@ class BlockHeadersStorage(val dataSource: DataSource) extends TransactionalKeyVa
 
 object BlockHeadersStorage {
   type BlockHeaderHash = ByteString
-
-  import boopickle.DefaultBasic._
-
-  implicit val byteStringPickler: Pickler[ByteString] = transformPickler[ByteString, Array[Byte]](ByteString(_))(_.toArray[Byte])
-  implicit val ecdsaSignaturePickler: Pickler[ECDSASignature] = generatePickler[ECDSASignature]
-  implicit val checkpointPickler: Pickler[Checkpoint] = generatePickler[Checkpoint]
-  implicit val blockHeaderPickler: Pickler[BlockHeader] = generatePickler[BlockHeader]
 }

src/main/scala/io/iohk/ethereum/domain/Block.scala

@@ -1,7 +1,7 @@
 package io.iohk.ethereum.domain
 
 import akka.util.ByteString
-import io.iohk.ethereum.domain.BlockHeader._
+import io.iohk.ethereum.domain.BlockHeaderImplicits._
 import io.iohk.ethereum.rlp.{RLPEncodeable, RLPList, RLPSerializable, rawDecode}
 
 /**

src/main/scala/io/iohk/ethereum/domain/BlockBody.scala

@@ -1,6 +1,6 @@
 package io.iohk.ethereum.domain
 
-import io.iohk.ethereum.domain.BlockHeader._
+import io.iohk.ethereum.domain.BlockHeaderImplicits._
 import io.iohk.ethereum.rlp.{RLPEncodeable, RLPList, RLPSerializable, rawDecode}
 
 case class BlockBody(transactionList: Seq[SignedTransaction], uncleNodesList: Seq[BlockHeader]) {