diff --git a/Cargo.lock b/Cargo.lock
index c79806e732..bf7b0c295c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -461,6 +461,8 @@ dependencies = [
"primitives",
"rand 0.6.1",
"rlp",
+ "rlp_derive",
+ "snap",
"tempfile",
"trie-standardmap",
]
diff --git a/codechain/codechain.yml b/codechain/codechain.yml
index 0558880d58..ff0964615c 100644
--- a/codechain/codechain.yml
+++ b/codechain/codechain.yml
@@ -260,6 +260,16 @@ args:
takes_value: true
conflicts_with:
- no-discovery
+ - snapshot-hash:
+ long: snapshot-hash
+ value_name: HASH
+ requires: snapshot-number
+ takes_value: true
+ - snapshot-number:
+ long: snapshot-number
+ value_name: NUM
+ requires: snapshot-hash
+ takes_value: true
- no-snapshot:
long: no-snapshot
help: Disable snapshots
diff --git a/codechain/config/mod.rs b/codechain/config/mod.rs
index e4ea5538d6..e5dac23f3b 100644
--- a/codechain/config/mod.rs
+++ b/codechain/config/mod.rs
@@ -25,6 +25,7 @@ use cidr::IpCidr;
use ckey::PlatformAddress;
use clap;
use cnetwork::{FilterEntry, NetworkConfig, SocketAddr};
+use primitives::H256;
use toml;
pub use self::chain_type::ChainType;
@@ -274,6 +275,8 @@ pub struct Network {
pub min_peers: Option<usize>,
pub max_peers: Option<usize>,
pub sync: Option<bool>,
+ pub snapshot_hash: Option<H256>,
+ pub snapshot_number: Option<u64>,
pub transaction_relay: Option<bool>,
pub discovery: Option<bool>,
pub discovery_type: Option<String>,
@@ -313,6 +316,8 @@ fn default_enable_devel_api() -> bool {
pub struct Snapshot {
pub disable: Option<bool>,
pub path: Option<String>,
+ // Snapshot's age in blocks
+ pub expiration: Option<u64>,
}
#[derive(Deserialize)]
@@ -575,6 +580,12 @@ impl Network {
if other.sync.is_some() {
self.sync = other.sync;
}
+ if other.snapshot_hash.is_some() {
+ self.snapshot_hash = other.snapshot_hash;
+ }
+ if other.snapshot_number.is_some() {
+ self.snapshot_number = other.snapshot_number;
+ }
if other.transaction_relay.is_some() {
self.transaction_relay = other.transaction_relay;
}
@@ -627,6 +638,12 @@ impl Network {
if matches.is_present("no-sync") {
self.sync = Some(false);
}
+ if let Some(snapshot_hash) = matches.value_of("snapshot-hash") {
+ self.snapshot_hash = Some(snapshot_hash.parse().map_err(|_| "Invalid snapshot-hash")?);
+ }
+ if let Some(snapshot_number) = matches.value_of("snapshot-number") {
+ self.snapshot_number = Some(snapshot_number.parse().map_err(|_| "Invalid snapshot-number")?);
+ }
if matches.is_present("no-tx-relay") {
self.transaction_relay = Some(false);
}
@@ -739,6 +756,9 @@ impl Snapshot {
if other.path.is_some() {
self.path = other.path.clone();
}
+ if other.expiration.is_some() {
+ self.expiration = other.expiration;
+ }
}
pub fn overwrite_with(&mut self, matches: &clap::ArgMatches) -> Result<(), String> {
diff --git a/codechain/config/presets/config.dev.toml b/codechain/config/presets/config.dev.toml
index 2b8890b4f7..279704592a 100644
--- a/codechain/config/presets/config.dev.toml
+++ b/codechain/config/presets/config.dev.toml
@@ -52,6 +52,7 @@ max_connections = 100
[snapshot]
disable = false
path = "snapshot"
+expiration = 100000 # blocks. About a week
[stratum]
disable = false
diff --git a/codechain/config/presets/config.prod.toml b/codechain/config/presets/config.prod.toml
index b67e2746bb..013883673d 100644
--- a/codechain/config/presets/config.prod.toml
+++ b/codechain/config/presets/config.prod.toml
@@ -52,6 +52,7 @@ max_connections = 100
[snapshot]
disable = true
path = "snapshot"
+expiration = 100000 # blocks. About a week
[stratum]
disable = true
diff --git a/codechain/rpc.rs b/codechain/rpc.rs
index 5a3f696ca4..b3219e25d8 100644
--- a/codechain/rpc.rs
+++ b/codechain/rpc.rs
@@ -15,8 +15,8 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::io;
-use std::net::SocketAddr;
+use crate::config::Config;
use crate::rpc_apis;
use crpc::{
jsonrpc_core, start_http, start_ipc, start_ws, HttpServer, IpcServer, MetaIoHandler, Middleware, WsError, WsServer,
@@ -33,38 +33,27 @@ pub struct RpcHttpConfig {
}
pub fn rpc_http_start(
- cfg: RpcHttpConfig,
- enable_devel_api: bool,
- deps: &rpc_apis::ApiDependencies,
+ server: MetaIoHandler<(), impl Middleware<()>>,
+ config: RpcHttpConfig,
) -> Result<HttpServer, String> {
- let url = format!("{}:{}", cfg.interface, cfg.port);
+ let url = format!("{}:{}", config.interface, config.port);
let addr = url.parse().map_err(|_| format!("Invalid JSONRPC listen host/port given: {}", url))?;
- let server = setup_http_rpc_server(&addr, cfg.cors.clone(), cfg.hosts.clone(), enable_devel_api, deps)?;
- cinfo!(RPC, "RPC Listening on {}", url);
- if let Some(hosts) = cfg.hosts {
- cinfo!(RPC, "Allowed hosts are {:?}", hosts);
- }
- if let Some(cors) = cfg.cors {
- cinfo!(RPC, "CORS domains are {:?}", cors);
- }
- Ok(server)
-}
-
-fn setup_http_rpc_server(
- url: &SocketAddr,
- cors_domains: Option<Vec<String>>,
- allowed_hosts: Option<Vec<String>>,
- enable_devel_api: bool,
- deps: &rpc_apis::ApiDependencies,
-) -> Result<HttpServer, String> {
- let server = setup_rpc_server(enable_devel_api, deps);
- let start_result = start_http(url, cors_domains, allowed_hosts, server);
+ let start_result = start_http(&addr, config.cors.clone(), config.hosts.clone(), server);
match start_result {
Err(ref err) if err.kind() == io::ErrorKind::AddrInUse => {
Err(format!("RPC address {} is already in use, make sure that another instance of a CodeChain node is not running or change the address using the --jsonrpc-port option.", url))
},
Err(e) => Err(format!("RPC error: {:?}", e)),
- Ok(server) => Ok(server),
+ Ok(server) => {
+ cinfo!(RPC, "RPC Listening on {}", url);
+ if let Some(hosts) = config.hosts {
+ cinfo!(RPC, "Allowed hosts are {:?}", hosts);
+ }
+ if let Some(cors) = config.cors {
+ cinfo!(RPC, "CORS domains are {:?}", cors);
+ }
+ Ok(server)
+ },
}
}
@@ -74,19 +63,17 @@ pub struct RpcIpcConfig {
}
pub fn rpc_ipc_start(
- cfg: &RpcIpcConfig,
- enable_devel_api: bool,
- deps: &rpc_apis::ApiDependencies,
+ server: MetaIoHandler<(), impl Middleware<()>>,
+ config: RpcIpcConfig,
) -> Result<IpcServer, String> {
- let server = setup_rpc_server(enable_devel_api, deps);
- let start_result = start_ipc(&cfg.socket_addr, server);
+ let start_result = start_ipc(&config.socket_addr, server);
match start_result {
Err(ref err) if err.kind() == io::ErrorKind::AddrInUse => {
- Err(format!("IPC address {} is already in use, make sure that another instance of a Codechain node is not running or change the address using the --ipc-path options.", cfg.socket_addr))
+ Err(format!("IPC address {} is already in use, make sure that another instance of a Codechain node is not running or change the address using the --ipc-path options.", config.socket_addr))
},
Err(e) => Err(format!("IPC error: {:?}", e)),
Ok(server) => {
- cinfo!(RPC, "IPC Listening on {}", cfg.socket_addr);
+ cinfo!(RPC, "IPC Listening on {}", config.socket_addr);
Ok(server)
},
}
@@ -99,15 +86,10 @@ pub struct RpcWsConfig {
pub max_connections: usize,
}
-pub fn rpc_ws_start(
- cfg: &RpcWsConfig,
- enable_devel_api: bool,
- deps: &rpc_apis::ApiDependencies,
-) -> Result<WsServer, String> {
- let server = setup_rpc_server(enable_devel_api, deps);
- let url = format!("{}:{}", cfg.interface, cfg.port);
+pub fn rpc_ws_start(server: MetaIoHandler<(), impl Middleware<()>>, config: RpcWsConfig) -> Result<WsServer, String> {
+ let url = format!("{}:{}", config.interface, config.port);
let addr = url.parse().map_err(|_| format!("Invalid WebSockets listen host/port given: {}", url))?;
- let start_result = start_ws(&addr, server, cfg.max_connections);
+ let start_result = start_ws(&addr, server, config.max_connections);
match start_result {
Err(WsError::Io(ref err)) if err.kind() == io::ErrorKind::AddrInUse => {
Err(format!("WebSockets address {} is already in use, make sure that another instance of a Codechain node is not running or change the address using the --ws-port options.", addr))
@@ -120,12 +102,9 @@ pub fn rpc_ws_start(
}
}
-fn setup_rpc_server(
- enable_devel_api: bool,
- deps: &rpc_apis::ApiDependencies,
-) -> MetaIoHandler<(), impl Middleware<()>> {
+pub fn setup_rpc_server(config: &Config, deps: &rpc_apis::ApiDependencies) -> MetaIoHandler<(), impl Middleware<()>> {
let mut handler = MetaIoHandler::with_middleware(LogMiddleware::new());
- deps.extend_api(enable_devel_api, &mut handler);
+ deps.extend_api(config, &mut handler);
rpc_apis::setup_rpc(handler)
}
diff --git a/codechain/rpc_apis.rs b/codechain/rpc_apis.rs
index 3f4d3c016a..bc701275df 100644
--- a/codechain/rpc_apis.rs
+++ b/codechain/rpc_apis.rs
@@ -22,6 +22,8 @@ use cnetwork::{EventSender, NetworkControl};
use crpc::{MetaIoHandler, Middleware, Params, Value};
use csync::BlockSyncEvent;
+use crate::config::Config;
+
pub struct ApiDependencies {
pub client: Arc<Client>,
pub miner: Arc<Miner>,
@@ -31,11 +33,12 @@ pub struct ApiDependencies {
}
impl ApiDependencies {
- pub fn extend_api(&self, enable_devel_api: bool, handler: &mut MetaIoHandler<(), impl Middleware<()>>) {
+ pub fn extend_api(&self, config: &Config, handler: &mut MetaIoHandler<(), impl Middleware<()>>) {
use crpc::v1::*;
handler.extend_with(ChainClient::new(Arc::clone(&self.client)).to_delegate());
handler.extend_with(MempoolClient::new(Arc::clone(&self.client)).to_delegate());
- if enable_devel_api {
+ handler.extend_with(SnapshotClient::new(Arc::clone(&self.client), config.snapshot.path.clone()).to_delegate());
+ if config.rpc.enable_devel_api {
handler.extend_with(
DevelClient::new(Arc::clone(&self.client), Arc::clone(&self.miner), self.block_sync.clone())
.to_delegate(),
diff --git a/codechain/run_node.rs b/codechain/run_node.rs
index 9ec3646293..bf04996b99 100644
--- a/codechain/run_node.rs
+++ b/codechain/run_node.rs
@@ -19,6 +19,7 @@ use std::path::Path;
use std::sync::{Arc, Weak};
use std::time::{SystemTime, UNIX_EPOCH};
+use ccore::snapshot_notify;
use ccore::{
AccountProvider, AccountProviderError, BlockId, ChainNotify, Client, ClientConfig, ClientService, EngineInfo,
EngineType, Miner, MinerService, Scheme, Stratum, StratumConfig, StratumError, NUM_COLUMNS,
@@ -30,9 +31,11 @@ use ckeystore::KeyStore;
use clap::ArgMatches;
use clogger::{self, EmailAlarm, LoggerConfig};
use cnetwork::{Filters, NetworkConfig, NetworkControl, NetworkService, RoutingTable, SocketAddr};
-use csync::{BlockSyncExtension, BlockSyncSender, SnapshotService, TransactionSyncExtension};
+use csync::snapshot::Service as SnapshotService;
+use csync::{BlockSyncExtension, BlockSyncSender, TransactionSyncExtension};
use ctimer::TimerLoop;
use ctrlc::CtrlC;
+use ctypes::BlockHash;
use fdlimit::raise_fd_limit;
use kvdb::KeyValueDB;
use kvdb_rocksdb::{Database, DatabaseConfig};
@@ -42,7 +45,7 @@ use crate::config::{self, load_config};
use crate::constants::{DEFAULT_DB_PATH, DEFAULT_KEYS_PATH};
use crate::dummy_network_service::DummyNetworkService;
use crate::json::PasswordFile;
-use crate::rpc::{rpc_http_start, rpc_ipc_start, rpc_ws_start};
+use crate::rpc::{rpc_http_start, rpc_ipc_start, rpc_ws_start, setup_rpc_server};
use crate::rpc_apis::ApiDependencies;
fn network_start(
@@ -283,7 +286,7 @@ pub fn run_node(matches: &ArgMatches) -> Result<(), String> {
let network_config = config.network_config()?;
// XXX: What should we do if the network id has been changed.
let c = client.client();
- let network_id = c.common_params(BlockId::Latest).unwrap().network_id();
+ let network_id = c.common_params(BlockId::Number(0)).unwrap().network_id();
let routing_table = RoutingTable::new();
let service = network_start(network_id, timer_loop, &network_config, Arc::clone(&routing_table))?;
@@ -296,7 +299,14 @@ pub fn run_node(matches: &ArgMatches) -> Result<(), String> {
if config.network.sync.unwrap() {
let sync_sender = {
let client = client.client();
- service.register_extension(move |api| BlockSyncExtension::new(client, api))
+ let snapshot_target = match (config.network.snapshot_hash, config.network.snapshot_number) {
+ (Some(hash), Some(num)) => Some((BlockHash::from(hash), num)),
+ _ => None,
+ };
+ let snapshot_dir = config.snapshot.path.clone();
+ service.register_extension(move |api| {
+ BlockSyncExtension::new(client, api, snapshot_target, snapshot_dir)
+ })
};
let sync = Arc::new(BlockSyncSender::from(sync_sender.clone()));
client.client().add_notify(Arc::downgrade(&sync) as Weak<dyn ChainNotify>);
@@ -316,36 +326,43 @@ pub fn run_node(matches: &ArgMatches) -> Result<(), String> {
}
};
- let rpc_apis_deps = ApiDependencies {
- client: client.client(),
- miner: Arc::clone(&miner),
- network_control: Arc::clone(&network_service),
- account_provider: ap,
- block_sync: maybe_sync_sender,
- };
+ let (rpc_server, ipc_server, ws_server) = {
+ let rpc_apis_deps = ApiDependencies {
+ client: client.client(),
+ miner: Arc::clone(&miner),
+ network_control: Arc::clone(&network_service),
+ account_provider: ap,
+ block_sync: maybe_sync_sender,
+ };
+
+ let rpc_server = {
+ if !config.rpc.disable.unwrap() {
+ let server = setup_rpc_server(&config, &rpc_apis_deps);
+ Some(rpc_http_start(server, config.rpc_http_config())?)
+ } else {
+ None
+ }
+ };
- let rpc_server = {
- if !config.rpc.disable.unwrap() {
- Some(rpc_http_start(config.rpc_http_config(), config.rpc.enable_devel_api, &rpc_apis_deps)?)
- } else {
- None
- }
- };
+ let ipc_server = {
+ if !config.ipc.disable.unwrap() {
+ let server = setup_rpc_server(&config, &rpc_apis_deps);
+ Some(rpc_ipc_start(server, config.rpc_ipc_config())?)
+ } else {
+ None
+ }
+ };
- let ipc_server = {
- if !config.ipc.disable.unwrap() {
- Some(rpc_ipc_start(&config.rpc_ipc_config(), config.rpc.enable_devel_api, &rpc_apis_deps)?)
- } else {
- None
- }
- };
+ let ws_server = {
+ if !config.ws.disable.unwrap() {
+ let server = setup_rpc_server(&config, &rpc_apis_deps);
+ Some(rpc_ws_start(server, config.rpc_ws_config())?)
+ } else {
+ None
+ }
+ };
- let ws_server = {
- if !config.ws.disable.unwrap() {
- Some(rpc_ws_start(&config.rpc_ws_config(), config.rpc.enable_devel_api, &rpc_apis_deps)?)
- } else {
- None
- }
+ (rpc_server, ipc_server, ws_server)
};
if (!config.stratum.disable.unwrap()) && (miner.engine_type() == EngineType::PoW) {
@@ -353,12 +370,13 @@ pub fn run_node(matches: &ArgMatches) -> Result<(), String> {
}
let _snapshot_service = {
+ let client = client.client();
+ let (tx, rx) = snapshot_notify::create();
+ client.engine().register_snapshot_notify_sender(tx);
+
if !config.snapshot.disable.unwrap() {
- // FIXME: Let's make it load snapshot period dynamically to support changing the period.
- let client = client.client();
- let snapshot_period = client.common_params(BlockId::Latest).unwrap().snapshot_period();
- let service = SnapshotService::new(Arc::clone(&client), config.snapshot.path.unwrap(), snapshot_period);
- client.add_notify(Arc::downgrade(&service) as Weak<dyn ChainNotify>);
+ let service =
+ Arc::new(SnapshotService::new(client, rx, config.snapshot.path.unwrap(), config.snapshot.expiration));
Some(service)
} else {
None
@@ -367,6 +385,7 @@ pub fn run_node(matches: &ArgMatches) -> Result<(), String> {
// drop the scheme to free up genesis state.
drop(scheme);
+ client.client().engine().register_is_done();
cinfo!(TEST_SCRIPT, "Initialization complete");
diff --git a/core/src/block.rs b/core/src/block.rs
index 103d5903a7..ab35efe3cf 100644
--- a/core/src/block.rs
+++ b/core/src/block.rs
@@ -33,6 +33,7 @@ use crate::client::{EngineInfo, TermInfo};
use crate::consensus::CodeChainEngine;
use crate::error::{BlockError, Error};
use crate::transaction::{SignedTransaction, UnverifiedTransaction};
+use crate::BlockId;
/// A block, encoded as it is on the block chain.
#[derive(Debug, Clone, PartialEq)]
@@ -220,25 +221,20 @@ impl<'x> OpenBlock<'x> {
pub fn close(
mut self,
parent_header: &Header,
- parent_common_params: &CommonParams,
term_common_params: Option<&CommonParams>,
) -> Result<ClosedBlock, Error> {
let unclosed_state = self.block.state.clone();
- if let Err(e) =
- self.engine.on_close_block(&mut self.block, parent_header, parent_common_params, term_common_params)
- {
+ if let Err(e) = self.engine.on_close_block(&mut self.block, term_common_params) {
warn!("Encountered error on closing the block: {}", e);
return Err(e)
}
let header = self.block.header().clone();
for handler in self.engine.action_handlers() {
- handler.on_close_block(self.block.state_mut(), &header, parent_header, parent_common_params).map_err(
- |e| {
- warn!("Encountered error in {}::on_close_block", handler.name());
- e
- },
- )?;
+ handler.on_close_block(self.block.state_mut(), &header).map_err(|e| {
+ warn!("Encountered error in {}::on_close_block", handler.name());
+ e
+ })?;
}
let state_root = self.block.state.commit().map_err(|e| {
@@ -262,23 +258,18 @@ impl<'x> OpenBlock<'x> {
pub fn close_and_lock(
mut self,
parent_header: &Header,
- parent_common_params: &CommonParams,
term_common_params: Option<&CommonParams>,
) -> Result<LockedBlock, Error> {
- if let Err(e) =
- self.engine.on_close_block(&mut self.block, parent_header, parent_common_params, term_common_params)
- {
+ if let Err(e) = self.engine.on_close_block(&mut self.block, term_common_params) {
warn!("Encountered error on closing the block: {}", e);
return Err(e)
}
let header = self.block.header().clone();
for handler in self.engine.action_handlers() {
- handler.on_close_block(self.block.state_mut(), &header, parent_header, parent_common_params).map_err(
- |e| {
- warn!("Encountered error in {}::on_close_block", handler.name());
- e
- },
- )?;
+ handler.on_close_block(self.block.state_mut(), &header).map_err(|e| {
+ warn!("Encountered error in {}::on_close_block", handler.name());
+ e
+ })?;
}
let state_root = self.block.state.commit().map_err(|e| {
@@ -322,6 +313,10 @@ impl<'x> OpenBlock<'x> {
self.block.header.set_seal(seal);
Ok(())
}
+
+ pub fn inner_mut(&mut self) -> &mut ExecutedBlock {
+ &mut self.block
+ }
}
/// Just like `OpenBlock`, except that we've applied `Engine::on_close_block`, finished up the non-seal header fields.
@@ -501,20 +496,11 @@ pub fn enact<C: ChainTimeInfo + EngineInfo + FindActionHandler + TermInfo>(
let mut b = OpenBlock::try_new(engine, db, parent, Address::default(), vec![])?;
b.populate_from(header);
+ engine.on_open_block(b.inner_mut())?;
b.push_transactions(transactions, client, parent.number(), parent.timestamp())?;
- let parent_common_params = client.common_params((*header.parent_hash()).into()).unwrap();
- let term_common_params = {
- let block_number = client
- .last_term_finished_block_num((*header.parent_hash()).into())
- .expect("The block of the parent hash should exist");
- if block_number == 0 {
- None
- } else {
- Some(client.common_params((block_number).into()).expect("Common params should exist"))
- }
- };
- b.close_and_lock(parent, &parent_common_params, term_common_params.as_ref())
+ let term_common_params = client.term_common_params(BlockId::Hash(*header.parent_hash()));
+ b.close_and_lock(parent, term_common_params.as_ref())
}
#[cfg(test)]
@@ -532,9 +518,8 @@ mod tests {
let genesis_header = scheme.genesis_header();
let db = scheme.ensure_genesis_state(get_temp_state_db()).unwrap();
let b = OpenBlock::try_new(&*scheme.engine, db, &genesis_header, Address::default(), vec![]).unwrap();
- let parent_common_params = CommonParams::default_for_test();
let term_common_params = CommonParams::default_for_test();
- let b = b.close_and_lock(&genesis_header, &parent_common_params, Some(&term_common_params)).unwrap();
+ let b = b.close_and_lock(&genesis_header, Some(&term_common_params)).unwrap();
let _ = b.seal(&*scheme.engine, vec![]);
}
}
diff --git a/core/src/blockchain/blockchain.rs b/core/src/blockchain/blockchain.rs
index 0ed70027f2..bbf028afe3 100644
--- a/core/src/blockchain/blockchain.rs
+++ b/core/src/blockchain/blockchain.rs
@@ -110,6 +110,45 @@ impl BlockChain {
}
}
+ pub fn insert_floating_header(&self, batch: &mut DBTransaction, header: &HeaderView) {
+ self.headerchain.insert_floating_header(batch, header);
+ }
+
+ pub fn insert_floating_block(&self, batch: &mut DBTransaction, bytes: &[u8]) {
+ let block = BlockView::new(bytes);
+ let header = block.header_view();
+ let hash = header.hash();
+
+ ctrace!(BLOCKCHAIN, "Inserting bootstrap block #{}({}) to the blockchain.", header.number(), hash);
+
+ if self.is_known(&hash) {
+ cdebug!(BLOCKCHAIN, "Block #{}({}) is already known.", header.number(), hash);
+ return
+ }
+
+ self.insert_floating_header(batch, &header);
+ self.body_db.insert_body(batch, &block);
+ }
+
+ pub fn force_update_best_block(&self, batch: &mut DBTransaction, hash: &BlockHash) {
+ ctrace!(BLOCKCHAIN, "Forcefully updating the best block to {}", hash);
+
+ assert!(self.is_known(hash));
+ assert!(self.pending_best_block_hash.read().is_none());
+ assert!(self.pending_best_proposal_block_hash.read().is_none());
+
+ let block = self.block(hash).expect("Target block is known");
+ self.headerchain.force_update_best_header(batch, hash);
+ self.body_db.update_best_block(batch, &BestBlockChanged::CanonChainAppended {
+ best_block: block.into_inner(),
+ });
+
+ batch.put(db::COL_EXTRA, BEST_BLOCK_KEY, hash);
+ *self.pending_best_block_hash.write() = Some(*hash);
+ batch.put(db::COL_EXTRA, BEST_PROPOSAL_BLOCK_KEY, hash);
+ *self.pending_best_proposal_block_hash.write() = Some(*hash);
+ }
+
/// Inserts the block into backing cache database.
/// Expects the block to be valid and already verified.
/// If the block is already known, does nothing.
diff --git a/core/src/blockchain/headerchain.rs b/core/src/blockchain/headerchain.rs
index 10700264a5..3aa3d3ec74 100644
--- a/core/src/blockchain/headerchain.rs
+++ b/core/src/blockchain/headerchain.rs
@@ -115,6 +115,50 @@ impl HeaderChain {
}
}
+ /// Inserts a floating header into backing cache database.
+ /// Expects the header to be valid.
+ /// If the header is already known, does nothing.
+ pub fn insert_floating_header(&self, batch: &mut DBTransaction, header: &HeaderView) {
+ let hash = header.hash();
+
+ ctrace!(HEADERCHAIN, "Inserting a floating block header #{}({}) to the headerchain.", header.number(), hash);
+
+ if self.is_known_header(&hash) {
+ ctrace!(HEADERCHAIN, "Block header #{}({}) is already known.", header.number(), hash);
+ return
+ }
+
+ let compressed_header = compress(header.rlp().as_raw(), blocks_swapper());
+ batch.put(db::COL_HEADERS, &hash, &compressed_header);
+
+ let mut new_hashes = HashMap::new();
+ new_hashes.insert(header.number(), hash);
+ let mut new_details = HashMap::new();
+ new_details.insert(hash, BlockDetails {
+ number: header.number(),
+ total_score: 0.into(),
+ parent: header.parent_hash(),
+ });
+
+ let mut pending_hashes = self.pending_hashes.write();
+ let mut pending_details = self.pending_details.write();
+
+ batch.extend_with_cache(db::COL_EXTRA, &mut *pending_details, new_details, CacheUpdatePolicy::Overwrite);
+ batch.extend_with_cache(db::COL_EXTRA, &mut *pending_hashes, new_hashes, CacheUpdatePolicy::Overwrite);
+ }
+
+ pub fn force_update_best_header(&self, batch: &mut DBTransaction, hash: &BlockHash) {
+ ctrace!(HEADERCHAIN, "Forcefully updating the best header to {}", hash);
+ assert!(self.is_known_header(hash));
+ assert!(self.pending_best_header_hash.read().is_none());
+ assert!(self.pending_best_proposal_block_hash.read().is_none());
+
+ batch.put(db::COL_EXTRA, BEST_HEADER_KEY, hash);
+ *self.pending_best_header_hash.write() = Some(*hash);
+ batch.put(db::COL_EXTRA, BEST_PROPOSAL_HEADER_KEY, hash);
+ *self.pending_best_proposal_block_hash.write() = Some(*hash);
+ }
+
/// Inserts the header into backing cache database.
/// Expects the header to be valid and already verified.
/// If the header is already known, does nothing.
diff --git a/core/src/client/client.rs b/core/src/client/client.rs
index d0617431c1..22c6192684 100644
--- a/core/src/client/client.rs
+++ b/core/src/client/client.rs
@@ -26,6 +26,7 @@ use cstate::{
ActionHandler, AssetScheme, FindActionHandler, OwnedAsset, StateDB, StateResult, Text, TopLevelState, TopStateView,
};
use ctimer::{TimeoutHandler, TimerApi, TimerScheduleError, TimerToken};
+use ctypes::header::Header;
use ctypes::transaction::{AssetTransferInput, PartialHashing, ShardTransaction};
use ctypes::{BlockHash, BlockNumber, CommonParams, ShardId, Tracker, TxHash};
use cvm::{decode, execute, ChainTimeInfo, ScriptResult, VMConfig};
@@ -42,9 +43,9 @@ use super::{
ClientConfig, DatabaseClient, EngineClient, EngineInfo, Error as ClientError, ExecuteClient, ImportBlock,
ImportResult, MiningBlockChainClient, Shard, StateInfo, StateOrBlock, TextClient,
};
-use crate::block::{ClosedBlock, IsBlock, OpenBlock, SealedBlock};
+use crate::block::{Block, ClosedBlock, IsBlock, OpenBlock, SealedBlock};
use crate::blockchain::{BlockChain, BlockProvider, BodyProvider, HeaderProvider, InvoiceProvider, TransactionAddress};
-use crate::client::{ConsensusClient, TermInfo};
+use crate::client::{ConsensusClient, SnapshotClient, TermInfo};
use crate::consensus::{CodeChainEngine, EngineError};
use crate::encoded;
use crate::error::{BlockImportError, Error, ImportError, SchemeError};
@@ -655,6 +656,28 @@ impl ImportBlock for Client {
Ok(self.importer.header_queue.import(unverified)?)
}
+ fn import_trusted_header(&self, header: &Header) -> Result<BlockHash, BlockImportError> {
+ if self.block_chain().is_known_header(&header.hash()) {
+ return Err(BlockImportError::Import(ImportError::AlreadyInChain))
+ }
+ let import_lock = self.importer.import_lock.lock();
+ self.importer.import_trusted_header(header, self, &import_lock);
+ Ok(header.hash())
+ }
+
+ fn import_trusted_block(&self, block: &Block) -> Result<BlockHash, BlockImportError> {
+ if self.block_chain().is_known(&block.header.hash()) {
+ return Err(BlockImportError::Import(ImportError::AlreadyInChain))
+ }
+ let import_lock = self.importer.import_lock.lock();
+ self.importer.import_trusted_block(block, self, &import_lock);
+ Ok(block.header.hash())
+ }
+
+ fn force_update_best_block(&self, hash: &BlockHash) {
+ self.importer.force_update_best_block(hash, self)
+ }
+
fn import_sealed_block(&self, block: &SealedBlock) -> ImportResult {
let h = block.header().hash();
let route = {
@@ -813,6 +836,23 @@ impl TermInfo for Client {
.map(|state| state.metadata().unwrap().expect("Metadata always exist"))
.map(|metadata| metadata.current_term_id())
}
+
+ fn term_common_params(&self, id: BlockId) -> Option<CommonParams> {
+ let state = self.state_at(id)?;
+ let metadata = state.metadata().unwrap().expect("Metadata always exist");
+
+ if let Some(term_params) = metadata.term_params() {
+ Some(*term_params)
+ } else {
+ let block_number =
+ self.last_term_finished_block_num(id).expect("The block of the parent hash should exist");
+ if block_number == 0 {
+ None
+ } else {
+ Some(self.common_params((block_number).into()).expect("Common params should exist"))
+ }
+ }
+ }
}
impl AccountData for Client {
@@ -926,3 +966,11 @@ impl FindActionHandler for Client {
self.engine.find_action_handler_for(id)
}
}
+
+impl SnapshotClient for Client {
+ fn notify_snapshot(&self, id: BlockId) {
+ if let Some(header) = self.block_header(&id) {
+ self.engine.send_snapshot_notify(header.hash())
+ }
+ }
+}
diff --git a/core/src/client/importer.rs b/core/src/client/importer.rs
index 8400fc7c36..4290ee7ab9 100644
--- a/core/src/client/importer.rs
+++ b/core/src/client/importer.rs
@@ -19,14 +19,14 @@ use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use cio::IoChannel;
-use ctypes::header::Header;
+use ctypes::header::{Header, Seal};
use ctypes::BlockHash;
use kvdb::DBTransaction;
use parking_lot::{Mutex, MutexGuard};
use rlp::Encodable;
use super::{BlockChainTrait, Client, ClientConfig};
-use crate::block::{enact, IsBlock, LockedBlock};
+use crate::block::{enact, Block, IsBlock, LockedBlock};
use crate::blockchain::{BodyProvider, HeaderProvider, ImportRoute};
use crate::client::EngineInfo;
use crate::consensus::CodeChainEngine;
@@ -100,7 +100,7 @@ impl Importer {
}
{
- let headers: Vec<&Header> = blocks.iter().map(|block| &block.header).collect();
+ let headers: Vec<_> = blocks.iter().map(|block| &block.header).collect();
self.import_headers(headers, client, &import_lock);
}
@@ -362,6 +362,51 @@ impl Importer {
imported.len()
}
+ pub fn import_trusted_header<'a>(&'a self, header: &'a Header, client: &Client, _importer_lock: &MutexGuard<()>) {
+ let hash = header.hash();
+ ctrace!(CLIENT, "Importing trusted header #{}-{:?}", header.number(), hash);
+
+ {
+ let chain = client.block_chain();
+ let mut batch = DBTransaction::new();
+ chain.insert_floating_header(&mut batch, &HeaderView::new(&header.rlp_bytes()));
+ client.db().write_buffered(batch);
+ chain.commit();
+ }
+ client.new_headers(&[hash], &[], &[], &[], &[], None);
+
+ client.db().flush().expect("DB flush failed.");
+ }
+
+ pub fn import_trusted_block<'a>(&'a self, block: &'a Block, client: &Client, importer_lock: &MutexGuard<()>) {
+ let header = &block.header;
+ let hash = header.hash();
+ ctrace!(CLIENT, "Importing trusted block #{}-{:?}", header.number(), hash);
+
+ self.import_trusted_header(header, client, importer_lock);
+ {
+ let chain = client.block_chain();
+ let mut batch = DBTransaction::new();
+ chain.insert_floating_block(&mut batch, &block.rlp_bytes(&Seal::With));
+ client.db().write_buffered(batch);
+ chain.commit();
+ }
+ self.miner.chain_new_blocks(client, &[hash], &[], &[], &[]);
+ client.new_blocks(&[hash], &[], &[], &[], &[]);
+
+ client.db().flush().expect("DB flush failed.");
+ }
+
+ pub fn force_update_best_block(&self, hash: &BlockHash, client: &Client) {
+ let chain = client.block_chain();
+ let mut batch = DBTransaction::new();
+ chain.force_update_best_block(&mut batch, hash);
+ client.db().write_buffered(batch);
+ chain.commit();
+
+ client.db().flush().expect("DB flush failed.");
+ }
+
fn check_header(&self, header: &Header, parent: &Header) -> bool {
// FIXME: self.verifier.verify_block_family
if let Err(e) = self.engine.verify_block_family(&header, &parent) {
diff --git a/core/src/client/mod.rs b/core/src/client/mod.rs
index eb3adaf3b3..02c71223f4 100644
--- a/core/src/client/mod.rs
+++ b/core/src/client/mod.rs
@@ -20,6 +20,7 @@ mod client;
mod config;
mod error;
mod importer;
+pub mod snapshot_notify;
mod test_client;
pub use self::chain_notify::ChainNotify;
@@ -36,13 +37,14 @@ use ckey::{Address, NetworkId, PlatformAddress, Public};
use cmerkle::Result as TrieResult;
use cnetwork::NodeId;
use cstate::{AssetScheme, FindActionHandler, OwnedAsset, StateResult, Text, TopLevelState, TopStateView};
+use ctypes::header::Header;
use ctypes::transaction::{AssetTransferInput, PartialHashing, ShardTransaction};
use ctypes::{BlockHash, BlockNumber, CommonParams, ShardId, Tracker, TxHash};
use cvm::ChainTimeInfo;
use kvdb::KeyValueDB;
use primitives::{Bytes, H160, H256, U256};
-use crate::block::{ClosedBlock, OpenBlock, SealedBlock};
+use crate::block::{Block, ClosedBlock, OpenBlock, SealedBlock};
use crate::blockchain_info::BlockChainInfo;
use crate::consensus::EngineError;
use crate::encoded;
@@ -119,6 +121,7 @@ pub trait ConsensusClient: BlockChainClient + EngineClient + EngineInfo + TermIn
pub trait TermInfo {
fn last_term_finished_block_num(&self, id: BlockId) -> Option<BlockNumber>;
fn current_term_id(&self, id: BlockId) -> Option<u64>;
+ fn term_common_params(&self, id: BlockId) -> Option<CommonParams>;
}
/// Provides methods to access account info
@@ -200,6 +203,17 @@ pub trait ImportBlock {
/// Import a header into the blockchain
fn import_header(&self, bytes: Bytes) -> Result<BlockHash, BlockImportError>;
+ /// Import a trusted header into the blockchain
+ /// Trusted header doesn't go through any verifications and doesn't update the best header
+ fn import_trusted_header(&self, header: &Header) -> Result<BlockHash, BlockImportError>;
+
+ /// Import a trusted block into the blockchain
+ /// Trusted block doesn't go through any verifications and doesn't update the best block
+ fn import_trusted_block(&self, block: &Block) -> Result<BlockHash, BlockImportError>;
+
+ /// Forcefully update the best block
+ fn force_update_best_block(&self, hash: &BlockHash);
+
/// Import sealed block. Skips all verifications.
fn import_sealed_block(&self, block: &SealedBlock) -> ImportResult;
@@ -342,3 +356,7 @@ pub trait StateInfo {
/// is unknown.
fn state_at(&self, id: BlockId) -> Option<TopLevelState>;
}
+
+pub trait SnapshotClient {
+ fn notify_snapshot(&self, id: BlockId);
+}
diff --git a/core/src/client/snapshot_notify.rs b/core/src/client/snapshot_notify.rs
new file mode 100644
index 0000000000..8e0a372cbb
--- /dev/null
+++ b/core/src/client/snapshot_notify.rs
@@ -0,0 +1,81 @@
+use ctypes::BlockHash;
+
+use parking_lot::RwLock;
+use std::sync::mpsc::{sync_channel, Receiver, RecvError, SyncSender};
+use std::sync::{Arc, Weak};
+
+pub fn create() -> (NotifySender, NotifyReceiverSource) {
+ let (tx, rx) = sync_channel(1);
+ let tx = Arc::new(RwLock::new(Some(tx)));
+ let tx_weak = Arc::downgrade(&tx);
+ (
+ NotifySender {
+ tx,
+ },
+ NotifyReceiverSource(
+ ReceiverCanceller {
+ tx: tx_weak,
+ },
+ NotifyReceiver {
+ rx,
+ },
+ ),
+ )
+}
+
+pub struct NotifySender {
+ tx: Arc<RwLock<Option<SyncSender<BlockHash>>>>,
+}
+
+impl NotifySender {
+ pub fn notify(&self, block_hash: BlockHash) {
+ let guard = self.tx.read();
+ if let Some(tx) = guard.as_ref() {
+ // TODO: Ignore the error. Receiver thread might be terminated or congested.
+ let _ = tx.try_send(block_hash);
+ } else {
+ // ReceiverCanceller is dropped.
+ }
+ }
+}
+
+pub struct NotifyReceiverSource(pub ReceiverCanceller, pub NotifyReceiver);
+
+/// Dropping this makes the receiver stopped.
+///
+/// `recv()` method of the `Receiver` will stop and return `RecvError` when corresponding `Sender` is dropped.
+/// This is an inherited behaviour of `std::sync::mpsc::{Sender, Receiver}`.
+/// However, we need another way to stop the `Receiver`, since `Sender` is usually shared throughout our codes.
+/// We can't collect them all and destory one by one. We need a kill switch.
+///
+/// `ReceiverCanceller` holds weak reference to the `Sender`, so it doesn't prohibit the default behaviour.
+/// Then, we can upgrade the weak reference and get the shared reference to `Sender` itself, and manually drop it with this.
+pub struct ReceiverCanceller {
+ tx: Weak<RwLock<Option<SyncSender<BlockHash>>>>,
+}
+
+impl Drop for ReceiverCanceller {
+ fn drop(&mut self) {
+ if let Some(tx) = self.tx.upgrade() {
+ let mut guard = tx.write();
+ if let Some(sender) = guard.take() {
+ drop(sender)
+ }
+ } else {
+ // All NotifySender is dropped. No droppable Sender.
+ }
+ }
+}
+
+/// Receiver is dropped when
+/// 1. There are no NotifySenders out there.
+/// 2. ReceiverCanceller is dropped. See the comment of `ReceiverCanceller`.
+pub struct NotifyReceiver {
+ rx: Receiver<BlockHash>,
+}
+
+impl NotifyReceiver {
+ pub fn recv(&self) -> Result<BlockHash, RecvError> {
+ self.rx.recv()
+ }
+}
diff --git a/core/src/client/test_client.rs b/core/src/client/test_client.rs
index 0a279626e7..b2b0514778 100644
--- a/core/src/client/test_client.rs
+++ b/core/src/client/test_client.rs
@@ -42,6 +42,7 @@ use cnetwork::NodeId;
use cstate::tests::helpers::empty_top_state;
use cstate::{FindActionHandler, StateDB, TopLevelState};
use ctimer::{TimeoutHandler, TimerToken};
+use ctypes::header::Header;
use ctypes::transaction::{Action, Transaction};
use ctypes::{BlockHash, BlockNumber, CommonParams, Header as BlockHeader, Tracker, TxHash};
use cvm::ChainTimeInfo;
@@ -52,7 +53,7 @@ use parking_lot::RwLock;
use primitives::{Bytes, H256, U256};
use rlp::*;
-use crate::block::{ClosedBlock, OpenBlock, SealedBlock};
+use crate::block::{Block, ClosedBlock, OpenBlock, SealedBlock};
use crate::blockchain_info::BlockChainInfo;
use crate::client::{
AccountData, BlockChainClient, BlockChainTrait, BlockProducer, BlockStatus, ConsensusClient, EngineInfo,
@@ -509,6 +510,18 @@ impl ImportBlock for TestBlockChainClient {
unimplemented!()
}
+ fn import_trusted_header(&self, _header: &Header) -> Result<BlockHash, BlockImportError> {
+ unimplemented!()
+ }
+
+ fn import_trusted_block(&self, _block: &Block) -> Result<BlockHash, BlockImportError> {
+ unimplemented!()
+ }
+
+ fn force_update_best_block(&self, _hash: &BlockHash) {
+ unimplemented!()
+ }
+
fn import_sealed_block(&self, _block: &SealedBlock) -> ImportResult {
Ok(H256::default().into())
}
@@ -645,7 +658,7 @@ impl super::EngineClient for TestBlockChainClient {
impl EngineInfo for TestBlockChainClient {
fn common_params(&self, _block_id: BlockId) -> Option<CommonParams> {
- unimplemented!()
+ Some(*self.scheme.engine.machine().genesis_common_params())
}
fn metadata_seq(&self, _block_id: BlockId) -> Option<u64> {
@@ -679,6 +692,10 @@ impl TermInfo for TestBlockChainClient {
fn current_term_id(&self, _id: BlockId) -> Option<u64> {
self.term_id
}
+
+ fn term_common_params(&self, _id: BlockId) -> Option<CommonParams> {
+ None
+ }
}
impl StateInfo for TestBlockChainClient {
diff --git a/core/src/consensus/blake_pow/mod.rs b/core/src/consensus/blake_pow/mod.rs
index 8e0b20a6e3..010fa35ae6 100644
--- a/core/src/consensus/blake_pow/mod.rs
+++ b/core/src/consensus/blake_pow/mod.rs
@@ -163,8 +163,6 @@ impl ConsensusEngine for BlakePoW {
fn on_close_block(
&self,
block: &mut ExecutedBlock,
- _parent_header: &Header,
- _parent_common_params: &CommonParams,
_term_common_params: Option<&CommonParams>,
) -> Result<(), Error> {
let author = *block.header().author();
diff --git a/core/src/consensus/cuckoo/mod.rs b/core/src/consensus/cuckoo/mod.rs
index ad088f2454..d904120d67 100644
--- a/core/src/consensus/cuckoo/mod.rs
+++ b/core/src/consensus/cuckoo/mod.rs
@@ -173,8 +173,6 @@ impl ConsensusEngine for Cuckoo {
fn on_close_block(
&self,
block: &mut ExecutedBlock,
- _parent_header: &Header,
- _parent_common_params: &CommonParams,
_term_common_params: Option<&CommonParams>,
) -> Result<(), Error> {
let author = *block.header().author();
@@ -261,21 +259,13 @@ mod tests {
#[test]
fn on_close_block() {
let scheme = Scheme::new_test_cuckoo();
- let genesis_header = scheme.genesis_header();
let engine = &*scheme.engine;
let db = scheme.ensure_genesis_state(get_temp_state_db()).unwrap();
let header = Header::default();
let block = OpenBlock::try_new(engine, db, &header, Default::default(), vec![]).unwrap();
let mut executed_block = block.block().clone();
- assert!(engine
- .on_close_block(
- &mut executed_block,
- &genesis_header,
- &CommonParams::default_for_test(),
- Some(&CommonParams::default_for_test())
- )
- .is_ok());
+ assert!(engine.on_close_block(&mut executed_block, Some(&CommonParams::default_for_test())).is_ok());
assert_eq!(0xd, engine.machine().balance(&executed_block, header.author()).unwrap());
}
diff --git a/core/src/consensus/mod.rs b/core/src/consensus/mod.rs
index 6841b859d4..843f2f2663 100644
--- a/core/src/consensus/mod.rs
+++ b/core/src/consensus/mod.rs
@@ -51,6 +51,7 @@ use primitives::{Bytes, U256};
use self::bit_set::BitSet;
use crate::account_provider::AccountProvider;
use crate::block::{ExecutedBlock, SealedBlock};
+use crate::client::snapshot_notify::NotifySender as SnapshotNotifySender;
use crate::client::ConsensusClient;
use crate::codechain_machine::CodeChainMachine;
use crate::error::Error;
@@ -221,12 +222,15 @@ pub trait ConsensusEngine: Sync + Send {
/// Stops any services that the may hold the Engine and makes it safe to drop.
fn stop(&self) {}
+ /// Block transformation functions, before the transactions.
+ fn on_open_block(&self, _block: &mut ExecutedBlock) -> Result<(), Error> {
+ Ok(())
+ }
+
/// Block transformation functions, after the transactions.
fn on_close_block(
&self,
_block: &mut ExecutedBlock,
- _parent_header: &Header,
- _parent_common_params: &CommonParams,
_term_common_params: Option<&CommonParams>,
) -> Result<(), Error> {
Ok(())
@@ -262,6 +266,12 @@ pub trait ConsensusEngine: Sync + Send {
fn register_chain_notify(&self, _: &Client) {}
+ fn register_snapshot_notify_sender(&self, _sender: SnapshotNotifySender) {}
+
+ fn register_is_done(&self) {}
+
+ fn send_snapshot_notify(&self, _block_hash: BlockHash) {}
+
fn get_best_block_from_best_proposal_header(&self, header: &HeaderView) -> BlockHash {
header.hash()
}
diff --git a/core/src/consensus/null_engine/mod.rs b/core/src/consensus/null_engine/mod.rs
index 832f6165b7..26f40199d1 100644
--- a/core/src/consensus/null_engine/mod.rs
+++ b/core/src/consensus/null_engine/mod.rs
@@ -17,7 +17,7 @@
mod params;
use ckey::Address;
-use ctypes::{CommonParams, Header};
+use ctypes::CommonParams;
use self::params::NullEngineParams;
use super::ConsensusEngine;
@@ -58,8 +58,6 @@ impl ConsensusEngine for NullEngine {
fn on_close_block(
&self,
block: &mut ExecutedBlock,
- _parent_header: &Header,
- _parent_common_params: &CommonParams,
_term_common_params: Option<&CommonParams>,
) -> Result<(), Error> {
let (author, total_reward) = {
diff --git a/core/src/consensus/simple_poa/mod.rs b/core/src/consensus/simple_poa/mod.rs
index 848af86b70..a6b72120b8 100644
--- a/core/src/consensus/simple_poa/mod.rs
+++ b/core/src/consensus/simple_poa/mod.rs
@@ -121,8 +121,6 @@ impl ConsensusEngine for SimplePoA {
fn on_close_block(
&self,
block: &mut ExecutedBlock,
- _parent_header: &Header,
- _parent_common_params: &CommonParams,
_term_common_params: Option<&CommonParams>,
) -> Result<(), Error> {
let author = *block.header().author();
@@ -186,9 +184,8 @@ mod tests {
let db = scheme.ensure_genesis_state(get_temp_state_db()).unwrap();
let genesis_header = scheme.genesis_header();
let b = OpenBlock::try_new(engine, db, &genesis_header, Default::default(), vec![]).unwrap();
- let parent_common_params = CommonParams::default_for_test();
let term_common_params = CommonParams::default_for_test();
- let b = b.close_and_lock(&genesis_header, &parent_common_params, Some(&term_common_params)).unwrap();
+ let b = b.close_and_lock(&genesis_header, Some(&term_common_params)).unwrap();
if let Some(seal) = engine.generate_seal(Some(b.block()), &genesis_header).seal_fields() {
assert!(b.try_seal(engine, seal).is_ok());
}
diff --git a/core/src/consensus/solo/mod.rs b/core/src/consensus/solo/mod.rs
index 6faf5dc0d5..0c7632e54e 100644
--- a/core/src/consensus/solo/mod.rs
+++ b/core/src/consensus/solo/mod.rs
@@ -16,25 +16,30 @@
mod params;
-use std::sync::Arc;
+use std::sync::{Arc, Weak};
use ckey::Address;
use cstate::{ActionHandler, HitHandler};
-use ctypes::{CommonParams, Header};
+use ctypes::{BlockHash, CommonParams, Header};
+use parking_lot::RwLock;
use self::params::SoloParams;
use super::stake;
use super::{ConsensusEngine, Seal};
use crate::block::{ExecutedBlock, IsBlock};
+use crate::client::snapshot_notify::NotifySender;
+use crate::client::ConsensusClient;
use crate::codechain_machine::CodeChainMachine;
use crate::consensus::{EngineError, EngineType};
use crate::error::Error;
/// A consensus engine which does not provide any consensus mechanism.
pub struct Solo {
+ client: RwLock<Option<Weak<dyn ConsensusClient>>>,
params: SoloParams,
machine: CodeChainMachine,
action_handlers: Vec<Arc<dyn ActionHandler>>,
+ snapshot_notify_sender: Arc<RwLock<Option<NotifySender>>>,
}
impl Solo {
@@ -47,11 +52,17 @@ impl Solo {
action_handlers.push(Arc::new(stake::Stake::new(params.genesis_stakes.clone())));
Solo {
+ client: Default::default(),
params,
machine,
action_handlers,
+ snapshot_notify_sender: Arc::new(RwLock::new(None)),
}
}
+
+ fn client(&self) -> Option<Arc<dyn ConsensusClient>> {
+ self.client.read().as_ref()?.upgrade()
+ }
}
impl ConsensusEngine for Solo {
@@ -78,10 +89,13 @@ impl ConsensusEngine for Solo {
fn on_close_block(
&self,
block: &mut ExecutedBlock,
- parent_header: &Header,
- parent_common_params: &CommonParams,
_term_common_params: Option<&CommonParams>,
) -> Result<(), Error> {
+ let client = self.client().ok_or(EngineError::CannotOpenBlock)?;
+
+ let parent_hash = *block.header().parent_hash();
+ let parent = client.block_header(&parent_hash.into()).expect("Parent header must exist");
+ let parent_common_params = client.common_params(parent_hash.into()).expect("CommonParams of parent must exist");
let author = *block.header().author();
let (total_reward, total_min_fee) = {
let transactions = block.transactions();
@@ -107,18 +121,18 @@ impl ConsensusEngine for Solo {
self.machine.add_balance(block, &author, block_author_reward)?;
return Ok(())
}
- stake::add_intermediate_rewards(block.state_mut(), author, block_author_reward)?;
+ stake::v0::add_intermediate_rewards(block.state_mut(), author, block_author_reward)?;
let last_term_finished_block_num = {
let header = block.header();
let current_term_period = header.timestamp() / term_seconds;
- let parent_term_period = parent_header.timestamp() / term_seconds;
+ let parent_term_period = parent.timestamp() / term_seconds;
if current_term_period == parent_term_period {
return Ok(())
}
header.number()
};
- stake::move_current_to_previous_intermediate_rewards(&mut block.state_mut())?;
- let rewards = stake::drain_previous_rewards(&mut block.state_mut())?;
+ stake::v0::move_current_to_previous_intermediate_rewards(&mut block.state_mut())?;
+ let rewards = stake::v0::drain_previous_rewards(&mut block.state_mut())?;
for (address, reward) in rewards {
self.machine.add_balance(block, &address, reward)?;
}
@@ -127,6 +141,10 @@ impl ConsensusEngine for Solo {
Ok(())
}
+ fn register_client(&self, client: Weak<dyn ConsensusClient>) {
+ *self.client.write() = Some(Weak::clone(&client));
+ }
+
fn block_reward(&self, _block_number: u64) -> u64 {
self.params.block_reward
}
@@ -135,6 +153,18 @@ impl ConsensusEngine for Solo {
1
}
+ fn register_snapshot_notify_sender(&self, sender: NotifySender) {
+ let mut guard = self.snapshot_notify_sender.write();
+ assert!(guard.is_none(), "snapshot_notify_sender is registered twice");
+ *guard = Some(sender);
+ }
+
+ fn send_snapshot_notify(&self, block_hash: BlockHash) {
+ if let Some(sender) = self.snapshot_notify_sender.read().as_ref() {
+ sender.notify(block_hash)
+ }
+ }
+
fn action_handlers(&self) -> &[Arc<dyn ActionHandler>] {
&self.action_handlers
}
@@ -146,25 +176,29 @@ impl ConsensusEngine for Solo {
#[cfg(test)]
mod tests {
+ use std::sync::Arc;
+
use ctypes::{CommonParams, Header};
use primitives::H520;
use crate::block::{IsBlock, OpenBlock};
+ use crate::client::{ConsensusClient, TestBlockChainClient};
use crate::scheme::Scheme;
use crate::tests::helpers::get_temp_state_db;
#[test]
fn seal() {
let scheme = Scheme::new_test_solo();
- let engine = &*scheme.engine;
- let db = scheme.ensure_genesis_state(get_temp_state_db()).unwrap();
- let genesis_header = scheme.genesis_header();
- let b = OpenBlock::try_new(engine, db, &genesis_header, Default::default(), vec![]).unwrap();
- let parent_common_params = CommonParams::default_for_test();
+ let client = Arc::new(TestBlockChainClient::new_with_scheme(scheme));
+ let engine = client.scheme.engine.clone();
+ engine.register_client(Arc::downgrade(&(client.clone() as Arc<dyn ConsensusClient>)));
+ let db = client.scheme.ensure_genesis_state(get_temp_state_db()).unwrap();
+ let genesis_header = client.scheme.genesis_header();
+ let b = OpenBlock::try_new(&*engine, db, &genesis_header, Default::default(), vec![]).unwrap();
let term_common_params = CommonParams::default_for_test();
- let b = b.close_and_lock(&genesis_header, &parent_common_params, Some(&term_common_params)).unwrap();
+ let b = b.close_and_lock(&genesis_header, Some(&term_common_params)).unwrap();
if let Some(seal) = engine.generate_seal(Some(b.block()), &genesis_header).seal_fields() {
- assert!(b.try_seal(engine, seal).is_ok());
+ assert!(b.try_seal(&*engine, seal).is_ok());
}
}
diff --git a/core/src/consensus/stake/action_data.rs b/core/src/consensus/stake/action_data.rs
index acc1add3b3..af14c8b33e 100644
--- a/core/src/consensus/stake/action_data.rs
+++ b/core/src/consensus/stake/action_data.rs
@@ -18,7 +18,6 @@ use std::cmp::Ordering;
use std::collections::btree_map::{BTreeMap, Entry};
use std::collections::btree_set::{self, BTreeSet};
use std::collections::{btree_map, HashMap, HashSet};
-use std::mem;
use std::ops::Deref;
use std::vec;
@@ -408,51 +407,116 @@ impl IntoIterator for Validators {
}
}
-#[derive(Default, Debug, PartialEq)]
-pub struct IntermediateRewards {
- previous: BTreeMap<Address, u64>,
- current: BTreeMap<Address, u64>,
-}
+pub mod v0 {
+ use std::mem;
-impl IntermediateRewards {
- pub fn load_from_state(state: &TopLevelState) -> StateResult<Self> {
- let key = get_intermediate_rewards_key();
- let action_data = state.action_data(&key)?;
- let (previous, current) = decode_map_tuple(action_data.as_ref());
+ use super::*;
- Ok(Self {
- previous,
- current,
- })
+ #[derive(Default, Debug, PartialEq)]
+ pub struct IntermediateRewards {
+ pub(super) previous: BTreeMap<Address, u64>,
+ pub(super) current: BTreeMap<Address, u64>,
}
- pub fn save_to_state(&self, state: &mut TopLevelState) -> StateResult<()> {
- let key = get_intermediate_rewards_key();
- if self.previous.is_empty() && self.current.is_empty() {
- state.remove_action_data(&key);
- } else {
- let encoded = encode_map_tuple(&self.previous, &self.current);
- state.update_action_data(&key, encoded)?;
+ impl IntermediateRewards {
+ pub fn load_from_state(state: &TopLevelState) -> StateResult<Self> {
+ let key = get_intermediate_rewards_key();
+ let action_data = state.action_data(&key)?;
+ let (previous, current) = decode_map_tuple(action_data.as_ref());
+
+ Ok(Self {
+ previous,
+ current,
+ })
}
- Ok(())
- }
- pub fn add_quantity(&mut self, address: Address, quantity: StakeQuantity) {
- if quantity == 0 {
- return
+ pub fn save_to_state(&self, state: &mut TopLevelState) -> StateResult<()> {
+ let key = get_intermediate_rewards_key();
+ if self.previous.is_empty() && self.current.is_empty() {
+ state.remove_action_data(&key);
+ } else {
+ let encoded = encode_map_tuple(&self.previous, &self.current);
+ state.update_action_data(&key, encoded)?;
+ }
+ Ok(())
+ }
+
+ pub fn add_quantity(&mut self, address: Address, quantity: StakeQuantity) {
+ if quantity == 0 {
+ return
+ }
+ *self.current.entry(address).or_insert(0) += quantity;
+ }
+
+ pub fn drain_previous(&mut self) -> BTreeMap<Address, u64> {
+ let mut new = BTreeMap::new();
+ mem::swap(&mut new, &mut self.previous);
+ new
+ }
+
+ pub fn move_current_to_previous(&mut self) {
+ assert!(self.previous.is_empty());
+ mem::swap(&mut self.previous, &mut self.current);
}
- *self.current.entry(address).or_insert(0) += quantity;
}
+}
+
+pub mod v1 {
+ use std::mem;
+
+ use super::*;
- pub fn drain_previous(&mut self) -> BTreeMap<Address, u64> {
- let mut new = BTreeMap::new();
- mem::swap(&mut new, &mut self.previous);
- new
+ #[derive(Default, Debug, PartialEq)]
+ pub struct IntermediateRewards {
+ pub(super) current: BTreeMap<Address, u64>,
+ pub(super) calculated: BTreeMap<Address, u64>,
}
- pub fn move_current_to_previous(&mut self) {
- assert!(self.previous.is_empty());
- mem::swap(&mut self.previous, &mut self.current);
+ impl IntermediateRewards {
+ pub fn load_from_state(state: &TopLevelState) -> StateResult<Self> {
+ let key = get_intermediate_rewards_key();
+ let action_data = state.action_data(&key)?;
+ let (current, calculated) = decode_map_tuple(action_data.as_ref());
+
+ Ok(Self {
+ current,
+ calculated,
+ })
+ }
+
+ pub fn save_to_state(&self, state: &mut TopLevelState) -> StateResult<()> {
+ let key = get_intermediate_rewards_key();
+ if self.current.is_empty() && self.calculated.is_empty() {
+ state.remove_action_data(&key);
+ } else {
+ let encoded = encode_map_tuple(&self.current, &self.calculated);
+ state.update_action_data(&key, encoded)?;
+ }
+ Ok(())
+ }
+
+ pub fn add_quantity(&mut self, address: Address, quantity: StakeQuantity) {
+ if quantity == 0 {
+ return
+ }
+ *self.current.entry(address).or_insert(0) += quantity;
+ }
+
+ pub fn update_calculated(&mut self, rewards: BTreeMap<Address, u64>) {
+ self.calculated = rewards;
+ }
+
+ pub fn drain_current(&mut self) -> BTreeMap<Address, u64> {
+ let mut new = BTreeMap::new();
+ mem::swap(&mut new, &mut self.current);
+ new
+ }
+
+ pub fn drain_calculated(&mut self) -> BTreeMap<Address, u64> {
+ let mut new = BTreeMap::new();
+ mem::swap(&mut new, &mut self.calculated);
+ new
+ }
}
}
@@ -1129,39 +1193,39 @@ mod tests {
}
#[test]
- fn load_and_save_intermediate_rewards() {
+ fn load_and_save_intermediate_rewards_v0() {
let mut state = helpers::get_temp_state();
- let rewards = IntermediateRewards::load_from_state(&state).unwrap();
+ let rewards = v0::IntermediateRewards::load_from_state(&state).unwrap();
rewards.save_to_state(&mut state).unwrap();
}
#[test]
- fn add_quantity() {
+ fn add_quantity_v0() {
let address1 = Address::random();
let address2 = Address::random();
let mut state = helpers::get_temp_state();
- let mut origin_rewards = IntermediateRewards::load_from_state(&state).unwrap();
+ let mut origin_rewards = v0::IntermediateRewards::load_from_state(&state).unwrap();
origin_rewards.add_quantity(address1, 1);
origin_rewards.add_quantity(address2, 2);
origin_rewards.save_to_state(&mut state).unwrap();
- let recovered_rewards = IntermediateRewards::load_from_state(&state).unwrap();
+ let recovered_rewards = v0::IntermediateRewards::load_from_state(&state).unwrap();
assert_eq!(origin_rewards, recovered_rewards);
}
#[test]
- fn drain() {
+ fn drain_v0() {
let address1 = Address::random();
let address2 = Address::random();
let mut state = helpers::get_temp_state();
- let mut origin_rewards = IntermediateRewards::load_from_state(&state).unwrap();
+ let mut origin_rewards = v0::IntermediateRewards::load_from_state(&state).unwrap();
origin_rewards.add_quantity(address1, 1);
origin_rewards.add_quantity(address2, 2);
origin_rewards.save_to_state(&mut state).unwrap();
- let mut recovered_rewards = IntermediateRewards::load_from_state(&state).unwrap();
+ let mut recovered_rewards = v0::IntermediateRewards::load_from_state(&state).unwrap();
assert_eq!(origin_rewards, recovered_rewards);
let _drained = recovered_rewards.drain_previous();
recovered_rewards.save_to_state(&mut state).unwrap();
- let mut final_rewards = IntermediateRewards::load_from_state(&state).unwrap();
+ let mut final_rewards = v0::IntermediateRewards::load_from_state(&state).unwrap();
assert_eq!(BTreeMap::new(), final_rewards.previous);
let current = final_rewards.current.clone();
final_rewards.move_current_to_previous();
@@ -1169,6 +1233,59 @@ mod tests {
assert_eq!(current, final_rewards.previous);
}
+ #[test]
+ fn save_v0_and_load_v1_intermediate_rewards() {
+ let address1 = Address::random();
+ let address2 = Address::random();
+ let mut state = helpers::get_temp_state();
+ let mut origin_rewards = v0::IntermediateRewards::load_from_state(&state).unwrap();
+ origin_rewards.add_quantity(address1, 1);
+ origin_rewards.add_quantity(address2, 2);
+ origin_rewards.save_to_state(&mut state).unwrap();
+ let recovered_rewards = v1::IntermediateRewards::load_from_state(&state).unwrap();
+ assert_eq!(origin_rewards.previous, recovered_rewards.current);
+ assert_eq!(origin_rewards.current, recovered_rewards.calculated);
+ }
+
+ #[test]
+ fn load_and_save_intermediate_rewards_v1() {
+ let mut state = helpers::get_temp_state();
+ let rewards = v1::IntermediateRewards::load_from_state(&state).unwrap();
+ rewards.save_to_state(&mut state).unwrap();
+ }
+
+ #[test]
+ fn add_quantity_v1() {
+ let address1 = Address::random();
+ let address2 = Address::random();
+ let mut state = helpers::get_temp_state();
+ let mut origin_rewards = v1::IntermediateRewards::load_from_state(&state).unwrap();
+ origin_rewards.add_quantity(address1, 1);
+ origin_rewards.add_quantity(address2, 2);
+ origin_rewards.save_to_state(&mut state).unwrap();
+ let recovered_rewards = v1::IntermediateRewards::load_from_state(&state).unwrap();
+ assert_eq!(origin_rewards, recovered_rewards);
+ }
+
+ #[test]
+ fn drain_v1() {
+ let address1 = Address::random();
+ let address2 = Address::random();
+ let mut state = helpers::get_temp_state();
+ let mut origin_rewards = v1::IntermediateRewards::load_from_state(&state).unwrap();
+ origin_rewards.add_quantity(address1, 1);
+ origin_rewards.add_quantity(address2, 2);
+ origin_rewards.save_to_state(&mut state).unwrap();
+ let mut recovered_rewards = v1::IntermediateRewards::load_from_state(&state).unwrap();
+ assert_eq!(origin_rewards, recovered_rewards);
+ recovered_rewards.drain_current();
+ recovered_rewards.save_to_state(&mut state).unwrap();
+ let mut final_rewards = v1::IntermediateRewards::load_from_state(&state).unwrap();
+ assert_eq!(BTreeMap::new(), final_rewards.current);
+ final_rewards.drain_calculated();
+ assert_eq!(BTreeMap::new(), final_rewards.calculated);
+ }
+
#[test]
fn candidates_deposit_add() {
let mut state = helpers::get_temp_state();
diff --git a/core/src/consensus/stake/mod.rs b/core/src/consensus/stake/mod.rs
index cf0a07022a..5134367362 100644
--- a/core/src/consensus/stake/mod.rs
+++ b/core/src/consensus/stake/mod.rs
@@ -34,7 +34,7 @@ use primitives::{Bytes, H256};
use rlp::{Decodable, Rlp};
pub use self::action_data::{Banned, Validator, Validators};
-use self::action_data::{Candidates, Delegation, IntermediateRewards, Jail, ReleaseResult, StakeAccount, Stakeholders};
+use self::action_data::{Candidates, Delegation, Jail, ReleaseResult, StakeAccount, Stakeholders};
pub use self::actions::Action;
pub use self::distribute::fee_distribute;
use super::ValidatorSet;
@@ -156,13 +156,7 @@ impl ActionHandler for Stake {
action.verify(current_params, client, validators)
}
- fn on_close_block(
- &self,
- _state: &mut TopLevelState,
- _header: &Header,
- _parent_header: &Header,
- _parent_common_params: &CommonParams,
- ) -> StateResult<()> {
+ fn on_close_block(&self, _state: &mut TopLevelState, _header: &Header) -> StateResult<()> {
Ok(())
}
}
@@ -327,24 +321,61 @@ pub fn get_validators(state: &TopLevelState) -> StateResult<Validators> {
Validators::load_from_state(state)
}
-pub fn add_intermediate_rewards(state: &mut TopLevelState, address: Address, reward: u64) -> StateResult<()> {
- let mut rewards = IntermediateRewards::load_from_state(state)?;
- rewards.add_quantity(address, reward);
- rewards.save_to_state(state)?;
- Ok(())
-}
+pub mod v0 {
+ use super::action_data::v0::IntermediateRewards;
+ use super::*;
-pub fn drain_previous_rewards(state: &mut TopLevelState) -> StateResult<BTreeMap<Address, u64>> {
- let mut rewards = IntermediateRewards::load_from_state(state)?;
- let drained = rewards.drain_previous();
- rewards.save_to_state(state)?;
- Ok(drained)
+ pub fn add_intermediate_rewards(state: &mut TopLevelState, address: Address, reward: u64) -> StateResult<()> {
+ let mut rewards = IntermediateRewards::load_from_state(state)?;
+ rewards.add_quantity(address, reward);
+ rewards.save_to_state(state)?;
+ Ok(())
+ }
+
+ pub fn drain_previous_rewards(state: &mut TopLevelState) -> StateResult<BTreeMap<Address, u64>> {
+ let mut rewards = IntermediateRewards::load_from_state(state)?;
+ let drained = rewards.drain_previous();
+ rewards.save_to_state(state)?;
+ Ok(drained)
+ }
+
+ pub fn move_current_to_previous_intermediate_rewards(state: &mut TopLevelState) -> StateResult<()> {
+ let mut rewards = IntermediateRewards::load_from_state(state)?;
+ rewards.move_current_to_previous();
+ rewards.save_to_state(state)
+ }
}
-pub fn move_current_to_previous_intermediate_rewards(state: &mut TopLevelState) -> StateResult<()> {
- let mut rewards = IntermediateRewards::load_from_state(state)?;
- rewards.move_current_to_previous();
- rewards.save_to_state(state)
+pub mod v1 {
+ use super::action_data::v1::IntermediateRewards;
+ use super::*;
+
+ pub fn add_intermediate_rewards(state: &mut TopLevelState, address: Address, reward: u64) -> StateResult<()> {
+ let mut rewards = IntermediateRewards::load_from_state(state)?;
+ rewards.add_quantity(address, reward);
+ rewards.save_to_state(state)?;
+ Ok(())
+ }
+
+ pub fn drain_current_rewards(state: &mut TopLevelState) -> StateResult<BTreeMap<Address, u64>> {
+ let mut rewards = IntermediateRewards::load_from_state(state)?;
+ let drained = rewards.drain_current();
+ rewards.save_to_state(state)?;
+ Ok(drained)
+ }
+
+ pub fn update_calculated_rewards(state: &mut TopLevelState, values: HashMap<Address, u64>) -> StateResult<()> {
+ let mut rewards = IntermediateRewards::load_from_state(state)?;
+ rewards.update_calculated(values.into_iter().collect());
+ rewards.save_to_state(state)
+ }
+
+ pub fn drain_calculated_rewards(state: &mut TopLevelState) -> StateResult<BTreeMap<Address, u64>> {
+ let mut rewards = IntermediateRewards::load_from_state(state)?;
+ let drained = rewards.drain_calculated();
+ rewards.save_to_state(state)?;
+ Ok(drained)
+ }
}
pub fn update_validator_weights(state: &mut TopLevelState, block_author: &Address) -> StateResult<()> {
diff --git a/core/src/consensus/tendermint/engine.rs b/core/src/consensus/tendermint/engine.rs
index 3261e087a5..e17670ea11 100644
--- a/core/src/consensus/tendermint/engine.rs
+++ b/core/src/consensus/tendermint/engine.rs
@@ -24,8 +24,8 @@ use std::sync::{Arc, Weak};
use ckey::{public_to_address, Address};
use cnetwork::NetworkService;
use crossbeam_channel as crossbeam;
-use cstate::{ActionHandler, TopStateView};
-use ctypes::{BlockHash, BlockNumber, CommonParams, Header};
+use cstate::{ActionHandler, TopState, TopStateView};
+use ctypes::{BlockHash, CommonParams, Header};
use num_rational::Ratio;
use super::super::stake;
@@ -37,6 +37,7 @@ use super::worker;
use super::{ChainNotify, Tendermint, SEAL_FIELDS};
use crate::account_provider::AccountProvider;
use crate::block::*;
+use crate::client::snapshot_notify::NotifySender as SnapshotNotifySender;
use crate::client::{Client, ConsensusClient};
use crate::codechain_machine::CodeChainMachine;
use crate::consensus::tendermint::params::TimeGapParams;
@@ -134,17 +135,65 @@ impl ConsensusEngine for Tendermint {
fn stop(&self) {}
+ /// Block transformation functions, before the transactions.
+ fn on_open_block(&self, block: &mut ExecutedBlock) -> Result<(), Error> {
+ let client = self.client().ok_or(EngineError::CannotOpenBlock)?;
+
+ let block_number = block.header().number();
+ let metadata = block.state().metadata()?.expect("Metadata must exist");
+ let era = metadata.term_params().map_or(0, |p| p.era());
+ if block_number == metadata.last_term_finished_block_num() + 1 {
+ match era {
+ 0 => {}
+ 1 => {
+ let rewards = stake::v1::drain_current_rewards(block.state_mut())?;
+ let start_of_the_current_term = block_number;
+ let start_of_the_previous_term = {
+ let end_of_the_two_level_previous_term = client
+ .last_term_finished_block_num((metadata.last_term_finished_block_num() - 1).into())
+ .unwrap();
+
+ end_of_the_two_level_previous_term + 1
+ };
+
+ let banned = stake::Banned::load_from_state(block.state())?;
+ let start_of_the_current_term_header =
+ encoded::Header::new(block.header().clone().rlp_bytes().to_vec());
+
+ let pending_rewards = calculate_pending_rewards_of_the_previous_term(
+ &*client,
+ &*self.validators,
+ rewards,
+ start_of_the_current_term,
+ start_of_the_current_term_header,
+ start_of_the_previous_term,
+ &banned,
+ )?;
+
+ stake::v1::update_calculated_rewards(block.state_mut(), pending_rewards)?;
+ }
+ _ => unimplemented!(),
+ }
+ }
+ Ok(())
+ }
+
fn on_close_block(
&self,
block: &mut ExecutedBlock,
- parent_header: &Header,
- parent_common_params: &CommonParams,
term_common_params: Option<&CommonParams>,
) -> Result<(), Error> {
+ let client = self.client().ok_or(EngineError::CannotOpenBlock)?;
+
+ let parent_hash = *block.header().parent_hash();
+ let parent = client.block_header(&parent_hash.into()).expect("Parent header must exist").decode();
+ let parent_common_params = client.common_params(parent_hash.into()).expect("CommonParams of parent must exist");
let author = *block.header().author();
+ let block_number = block.header().number();
+
let (total_reward, total_min_fee) = {
let transactions = block.transactions();
- let block_reward = self.block_reward(block.header().number());
+ let block_reward = self.block_reward(block_number);
let total_min_fee: u64 = transactions.iter().map(|tx| tx.fee).sum();
let min_fee =
transactions.iter().map(|tx| CodeChainMachine::min_cost(&parent_common_params, &tx.action)).sum();
@@ -160,90 +209,102 @@ impl ConsensusEngine for Tendermint {
let block_author_reward = total_reward - total_min_fee + distributor.remaining_fee();
+ let era = term_common_params.map_or(0, |p| p.era());
let metadata = block.state().metadata()?.expect("Metadata must exist");
- if metadata.current_term_id() == 0 {
- self.machine.add_balance(block, &author, block_author_reward)?;
-
- if let Some(block_number) =
- block_number_if_term_changed(block.header(), parent_header, parent_common_params)
- {
- // First term change
- stake::on_term_close(block.state_mut(), block_number, &[])?;
+ let term = metadata.current_term_id();
+ let term_seconds = match term {
+ 0 => parent_common_params.term_seconds(),
+ _ => term_common_params.expect("TermCommonParams should exist").term_seconds(),
+ };
+
+ match term {
+ 0 => {
+ self.machine.add_balance(block, &author, block_author_reward)?;
+ }
+ _ => {
+ stake::update_validator_weights(block.state_mut(), &author)?;
+ match era {
+ 0 => stake::v0::add_intermediate_rewards(block.state_mut(), author, block_author_reward)?,
+ 1 => stake::v1::add_intermediate_rewards(block.state_mut(), author, block_author_reward)?,
+ _ => unimplemented!(),
+ }
}
- return Ok(())
}
- let block_author = *block.header().author();
- stake::update_validator_weights(&mut block.state_mut(), &block_author)?;
-
- stake::add_intermediate_rewards(block.state_mut(), author, block_author_reward)?;
-
- let term_common_params = term_common_params.expect("TermCommonParams should exist");
- let last_term_finished_block_num = if let Some(block_number) =
- block_number_if_term_changed(block.header(), parent_header, term_common_params)
- {
- block_number
- } else {
+ if !is_term_changed(block.header(), &parent, term_seconds) {
return Ok(())
- };
- let rewards = stake::drain_previous_rewards(&mut block.state_mut())?;
-
- let start_of_the_current_term = metadata.last_term_finished_block_num() + 1;
- let client = self
- .client
- .read()
- .as_ref()
- .ok_or(EngineError::CannotOpenBlock)?
- .upgrade()
- .ok_or(EngineError::CannotOpenBlock)?;
-
- let inactive_validators = if metadata.current_term_id() == 1 {
- assert!(rewards.is_empty());
-
- let validators = stake::Validators::load_from_state(block.state())?
- .into_iter()
- .map(|val| public_to_address(val.pubkey()))
- .collect();
- inactive_validators(&*client, start_of_the_current_term, block.header(), validators)
- } else {
- let start_of_the_previous_term = {
- let end_of_the_two_level_previous_term =
- client.last_term_finished_block_num((metadata.last_term_finished_block_num() - 1).into()).unwrap();
-
- end_of_the_two_level_previous_term + 1
- };
-
- let banned = stake::Banned::load_from_state(block.state())?;
- let start_of_the_current_term_header = if block.header().number() == start_of_the_current_term {
- encoded::Header::new(block.header().clone().rlp_bytes().to_vec())
- } else {
- client.block_header(&start_of_the_current_term.into()).unwrap()
- };
-
- let pending_rewards = calculate_pending_rewards_of_the_previous_term(
- &*client,
- &*self.validators,
- rewards,
- start_of_the_current_term,
- start_of_the_current_term_header,
- start_of_the_previous_term,
- &banned,
- )?;
-
- for (address, reward) in pending_rewards {
- self.machine.add_balance(block, &address, reward)?;
- }
+ }
- let validators = stake::Validators::load_from_state(block.state())?
- .into_iter()
- .map(|val| public_to_address(val.pubkey()))
- .collect();
- inactive_validators(&*client, start_of_the_current_term, block.header(), validators)
+ let inactive_validators = match (era, term) {
+ (0, 0) => Vec::new(),
+ (0, _) => {
+ let rewards = stake::v0::drain_previous_rewards(block.state_mut())?;
+ let start_of_the_current_term = metadata.last_term_finished_block_num() + 1;
+
+ if term > 1 {
+ let start_of_the_previous_term = {
+ let end_of_the_two_level_previous_term = client
+ .last_term_finished_block_num((metadata.last_term_finished_block_num() - 1).into())
+ .unwrap();
+
+ end_of_the_two_level_previous_term + 1
+ };
+
+ let banned = stake::Banned::load_from_state(block.state())?;
+ let start_of_the_current_term_header = if block_number == start_of_the_current_term {
+ encoded::Header::new(block.header().clone().rlp_bytes().to_vec())
+ } else {
+ client.block_header(&start_of_the_current_term.into()).unwrap()
+ };
+
+ let pending_rewards = calculate_pending_rewards_of_the_previous_term(
+ &*client,
+ &*self.validators,
+ rewards,
+ start_of_the_current_term,
+ start_of_the_current_term_header,
+ start_of_the_previous_term,
+ &banned,
+ )?;
+
+ for (address, reward) in pending_rewards {
+ self.machine.add_balance(block, &address, reward)?;
+ }
+ }
+
+ stake::v0::move_current_to_previous_intermediate_rewards(block.state_mut())?;
+
+ let validators = stake::Validators::load_from_state(block.state())?
+ .into_iter()
+ .map(|val| public_to_address(val.pubkey()))
+ .collect();
+ inactive_validators(&*client, start_of_the_current_term, block.header(), validators)
+ }
+ (1, _) => {
+ for (address, reward) in stake::v1::drain_calculated_rewards(block.state_mut())? {
+ self.machine.add_balance(block, &address, reward)?;
+ }
+
+ let start_of_the_current_term = metadata.last_term_finished_block_num() + 1;
+ let validators = stake::Validators::load_from_state(block.state())?
+ .into_iter()
+ .map(|val| public_to_address(val.pubkey()))
+ .collect();
+ inactive_validators(&*client, start_of_the_current_term, block.header(), validators)
+ }
+ _ => unimplemented!(),
};
- stake::move_current_to_previous_intermediate_rewards(&mut block.state_mut())?;
- stake::on_term_close(block.state_mut(), last_term_finished_block_num, &inactive_validators)?;
+ stake::on_term_close(block.state_mut(), block_number, &inactive_validators)?;
+ match term {
+ 0 => {}
+ _ => match metadata.params().map_or(0, |p| p.era()) {
+ 0 => {}
+ 1 => block.state_mut().snapshot_term_params()?,
+ _ => unimplemented!("It is not decided how we handle this"),
+ },
+ }
Ok(())
}
@@ -279,12 +340,8 @@ impl ConsensusEngine for Tendermint {
let inner = self.inner.clone();
let extension = service.register_extension(move |api| TendermintExtension::new(inner, timeouts, api));
- let client = Weak::clone(self.client.read().as_ref().unwrap());
+ let client = Arc::downgrade(&self.client().unwrap());
self.extension_initializer.send((extension, client)).unwrap();
-
- let (result, receiver) = crossbeam::bounded(1);
- self.inner.send(worker::Event::Restore(result)).unwrap();
- receiver.recv().unwrap();
}
fn register_time_gap_config_to_worker(&self, time_gap_params: TimeGapParams) {
@@ -303,6 +360,16 @@ impl ConsensusEngine for Tendermint {
client.add_notify(Arc::downgrade(&self.chain_notify) as Weak<dyn ChainNotify>);
}
+ fn register_snapshot_notify_sender(&self, sender: SnapshotNotifySender) {
+ self.snapshot_notify_sender_initializer.send(sender).unwrap();
+ }
+
+ fn register_is_done(&self) {
+ let (result, receiver) = crossbeam::bounded(1);
+ self.inner.send(worker::Event::Restore(result)).unwrap();
+ receiver.recv().unwrap();
+ }
+
fn get_best_block_from_best_proposal_header(&self, header: &HeaderView) -> BlockHash {
header.parent_hash()
}
@@ -323,13 +390,7 @@ impl ConsensusEngine for Tendermint {
}
fn possible_authors(&self, block_number: Option<u64>) -> Result<Option<Vec<Address>>, EngineError> {
- let client = self
- .client
- .read()
- .as_ref()
- .ok_or(EngineError::CannotOpenBlock)?
- .upgrade()
- .ok_or(EngineError::CannotOpenBlock)?;
+ let client = self.client().ok_or(EngineError::CannotOpenBlock)?;
let block_hash = match block_number {
None => {
client.block_header(&BlockId::Latest).expect("latest block must exist").hash() // the latest block
@@ -343,22 +404,15 @@ impl ConsensusEngine for Tendermint {
}
}
-fn block_number_if_term_changed(
- header: &Header,
- parent_header: &Header,
- common_params: &CommonParams,
-) -> Option<BlockNumber> {
- let term_seconds = common_params.term_seconds();
+pub(crate) fn is_term_changed(header: &Header, parent: &Header, term_seconds: u64) -> bool {
if term_seconds == 0 {
- return None
+ return false
}
let current_term_period = header.timestamp() / term_seconds;
- let parent_term_period = parent_header.timestamp() / term_seconds;
- if current_term_period == parent_term_period {
- return None
- }
- Some(header.number())
+ let parent_term_period = parent.timestamp() / term_seconds;
+
+ current_term_period != parent_term_period
}
fn inactive_validators(
diff --git a/core/src/consensus/tendermint/mod.rs b/core/src/consensus/tendermint/mod.rs
index 85edcd6a9a..13cd8a4e15 100644
--- a/core/src/consensus/tendermint/mod.rs
+++ b/core/src/consensus/tendermint/mod.rs
@@ -41,6 +41,7 @@ pub use self::types::{Height, Step, View};
use super::{stake, ValidatorSet};
use crate::client::ConsensusClient;
use crate::codechain_machine::CodeChainMachine;
+use crate::snapshot_notify::NotifySender as SnapshotNotifySender;
use crate::ChainNotify;
/// Timer token representing the consensus step timeouts.
@@ -58,6 +59,7 @@ pub struct Tendermint {
client: RwLock<Option<Weak<dyn ConsensusClient>>>,
external_params_initializer: crossbeam::Sender<TimeGapParams>,
extension_initializer: crossbeam::Sender<(crossbeam::Sender<network::Event>, Weak<dyn ConsensusClient>)>,
+ snapshot_notify_sender_initializer: crossbeam::Sender<SnapshotNotifySender>,
timeouts: TimeoutParams,
join: Option<JoinHandle<()>>,
quit_tendermint: crossbeam::Sender<()>,
@@ -93,8 +95,14 @@ impl Tendermint {
let timeouts = our_params.timeouts;
let machine = Arc::new(machine);
- let (join, external_params_initializer, extension_initializer, inner, quit_tendermint) =
- worker::spawn(our_params.validators);
+ let (
+ join,
+ external_params_initializer,
+ extension_initializer,
+ snapshot_notify_sender_initializer,
+ inner,
+ quit_tendermint,
+ ) = worker::spawn(our_params.validators);
let action_handlers: Vec<Arc<dyn ActionHandler>> = vec![stake.clone()];
let chain_notify = Arc::new(TendermintChainNotify::new(inner.clone()));
@@ -102,6 +110,7 @@ impl Tendermint {
client: Default::default(),
external_params_initializer,
extension_initializer,
+ snapshot_notify_sender_initializer,
timeouts,
join: Some(join),
quit_tendermint,
@@ -115,6 +124,10 @@ impl Tendermint {
has_signer: false.into(),
})
}
+
+ fn client(&self) -> Option<Arc<dyn ConsensusClient>> {
+ self.client.read().as_ref()?.upgrade()
+ }
}
const SEAL_FIELDS: usize = 4;
@@ -157,9 +170,8 @@ mod tests {
let genesis_header = scheme.genesis_header();
let b = OpenBlock::try_new(scheme.engine.as_ref(), db, &genesis_header, proposer, vec![]).unwrap();
let seal = scheme.engine.generate_seal(None, &genesis_header).seal_fields().unwrap();
- let common_params = CommonParams::default_for_test();
let term_common_params = CommonParams::default_for_test();
- let b = b.close(&genesis_header, &common_params, Some(&term_common_params)).unwrap();
+ let b = b.close(&genesis_header, Some(&term_common_params)).unwrap();
(b, seal)
}
diff --git a/core/src/consensus/tendermint/worker.rs b/core/src/consensus/tendermint/worker.rs
index aec882b908..f9abe27922 100644
--- a/core/src/consensus/tendermint/worker.rs
+++ b/core/src/consensus/tendermint/worker.rs
@@ -50,7 +50,9 @@ use crate::consensus::validator_set::{DynamicValidator, ValidatorSet};
use crate::consensus::{EngineError, Seal};
use crate::encoded;
use crate::error::{BlockError, Error};
+use crate::snapshot_notify::NotifySender as SnapshotNotifySender;
use crate::transaction::{SignedTransaction, UnverifiedTransaction};
+use crate::types::BlockStatus;
use crate::views::BlockView;
use crate::BlockId;
use std::cell::Cell;
@@ -59,6 +61,7 @@ type SpawnResult = (
JoinHandle<()>,
crossbeam::Sender<TimeGapParams>,
crossbeam::Sender<(crossbeam::Sender<network::Event>, Weak<dyn ConsensusClient>)>,
+ crossbeam::Sender<SnapshotNotifySender>,
crossbeam::Sender<Event>,
crossbeam::Sender<()>,
);
@@ -97,6 +100,7 @@ struct Worker {
time_gap_params: TimeGapParams,
timeout_token_nonce: usize,
vote_regression_checker: VoteRegressionChecker,
+ snapshot_notify_sender: SnapshotNotifySender,
}
pub enum Event {
@@ -180,6 +184,7 @@ impl Worker {
extension: EventSender<network::Event>,
client: Weak<dyn ConsensusClient>,
time_gap_params: TimeGapParams,
+ snapshot_notify_sender: SnapshotNotifySender,
) -> Self {
Worker {
client,
@@ -198,6 +203,7 @@ impl Worker {
time_gap_params,
timeout_token_nonce: ENGINE_TIMEOUT_TOKEN_NONCE_BASE,
vote_regression_checker: VoteRegressionChecker::new(),
+ snapshot_notify_sender,
}
}
@@ -206,6 +212,7 @@ impl Worker {
let (quit, quit_receiver) = crossbeam::bounded(1);
let (external_params_initializer, external_params_receiver) = crossbeam::bounded(1);
let (extension_initializer, extension_receiver) = crossbeam::bounded(1);
+ let (snapshot_notify_sender_initializer, snapshot_notify_sender_receiver) = crossbeam::bounded(1);
let join = Builder::new()
.name("tendermint".to_string())
.spawn(move || {
@@ -249,8 +256,29 @@ impl Worker {
return
}
};
+ // TODO: Make initialization steps to order insensitive.
+ let snapshot_notify_sender = crossbeam::select! {
+ recv(snapshot_notify_sender_receiver) -> msg => {
+ match msg {
+ Ok(sender) => sender,
+ Err(crossbeam::RecvError) => {
+ cerror!(ENGINE, "The tendermint extension is not initalized.");
+ return
+ }
+ }
+ }
+ recv(quit_receiver) -> msg => {
+ match msg {
+ Ok(()) => {},
+ Err(crossbeam::RecvError) => {
+ cerror!(ENGINE, "The quit channel for tendermint thread had been closed.");
+ }
+ }
+ return
+ }
+ };
validators.register_client(Weak::clone(&client));
- let mut inner = Self::new(validators, extension, client, time_gap_params);
+ let mut inner = Self::new(validators, extension, client, time_gap_params, snapshot_notify_sender);
loop {
crossbeam::select! {
recv(receiver) -> msg => {
@@ -374,7 +402,7 @@ impl Worker {
}
})
.unwrap();
- (join, external_params_initializer, extension_initializer, sender, quit)
+ (join, external_params_initializer, extension_initializer, snapshot_notify_sender_initializer, sender, quit)
}
/// The client is a thread-safe struct. Using it in multi-threads is safe.
@@ -934,7 +962,8 @@ impl Worker {
}
fn on_imported_proposal(&mut self, proposal: &Header) {
- if proposal.number() < 1 {
+ // NOTE: Only the genesis block and the snapshot target don't have the parent in the blockchain
+ if self.client().block_status(&BlockId::Hash(*proposal.parent_hash())) == BlockStatus::Unknown {
return
}
@@ -1616,6 +1645,8 @@ impl Worker {
}
};
+ self.send_snapshot_notify(c.as_ref(), enacted.as_slice());
+
if self.step.is_commit() && (imported.len() + enacted.len() == 1) {
let (_, committed_block_hash) = self.step.committed().expect("Commit state always has block_hash");
if imported.first() == Some(&committed_block_hash) {
@@ -1667,6 +1698,26 @@ impl Worker {
}
}
+ // Notify once for the latest block even if multiple blocks have been enacted.
+ fn send_snapshot_notify(&mut self, c: &dyn ConsensusClient, enacted: &[BlockHash]) {
+ let mut last_snapshot_point = None;
+ for block_hash in enacted.iter().rev() {
+ let block_id = BlockId::Hash(*block_hash);
+ let last_term_finished_block_num = c.last_term_finished_block_num(block_id).expect("Block is enacted");
+ let block_number = c.block_number(&block_id).expect("Block number should exist for enacted block");
+
+ if let Some(params) = c.term_common_params(block_id) {
+ if params.era() == 1 && (last_term_finished_block_num + 1 == block_number) {
+ last_snapshot_point = Some(block_hash);
+ }
+ }
+ }
+ if let Some(last_snapshot_point) = last_snapshot_point {
+ // TODO: Reduce the snapshot frequency.
+ self.snapshot_notify_sender.notify(*last_snapshot_point);
+ }
+ }
+
fn send_proposal_block(
&self,
signature: SchnorrSignature,
diff --git a/core/src/lib.rs b/core/src/lib.rs
index db2d091932..c0ada218b2 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -84,10 +84,11 @@ mod tests;
pub use crate::account_provider::{AccountProvider, Error as AccountProviderError};
pub use crate::block::Block;
+pub use crate::client::snapshot_notify;
pub use crate::client::{
AccountData, AssetClient, BlockChainClient, BlockChainTrait, ChainNotify, Client, ClientConfig, DatabaseClient,
- EngineClient, EngineInfo, ExecuteClient, ImportBlock, MiningBlockChainClient, Shard, StateInfo, TermInfo,
- TestBlockChainClient, TextClient,
+ EngineClient, EngineInfo, ExecuteClient, ImportBlock, MiningBlockChainClient, Shard, SnapshotClient, StateInfo,
+ TermInfo, TestBlockChainClient, TextClient,
};
pub use crate::consensus::{EngineType, TimeGapParams};
pub use crate::db::{COL_STATE, NUM_COLUMNS};
@@ -98,4 +99,4 @@ pub use crate::service::ClientService;
pub use crate::transaction::{
LocalizedTransaction, PendingSignedTransactions, SignedTransaction, UnverifiedTransaction,
};
-pub use crate::types::{BlockId, TransactionId};
+pub use crate::types::{BlockId, BlockStatus, TransactionId};
diff --git a/core/src/miner/miner.rs b/core/src/miner/miner.rs
index 8c7c446262..5b7c42d865 100644
--- a/core/src/miner/miner.rs
+++ b/core/src/miner/miner.rs
@@ -527,6 +527,7 @@ impl Miner {
return Ok(None)
}
}
+ self.engine.on_open_block(open_block.inner_mut())?;
let mut invalid_transactions = Vec::new();
@@ -603,18 +604,8 @@ impl Miner {
let parent_header = chain.block_header(&parent_hash.into()).expect("Parent header MUST exist");
(parent_header.decode(), parent_hash)
};
- let parent_common_params = chain.common_params(parent_hash.into()).unwrap();
- let term_common_params = {
- let block_number = chain
- .last_term_finished_block_num(parent_hash.into())
- .expect("The block of the parent hash should exist");
- if block_number == 0 {
- None
- } else {
- Some(chain.common_params((block_number).into()).expect("Common params should exist"))
- }
- };
- let block = open_block.close(&parent_header, &parent_common_params, term_common_params.as_ref())?;
+ let term_common_params = chain.term_common_params(parent_hash.into());
+ let block = open_block.close(&parent_header, term_common_params.as_ref())?;
let fetch_seq = |p: &Public| {
let address = public_to_address(p);
diff --git a/core/src/miner/sealing_queue.rs b/core/src/miner/sealing_queue.rs
index 3866274d04..16f282367e 100644
--- a/core/src/miner/sealing_queue.rs
+++ b/core/src/miner/sealing_queue.rs
@@ -87,9 +87,8 @@ mod tests {
let genesis_header = scheme.genesis_header();
let db = scheme.ensure_genesis_state(get_temp_state_db()).unwrap();
let b = OpenBlock::try_new(&*scheme.engine, db, &genesis_header, address, vec![]).unwrap();
- let common_params = CommonParams::default_for_test();
let term_common_params = CommonParams::default_for_test();
- b.close(&genesis_header, &common_params, Some(&term_common_params)).unwrap()
+ b.close(&genesis_header, Some(&term_common_params)).unwrap()
}
#[test]
diff --git a/rpc/src/v1/errors.rs b/rpc/src/v1/errors.rs
index 2dc961786b..eaf98cf5d4 100644
--- a/rpc/src/v1/errors.rs
+++ b/rpc/src/v1/errors.rs
@@ -294,6 +294,14 @@ pub fn invalid_custom_action(err: String) -> Error {
}
}
+pub fn io(error: std::io::Error) -> Error {
+ Error {
+ code: ErrorCode::InternalError,
+ message: format!("{}", error),
+ data: None,
+ }
+}
+
/// Internal error signifying a logic error in code.
/// Should not be used when function can just fail
/// because of invalid parameters or incomplete node state.
diff --git a/rpc/src/v1/impls/devel.rs b/rpc/src/v1/impls/devel.rs
index 588b4f8f83..75af7f3ffe 100644
--- a/rpc/src/v1/impls/devel.rs
+++ b/rpc/src/v1/impls/devel.rs
@@ -23,7 +23,7 @@ use std::vec::Vec;
use ccore::{
BlockId, DatabaseClient, EngineClient, EngineInfo, MinerService, MiningBlockChainClient, SignedTransaction,
- TermInfo, COL_STATE,
+ SnapshotClient, TermInfo, COL_STATE,
};
use ccrypto::Blake;
use cjson::bytes::Bytes;
@@ -33,7 +33,7 @@ use csync::BlockSyncEvent;
use ctypes::transaction::{
Action, AssetMintOutput, AssetOutPoint, AssetTransferInput, AssetTransferOutput, Transaction,
};
-use ctypes::{Tracker, TxHash};
+use ctypes::{BlockHash, Tracker, TxHash};
use jsonrpc_core::Result;
use kvdb::KeyValueDB;
use primitives::{H160, H256};
@@ -70,7 +70,7 @@ where
impl<C, M> Devel for DevelClient<C, M>
where
- C: DatabaseClient + EngineInfo + EngineClient + MiningBlockChainClient + TermInfo + 'static,
+ C: DatabaseClient + EngineInfo + EngineClient + MiningBlockChainClient + TermInfo + SnapshotClient + 'static,
M: MinerService + 'static,
{
fn get_state_trie_keys(&self, offset: usize, limit: usize) -> Result<Vec<H256>> {
@@ -108,6 +108,11 @@ where
}
}
+ fn snapshot(&self, block_hash: BlockHash) -> Result<()> {
+ self.client.notify_snapshot(BlockId::Hash(block_hash));
+ Ok(())
+ }
+
fn test_tps(&self, setting: TPSTestSetting) -> Result<f64> {
let common_params = self.client.common_params(BlockId::Latest).unwrap();
let mint_fee = common_params.min_asset_mint_cost();
diff --git a/rpc/src/v1/impls/mod.rs b/rpc/src/v1/impls/mod.rs
index 45e7678459..3360f5682e 100644
--- a/rpc/src/v1/impls/mod.rs
+++ b/rpc/src/v1/impls/mod.rs
@@ -21,6 +21,7 @@ mod engine;
mod mempool;
mod miner;
mod net;
+mod snapshot;
pub use self::account::AccountClient;
pub use self::chain::ChainClient;
@@ -29,3 +30,4 @@ pub use self::engine::EngineClient;
pub use self::mempool::MempoolClient;
pub use self::miner::MinerClient;
pub use self::net::NetClient;
+pub use self::snapshot::SnapshotClient;
diff --git a/rpc/src/v1/impls/snapshot.rs b/rpc/src/v1/impls/snapshot.rs
new file mode 100644
index 0000000000..f030c9bbd3
--- /dev/null
+++ b/rpc/src/v1/impls/snapshot.rs
@@ -0,0 +1,88 @@
+// Copyright 2018-2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::fs;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use ccore::{BlockChainClient, BlockId};
+use ctypes::BlockHash;
+use primitives::H256;
+
+use jsonrpc_core::Result;
+
+use super::super::errors;
+use super::super::traits::Snapshot;
+use super::super::types::BlockNumberAndHash;
+
+pub struct SnapshotClient<C>
+where
+ C: BlockChainClient, {
+ client: Arc<C>,
+ snapshot_path: Option<String>,
+}
+
+impl<C> SnapshotClient<C>
+where
+ C: BlockChainClient,
+{
+ pub fn new(client: Arc<C>, snapshot_path: Option<String>) -> Self {
+ SnapshotClient {
+ client,
+ snapshot_path,
+ }
+ }
+}
+
+impl<C> Snapshot for SnapshotClient<C>
+where
+ C: BlockChainClient + 'static,
+{
+ fn get_snapshot_list(&self) -> Result<Vec<BlockNumberAndHash>> {
+ if let Some(snapshot_path) = &self.snapshot_path {
+ let mut result = Vec::new();
+ for entry in fs::read_dir(snapshot_path).map_err(errors::io)? {
+ let entry = entry.map_err(errors::io)?;
+
+ // Check if the entry is a directory
+ let file_type = entry.file_type().map_err(errors::io)?;
+ if !file_type.is_dir() {
+ continue
+ }
+
+ let path = entry.path();
+ let name = match path.file_name().expect("Directories always have file name").to_str() {
+ Some(n) => n,
+ None => continue,
+ };
+ let hash = match H256::from_str(name) {
+ Ok(h) => BlockHash::from(h),
+ Err(_) => continue,
+ };
+ if let Some(number) = self.client.block_number(&BlockId::Hash(hash)) {
+ result.push(BlockNumberAndHash {
+ number,
+ hash,
+ });
+ }
+ }
+ result.sort_unstable_by(|a, b| b.number.cmp(&a.number));
+ Ok(result)
+ } else {
+ Ok(Vec::new())
+ }
+ }
+}
diff --git a/rpc/src/v1/traits/devel.rs b/rpc/src/v1/traits/devel.rs
index e8604e910e..565a331976 100644
--- a/rpc/src/v1/traits/devel.rs
+++ b/rpc/src/v1/traits/devel.rs
@@ -17,6 +17,7 @@
use std::net::SocketAddr;
use cjson::bytes::Bytes;
+use ctypes::BlockHash;
use jsonrpc_core::Result;
use primitives::H256;
@@ -39,6 +40,9 @@ pub trait Devel {
#[rpc(name = "devel_getBlockSyncPeers")]
fn get_block_sync_peers(&self) -> Result<Vec<SocketAddr>>;
+ #[rpc(name = "devel_snapshot")]
+ fn snapshot(&self, hash: BlockHash) -> Result<()>;
+
#[rpc(name = "devel_testTPS")]
fn test_tps(&self, setting: TPSTestSetting) -> Result<f64>;
}
diff --git a/rpc/src/v1/traits/mod.rs b/rpc/src/v1/traits/mod.rs
index 719f186e49..7f2bd04599 100644
--- a/rpc/src/v1/traits/mod.rs
+++ b/rpc/src/v1/traits/mod.rs
@@ -21,6 +21,7 @@ mod engine;
mod mempool;
mod miner;
mod net;
+mod snapshot;
pub use self::account::Account;
pub use self::chain::Chain;
@@ -29,3 +30,4 @@ pub use self::engine::Engine;
pub use self::mempool::Mempool;
pub use self::miner::Miner;
pub use self::net::Net;
+pub use self::snapshot::Snapshot;
diff --git a/rpc/src/v1/traits/snapshot.rs b/rpc/src/v1/traits/snapshot.rs
new file mode 100644
index 0000000000..0fd9c18366
--- /dev/null
+++ b/rpc/src/v1/traits/snapshot.rs
@@ -0,0 +1,26 @@
+// Copyright 2018-2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use jsonrpc_core::Result;
+
+use super::super::types::BlockNumberAndHash;
+
+#[rpc(server)]
+pub trait Snapshot {
+ /// Gets list of block numbers and block hashes of the snapshots.
+ #[rpc(name = "snapshot_getList")]
+ fn get_snapshot_list(&self) -> Result<Vec<BlockNumberAndHash>>;
+}
diff --git a/spec/Block-Synchronization-Extension.md b/spec/Block-Synchronization-Extension.md
index fb5a106f75..aa6f3f0372 100644
--- a/spec/Block-Synchronization-Extension.md
+++ b/spec/Block-Synchronization-Extension.md
@@ -19,13 +19,14 @@ Message :=
### Status
```
-Status(total_score, best_hash, genesis_hash)
+Status(nonce, best_hash, genesis_hash)
```
Send current chain status to peer.
* Identifier: 0x01
-* Restriction: None
+* Restriction:
+ * `nonce` SHOULD be monotonically increasing every time the message is sent.
## Request messages
@@ -53,32 +54,17 @@ Request corresponding bodies for each hash.
* Restriction:
* MUST include at least one item
-
-### GetStateHead
-
-```
-GetStateHead(block_hash)
-```
-
-Request corresponding state head for block of `block_hash`.
-
-* Identifier: 0x06
-* Restriction: Block number of requested block MUST be multiple of 214.
-
-
### GetStateChunk
```
-GetStateChunk(block_hash, tree_root)
+GetStateChunk(block_hash, [...chunk_roots])
```
-Request entire subtree starting from `tree_root`.
+Request corresponding snapshot chunk for each `chunk_root`.
-* Identifier: 0x08
+* Identifier: 0x0a
* Restriction:
- * Block number of requested block MUST be multiple of 214.
- * `tree_root` MUST be included in requested block’s state trie.
- * Depth of `tree_root` inside state trie MUST be equal to 2. (Depth of state root is 0)
+ * All values in `[...chunk_roots]` MUST be included in requested block’s state trie.
## Response messages
@@ -113,30 +99,15 @@ Response to `GetBodies` message. Snappy algorithm is used to compress content.
* If received body is zero-length array, it means either body value is [], or sender doesn’t have body for requested hash
-### StateHead
-
-```
-StateHead(compressed((key_0, value_0), …) | [])
-```
-
-Response to `GetStateHead` message. Key and value included in this messages are raw value stored in state trie. Snappy algorithm is used for compression of content.
-
-* Identifier: 0x07
-* Restriction:
- * State root of requested block MUST be included
- * For all nodes with depth of less than 2 included in this message, all of its child MUST also be included.
- * Content MUST be empty array if sender didn’t have requested data
-
-
### StateChunk
```
-StateChunk(compressed((key_0, value_0), …) | [])
+StateChunk([compressed([terminal_0, …] | []), ...])
```
-Response to `GetStateChunk` message. Details of message is same as `StateHead` message.
+Response to `GetStateChunk` message. Snappy algorithm is used for compression of content.
-* Identifier: 0x09
+* Identifier: 0x0b
* Restriction:
- * Node corresponding to tree_root in request MUST be included
- * Every nodes included in message MUST have all of its child in same message.
- * Content MUST be empty array if sender didn’t have requested data
+ * Number and order of chunks included in this message MUST be equal to request information.
+ * Node corresponding to `chunk_root` in request MUST be included
+ * If sender doesn’t have a chunk for the requested hash, corresponding chunk MUST be `[]`(uncompressed), not omitted.
diff --git a/spec/JSON-RPC.md b/spec/JSON-RPC.md
index cefee266dc..870192c22b 100644
--- a/spec/JSON-RPC.md
+++ b/spec/JSON-RPC.md
@@ -367,6 +367,7 @@ When `Transaction` is included in any response, there will be an additional fiel
***
* [devel_getStateTrieKeys](#devel_getstatetriekeys)
* [devel_getStateTrieValue](#devel_getstatetrievalue)
+ * [devel_snapshot](#devel_snapshot)
* [devel_startSealing](#devel_startsealing)
* [devel_stopSealing](#devel_stopsealing)
* [devel_getBlockSyncPeers](#devel_getblocksyncpeers)
@@ -2979,6 +2980,33 @@ Gets the value of the state trie with the given key.
[Back to **List of methods**](#list-of-methods)
+## devel_snapshot
+Snapshot the state of the given block hash.
+
+### Params
+ 1. key: `H256`
+
+### Returns
+
+### Request Example
+```
+ curl \
+ -H 'Content-Type: application/json' \
+ -d '{"jsonrpc": "2.0", "method": "devel_snapshot", "params": ["0xfc196ede542b03b55aee9f106004e7e3d7ea6a9600692e964b4735a260356b50"], "id": null}' \
+ localhost:8080
+```
+
+### Response Example
+```
+{
+ "jsonrpc":"2.0",
+ "result":[],
+ "id":null
+}
+```
+
+[Back to **List of methods**](#list-of-methods)
+
## devel_startSealing
Starts and enables sealing blocks by the miner.
diff --git a/state/src/action_handler/hit.rs b/state/src/action_handler/hit.rs
index 407524fe1f..7891b99e36 100644
--- a/state/src/action_handler/hit.rs
+++ b/state/src/action_handler/hit.rs
@@ -86,13 +86,7 @@ impl ActionHandler for HitHandler {
Ok(())
}
- fn on_close_block(
- &self,
- state: &mut TopLevelState,
- _header: &Header,
- _parent_header: &Header,
- _parent_common_params: &CommonParams,
- ) -> StateResult<()> {
+ fn on_close_block(&self, state: &mut TopLevelState, _header: &Header) -> StateResult<()> {
let address = self.close_count();
let action_data = state.action_data(&address)?.unwrap_or_default();
let prev_counter: u32 = rlp::decode(&*action_data).unwrap();
diff --git a/state/src/action_handler/mod.rs b/state/src/action_handler/mod.rs
index 519f8e62df..dd7c1da1ac 100644
--- a/state/src/action_handler/mod.rs
+++ b/state/src/action_handler/mod.rs
@@ -47,13 +47,7 @@ pub trait ActionHandler: Send + Sync {
Ok(some_action_data)
}
- fn on_close_block(
- &self,
- state: &mut TopLevelState,
- header: &Header,
- parent_header: &Header,
- parent_common_params: &CommonParams,
- ) -> StateResult<()>;
+ fn on_close_block(&self, state: &mut TopLevelState, header: &Header) -> StateResult<()>;
}
pub trait FindActionHandler {
diff --git a/state/src/impls/top_level.rs b/state/src/impls/top_level.rs
index b871efaa58..4d51ec0b3f 100644
--- a/state/src/impls/top_level.rs
+++ b/state/src/impls/top_level.rs
@@ -999,6 +999,12 @@ impl TopState for TopLevelState {
metadata.increase_seq();
Ok(())
}
+
+ fn snapshot_term_params(&mut self) -> StateResult<()> {
+ let mut metadata = self.get_metadata_mut()?;
+ metadata.snapshot_term_params();
+ Ok(())
+ }
}
fn is_active_account(state: &dyn TopStateView, address: &Address) -> TrieResult<bool> {
diff --git a/state/src/item/metadata.rs b/state/src/item/metadata.rs
index 6ad74076b3..39651888a4 100644
--- a/state/src/item/metadata.rs
+++ b/state/src/item/metadata.rs
@@ -34,6 +34,7 @@ pub struct Metadata {
term: TermMetadata,
seq: u64,
params: Option<CommonParams>,
+ term_params: Option<CommonParams>,
}
impl Metadata {
@@ -45,6 +46,7 @@ impl Metadata {
term: Default::default(),
seq: 0,
params: None,
+ term_params: None,
}
}
@@ -93,6 +95,14 @@ impl Metadata {
self.params = Some(params);
}
+ pub fn term_params(&self) -> Option<&CommonParams> {
+ self.term_params.as_ref()
+ }
+
+ pub fn snapshot_term_params(&mut self) {
+ self.term_params = self.params;
+ }
+
pub fn increase_term_id(&mut self, last_term_finished_block_num: u64) {
assert!(self.term.last_term_finished_block_num < last_term_finished_block_num);
self.term.last_term_finished_block_num = last_term_finished_block_num;
@@ -124,25 +134,31 @@ impl CacheableItem for Metadata {
const PREFIX: u8 = super::METADATA_PREFIX;
+const INITIAL_LEN: usize = 4;
+const TERM_LEN: usize = INITIAL_LEN + 2;
+const PARAMS_LEN: usize = TERM_LEN + 2;
+const TERM_PARAMS_LEN: usize = PARAMS_LEN + 1;
+const VALID_LEN: &[usize] = &[INITIAL_LEN, TERM_LEN, PARAMS_LEN, TERM_PARAMS_LEN];
+
impl Encodable for Metadata {
fn rlp_append(&self, s: &mut RlpStream) {
- const INITIAL_LEN: usize = 4;
- const TERM_LEN: usize = 2;
- const PARAMS_LEN: usize = 2;
- let mut len = INITIAL_LEN;
-
let term_changed = self.term != Default::default();
- if term_changed {
- len += TERM_LEN;
- }
-
let params_changed = self.seq != 0;
- if params_changed {
- if !term_changed {
- len += TERM_LEN;
+ let term_params_exist = self.term_params.is_some();
+
+ let len = if term_params_exist {
+ if !params_changed {
+ panic!("Term params only can be changed if params changed");
}
- len += PARAMS_LEN;
- }
+ TERM_PARAMS_LEN
+ } else if params_changed {
+ PARAMS_LEN
+ } else if term_changed {
+ TERM_LEN
+ } else {
+ INITIAL_LEN
+ };
+
s.begin_list(len)
.append(&PREFIX)
.append(&self.number_of_shards)
@@ -159,48 +175,63 @@ impl Encodable for Metadata {
}
s.append(&self.seq).append(self.params.as_ref().unwrap());
}
+ if term_params_exist {
+ if !params_changed {
+ unreachable!("Term params only can be changed if params changed");
+ }
+ s.append(self.term_params.as_ref().unwrap());
+ }
}
}
impl Decodable for Metadata {
fn decode(rlp: &Rlp) -> Result<Self, DecoderError> {
- let (term, seq, params) = match rlp.item_count()? {
- 4 => (TermMetadata::default(), 0, None),
- 6 => (
- TermMetadata {
- last_term_finished_block_num: rlp.val_at(4)?,
- current_term_id: rlp.val_at(5)?,
- },
- 0,
- None,
- ),
- 8 => (
- TermMetadata {
- last_term_finished_block_num: rlp.val_at(4)?,
- current_term_id: rlp.val_at(5)?,
- },
- rlp.val_at(6)?,
- Some(rlp.val_at(7)?),
- ),
- item_count => {
- return Err(DecoderError::RlpInvalidLength {
- got: item_count,
- expected: 4,
- })
- }
- };
+ let item_count = rlp.item_count()?;
+ if !VALID_LEN.contains(&item_count) {
+ return Err(DecoderError::RlpInvalidLength {
+ got: item_count,
+ expected: 4,
+ })
+ }
+
let prefix = rlp.val_at::<u8>(0)?;
if PREFIX != prefix {
cdebug!(STATE, "{} is not an expected prefix for asset", prefix);
return Err(DecoderError::Custom("Unexpected prefix"))
}
+ let number_of_shards = rlp.val_at(1)?;
+ let number_of_initial_shards = rlp.val_at(2)?;
+ let hashes = rlp.list_at(3)?;
+
+ let term = if item_count >= TERM_LEN {
+ TermMetadata {
+ last_term_finished_block_num: rlp.val_at(4)?,
+ current_term_id: rlp.val_at(5)?,
+ }
+ } else {
+ TermMetadata::default()
+ };
+
+ let (seq, params) = if item_count >= PARAMS_LEN {
+ (rlp.val_at(6)?, Some(rlp.val_at(7)?))
+ } else {
+ Default::default()
+ };
+
+ let term_params = if item_count >= TERM_PARAMS_LEN {
+ Some(rlp.val_at(8)?)
+ } else {
+ Default::default()
+ };
+
Ok(Self {
- number_of_shards: rlp.val_at(1)?,
- number_of_initial_shards: rlp.val_at(2)?,
- hashes: rlp.list_at(3)?,
+ number_of_shards,
+ number_of_initial_shards,
+ hashes,
term,
seq,
params,
+ term_params,
})
}
}
@@ -266,6 +297,7 @@ mod tests {
term: Default::default(),
seq: 0,
params: None,
+ term_params: None,
};
let mut rlp = RlpStream::new_list(4);
rlp.append(&PREFIX).append(&10u16).append(&1u16).append_list::<H256, H256>(&[]);
@@ -281,6 +313,7 @@ mod tests {
term: Default::default(),
seq: 3,
params: Some(CommonParams::default_for_test()),
+ term_params: Some(CommonParams::default_for_test()),
};
rlp_encode_and_decode_test!(metadata);
}
@@ -297,6 +330,7 @@ mod tests {
},
seq: 0,
params: None,
+ term_params: None,
};
rlp_encode_and_decode_test!(metadata);
}
@@ -313,6 +347,24 @@ mod tests {
},
seq: 3,
params: Some(CommonParams::default_for_test()),
+ term_params: Some(CommonParams::default_for_test()),
+ };
+ rlp_encode_and_decode_test!(metadata);
+ }
+
+ #[test]
+ fn metadata_with_term_and_seq_but_not_term_params() {
+ let metadata = Metadata {
+ number_of_shards: 10,
+ number_of_initial_shards: 1,
+ hashes: vec![],
+ term: TermMetadata {
+ last_term_finished_block_num: 1,
+ current_term_id: 100,
+ },
+ seq: 3,
+ params: Some(CommonParams::default_for_test()),
+ term_params: None,
};
rlp_encode_and_decode_test!(metadata);
}
diff --git a/state/src/traits.rs b/state/src/traits.rs
index 40d94688f7..c9414611d1 100644
--- a/state/src/traits.rs
+++ b/state/src/traits.rs
@@ -183,6 +183,7 @@ pub trait TopState {
fn remove_action_data(&mut self, key: &H256);
fn update_params(&mut self, metadata_seq: u64, params: CommonParams) -> StateResult<()>;
+ fn snapshot_term_params(&mut self) -> StateResult<()>;
}
pub trait StateWithCache {
diff --git a/sync/Cargo.toml b/sync/Cargo.toml
index c3ad555dc4..caf4846f4f 100644
--- a/sync/Cargo.toml
+++ b/sync/Cargo.toml
@@ -15,6 +15,7 @@ codechain-network = { path = "../network" }
codechain-state = { path = "../state" }
codechain-timer = { path = "../util/timer" }
codechain-types = { path = "../types" }
+hashdb = { path = "../util/hashdb" }
journaldb = { path = "../util/journaldb" }
kvdb = "0.1"
log = "0.4.6"
@@ -29,7 +30,6 @@ token-generator = "0.1.0"
util-error = { path = "../util/error" }
[dev-dependencies]
-hashdb = { path = "../util/hashdb" }
kvdb-memorydb = "0.1"
tempfile = "3.0.4"
trie-standardmap = { path = "../util/trie-standardmap" }
diff --git a/sync/src/block/downloader/header.rs b/sync/src/block/downloader/header.rs
index 963c2135e5..68857da46e 100644
--- a/sync/src/block/downloader/header.rs
+++ b/sync/src/block/downloader/header.rs
@@ -31,21 +31,14 @@ const MAX_HEADER_QUEUE_LENGTH: usize = 1024;
const MAX_RETRY: usize = 3;
const MAX_WAIT: u64 = 15;
-#[derive(Clone)]
-struct Pivot {
- hash: BlockHash,
- total_score: U256,
-}
-
#[derive(Clone)]
pub struct HeaderDownloader {
// NOTE: Use this member as minimum as possible.
client: Arc<dyn BlockChainClient>,
- total_score: U256,
+ nonce: U256,
best_hash: BlockHash,
-
- pivot: Pivot,
+ pivot: BlockHash,
request_time: Option<Instant>,
downloaded: HashMap<BlockHash, Header>,
queued: HashMap<BlockHash, Header>,
@@ -53,24 +46,15 @@ pub struct HeaderDownloader {
}
impl HeaderDownloader {
- pub fn total_score(&self) -> U256 {
- self.total_score
- }
-
- pub fn new(client: Arc<dyn BlockChainClient>, total_score: U256, best_hash: BlockHash) -> Self {
+ pub fn new(client: Arc<dyn BlockChainClient>, nonce: U256, best_hash: BlockHash) -> Self {
let best_header_hash = client.best_block_header().hash();
- let best_score = client.block_total_score(&BlockId::Latest).expect("Best block always exist");
Self {
client,
- total_score,
+ nonce,
best_hash,
-
- pivot: Pivot {
- hash: best_header_hash,
- total_score: best_score,
- },
+ pivot: best_header_hash,
request_time: None,
downloaded: HashMap::new(),
queued: HashMap::new(),
@@ -78,18 +62,23 @@ impl HeaderDownloader {
}
}
- pub fn update(&mut self, total_score: U256, best_hash: BlockHash) -> bool {
- match self.total_score.cmp(&total_score) {
+ pub fn update_pivot(&mut self, hash: BlockHash) {
+ self.pivot = hash;
+ }
+
+ pub fn best_hash(&self) -> BlockHash {
+ self.best_hash
+ }
+
+ pub fn update(&mut self, nonce: U256, best_hash: BlockHash) -> bool {
+ match self.nonce.cmp(&nonce) {
Ordering::Equal => true,
Ordering::Less => {
- self.total_score = total_score;
+ self.nonce = nonce;
self.best_hash = best_hash;
if self.client.block_header(&BlockId::Hash(best_hash)).is_some() {
- self.pivot = Pivot {
- hash: best_hash,
- total_score,
- }
+ self.pivot = best_hash;
}
true
}
@@ -108,25 +97,25 @@ impl HeaderDownloader {
/// Find header from queued headers, downloaded cache and then from blockchain
/// Panics if header dosn't exist
fn pivot_header(&self) -> Header {
- match self.queued.get(&self.pivot.hash) {
+ match self.queued.get(&self.pivot) {
Some(header) => header.clone(),
- None => match self.downloaded.get(&self.pivot.hash) {
+ None => match self.downloaded.get(&self.pivot) {
Some(header) => header.clone(),
- None => self.client.block_header(&BlockId::Hash(self.pivot.hash)).unwrap(),
+ None => self.client.block_header(&BlockId::Hash(self.pivot)).unwrap(),
},
}
}
- pub fn pivot_score(&self) -> U256 {
- self.pivot.total_score
- }
-
pub fn is_idle(&self) -> bool {
- let can_request = self.request_time.is_none() && self.total_score > self.pivot.total_score;
+ let can_request = self.request_time.is_none() && self.best_hash != self.pivot;
self.is_valid() && (can_request || self.is_expired())
}
+ pub fn is_caught_up(&self) -> bool {
+ self.pivot == self.best_hash
+ }
+
pub fn create_request(&mut self) -> Option<RequestMessage> {
if !self.is_idle() {
return None
@@ -154,19 +143,15 @@ impl HeaderDownloader {
let pivot_header = self.pivot_header();
// This happens when best_hash is imported by other peer.
- if self.best_hash == self.pivot.hash {
+ if self.best_hash == self.pivot {
ctrace!(SYNC, "Ignore received headers, pivot already reached the best hash");
- } else if first_header_hash == self.pivot.hash {
+ } else if first_header_hash == self.pivot {
for header in headers.iter() {
self.downloaded.insert(header.hash(), header.clone());
}
// FIXME: skip known headers
- let new_scores = headers[1..].iter().fold(U256::zero(), |acc, header| acc + header.score());
- self.pivot = Pivot {
- hash: headers.last().expect("Last downloaded header must exist").hash(),
- total_score: self.pivot.total_score + new_scores,
- }
+ self.pivot = headers.last().expect("Last downloaded header must exist").hash();
} else if first_header_number < pivot_header.number() {
ctrace!(
SYNC,
@@ -174,17 +159,14 @@ impl HeaderDownloader {
);
} else if first_header_number == pivot_header.number() {
if pivot_header.number() != 0 {
- self.pivot = Pivot {
- hash: pivot_header.parent_hash(),
- total_score: self.pivot.total_score - pivot_header.score(),
- }
+ self.pivot = pivot_header.parent_hash();
}
} else {
cerror!(
SYNC,
- "Invalid header update state. best_hash: {}, self.pivot.hash: {}, first_header_hash: {}",
+ "Invalid header update state. best_hash: {}, self.pivot: {}, first_header_hash: {}",
self.best_hash,
- self.pivot.hash,
+ self.pivot,
first_header_hash
);
}
@@ -203,10 +185,7 @@ impl HeaderDownloader {
self.downloaded.remove(&hash);
if self.best_hash == hash {
- self.pivot = Pivot {
- hash,
- total_score: self.total_score,
- }
+ self.pivot = hash;
}
}
self.queued.shrink_to_fit();
diff --git a/sync/src/block/extension.rs b/sync/src/block/extension.rs
index 1ea7eb4c5b..0899560367 100644
--- a/sync/src/block/extension.rs
+++ b/sync/src/block/extension.rs
@@ -16,20 +16,27 @@
use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
+use std::fs;
+use std::mem::discriminant;
use std::sync::Arc;
use std::time::Duration;
use ccore::encoded::Header as EncodedHeader;
use ccore::{
- Block, BlockChainClient, BlockChainTrait, BlockId, BlockImportError, ChainNotify, Client, ImportBlock, ImportError,
- UnverifiedTransaction,
+ Block, BlockChainClient, BlockChainTrait, BlockId, BlockImportError, BlockStatus, ChainNotify, Client, ImportBlock,
+ ImportError, UnverifiedTransaction,
};
+use cmerkle::snapshot::ChunkDecompressor;
+use cmerkle::snapshot::Restore as SnapshotRestore;
+use cmerkle::{skewed_merkle_root, TrieFactory};
use cnetwork::{Api, EventSender, NetworkExtension, NodeId};
use cstate::FindActionHandler;
use ctimer::TimerToken;
use ctypes::header::{Header, Seal};
use ctypes::transaction::Action;
use ctypes::{BlockHash, BlockNumber};
+use hashdb::AsHashDB;
+use kvdb::DBTransaction;
use primitives::{H256, U256};
use rand::prelude::SliceRandom;
use rand::thread_rng;
@@ -38,6 +45,7 @@ use token_generator::TokenGenerator;
use super::downloader::{BodyDownloader, HeaderDownloader};
use super::message::{Message, RequestMessage, ResponseMessage};
+use crate::snapshot::snapshot_path;
const SYNC_TIMER_TOKEN: TimerToken = 0;
const SYNC_EXPIRE_TOKEN_BEGIN: TimerToken = SYNC_TIMER_TOKEN + 1;
@@ -47,15 +55,28 @@ const SYNC_EXPIRE_TOKEN_END: TimerToken = SYNC_EXPIRE_TOKEN_BEGIN + SYNC_EXPIRE_
const SYNC_TIMER_INTERVAL: u64 = 1000;
const SYNC_EXPIRE_REQUEST_INTERVAL: u64 = 15000;
-const SNAPSHOT_PERIOD: u64 = (1 << 14);
-
#[derive(Debug, PartialEq)]
pub struct TokenInfo {
node_id: NodeId,
request_id: Option<u64>,
}
+#[derive(Debug)]
+enum State {
+ SnapshotHeader(BlockHash, u64),
+ SnapshotBody {
+ header: EncodedHeader,
+ prev_root: H256,
+ },
+ SnapshotChunk {
+ block: BlockHash,
+ restore: SnapshotRestore,
+ },
+ Full,
+}
+
pub struct Extension {
+ state: State,
requests: HashMap<NodeId, Vec<(u64, RequestMessage)>>,
connected_nodes: HashSet<NodeId>,
header_downloaders: HashMap<NodeId, HeaderDownloader>,
@@ -66,19 +87,30 @@ pub struct Extension {
client: Arc<Client>,
api: Box<dyn Api>,
last_request: u64,
+ nonce: u64,
+ snapshot_dir: Option<String>,
}
impl Extension {
- pub fn new(client: Arc<Client>, api: Box<dyn Api>) -> Extension {
+ pub fn new(
+ client: Arc<Client>,
+ api: Box<dyn Api>,
+ snapshot_target: Option<(BlockHash, u64)>,
+ snapshot_dir: Option<String>,
+ ) -> Extension {
api.set_timer(SYNC_TIMER_TOKEN, Duration::from_millis(SYNC_TIMER_INTERVAL)).expect("Timer set succeeds");
+ let state = Extension::initial_state(client.clone(), snapshot_target);
+ cdebug!(SYNC, "Initial state is {:?}", state);
let mut header = client.best_header();
let mut hollow_headers = vec![header.decode()];
while client.block_body(&BlockId::Hash(header.hash())).is_none() {
- header = client
- .block_header(&BlockId::Hash(header.parent_hash()))
- .expect("Every imported header must have parent");
- hollow_headers.push(header.decode());
+ if let Some(h) = client.block_header(&BlockId::Hash(header.parent_hash())) {
+ header = h;
+ hollow_headers.push(header.decode());
+ } else {
+ break
+ }
}
let mut body_downloader = BodyDownloader::default();
for neighbors in hollow_headers.windows(2).rev() {
@@ -90,6 +122,7 @@ impl Extension {
}
cinfo!(SYNC, "Sync extension initialized");
Extension {
+ state,
requests: Default::default(),
connected_nodes: Default::default(),
header_downloaders: Default::default(),
@@ -100,6 +133,38 @@ impl Extension {
client,
api,
last_request: Default::default(),
+ nonce: Default::default(),
+ snapshot_dir,
+ }
+ }
+
+ fn initial_state(client: Arc<Client>, snapshot_target: Option<(BlockHash, u64)>) -> State {
+ let (hash, num) = match snapshot_target {
+ Some(target) => target,
+ None => return State::Full,
+ };
+ let header = match client.block_header(&num.into()) {
+ Some(ref h) if h.hash() == hash => h.clone(),
+ _ => return State::SnapshotHeader(hash, num),
+ };
+ if client.block_body(&hash.into()).is_none() {
+ let parent_hash = header.parent_hash();
+ let parent =
+ client.block_header(&parent_hash.into()).expect("Parent header of the snapshot header must exist");
+ return State::SnapshotBody {
+ header,
+ prev_root: parent.transactions_root(),
+ }
+ }
+
+ let state_db = client.state_db().read();
+ let state_root = header.state_root();
+ match TrieFactory::readonly(state_db.as_hashdb(), &state_root) {
+ Ok(ref trie) if trie.is_complete() => State::Full,
+ _ => State::SnapshotChunk {
+ block: hash,
+ restore: SnapshotRestore::new(state_root),
+ },
}
}
@@ -109,6 +174,48 @@ impl Extension {
}
}
+ fn send_status(&mut self, id: &NodeId) {
+ if discriminant(&self.state) != discriminant(&State::Full) {
+ return
+ }
+
+ let chain_info = self.client.chain_info();
+ self.api.send(
+ id,
+ Arc::new(
+ Message::Status {
+ nonce: U256::from(self.nonce),
+ best_hash: chain_info.best_proposal_block_hash,
+ genesis_hash: chain_info.genesis_hash,
+ }
+ .rlp_bytes(),
+ ),
+ );
+ self.nonce += 1;
+ }
+
+ fn send_status_broadcast(&mut self) {
+ if discriminant(&self.state) != discriminant(&State::Full) {
+ return
+ }
+
+ let chain_info = self.client.chain_info();
+ for id in self.connected_nodes.iter() {
+ self.api.send(
+ id,
+ Arc::new(
+ Message::Status {
+ nonce: U256::from(self.nonce),
+ best_hash: chain_info.best_proposal_block_hash,
+ genesis_hash: chain_info.genesis_hash,
+ }
+ .rlp_bytes(),
+ ),
+ );
+ self.nonce += 1;
+ }
+ }
+
fn send_header_request(&mut self, id: &NodeId, request: RequestMessage) {
if let Some(requests) = self.requests.get_mut(id) {
ctrace!(SYNC, "Send header request to {}", id);
@@ -120,6 +227,14 @@ impl Extension {
}
fn send_body_request(&mut self, id: &NodeId) {
+ if let Some(downloader) = self.header_downloaders.get(&id) {
+ if self.client.block_status(&BlockId::Hash(downloader.best_hash())) == BlockStatus::InChain {
+ // Peer is lagging behind the local blockchain.
+ // We don't need to request block bodies to this peer
+ return
+ }
+ }
+
self.check_sync_variable();
if let Some(requests) = self.requests.get_mut(id) {
let have_body_request = {
@@ -153,6 +268,37 @@ impl Extension {
self.check_sync_variable();
}
+ fn send_chunk_request(&mut self, block: &BlockHash, root: &H256) {
+ let have_chunk_request = self.requests.values().flatten().any(|r| match r {
+ (_, RequestMessage::StateChunk(..)) => true,
+ _ => false,
+ });
+
+ if !have_chunk_request {
+ let mut peer_ids: Vec<_> = self.header_downloaders.keys().cloned().collect();
+ peer_ids.shuffle(&mut thread_rng());
+ if let Some(id) = peer_ids.first() {
+ if let Some(requests) = self.requests.get_mut(&id) {
+ let req = RequestMessage::StateChunk(*block, vec![*root]);
+ cdebug!(SYNC, "Request chunk to {} {:?}", id, req);
+ let request_id = self.last_request;
+ self.last_request += 1;
+ requests.push((request_id, req.clone()));
+ self.api.send(id, Arc::new(Message::Request(request_id, req).rlp_bytes()));
+
+ let token = &self.tokens[id];
+ let token_info = self.tokens_info.get_mut(token).unwrap();
+
+ let _ = self.api.clear_timer(*token);
+ self.api
+ .set_timer_once(*token, Duration::from_millis(SYNC_EXPIRE_REQUEST_INTERVAL))
+ .expect("Timer set succeeds");
+ token_info.request_id = Some(request_id);
+ }
+ }
+ }
+ }
+
fn check_sync_variable(&self) {
let mut has_error = false;
for id in self.header_downloaders.keys() {
@@ -164,7 +310,15 @@ impl Extension {
let body_requests: Vec<RequestMessage> = requests
.iter()
.filter_map(|r| match r {
- (_, RequestMessage::Bodies(..)) => Some(r.1.clone()),
+ (_, msg @ RequestMessage::Bodies(..)) => Some(msg.clone()),
+ _ => None,
+ })
+ .collect();
+
+ let chunk_requests: Vec<RequestMessage> = requests
+ .iter()
+ .filter_map(|r| match r {
+ (_, msg @ RequestMessage::StateChunk(..)) => Some(msg.clone()),
_ => None,
})
.collect();
@@ -177,16 +331,18 @@ impl Extension {
let token = &self.tokens[id];
let token_info = &self.tokens_info[token];
- match (token_info.request_id, body_requests.len()) {
+ match (token_info.request_id, body_requests.len() + chunk_requests.len()) {
(Some(_), 1) => {}
(None, 0) => {}
_ => {
cerror!(
SYNC,
- "request_id: {:?}, body_requests.len(): {}, body_requests: {:?}",
+ "request_id: {:?}, body_requests.len(): {}, body_requests: {:?}, chunk_requests.len(): {}, chunk_requests: {:?}",
token_info.request_id,
body_requests.len(),
- body_requests
+ body_requests,
+ chunk_requests.len(),
+ chunk_requests
);
has_error = true;
}
@@ -212,18 +368,8 @@ impl NetworkExtension<Event> for Extension {
fn on_node_added(&mut self, id: &NodeId, _version: u64) {
cinfo!(SYNC, "New peer detected #{}", id);
- let chain_info = self.client.chain_info();
- self.api.send(
- id,
- Arc::new(
- Message::Status {
- total_score: chain_info.best_proposal_score,
- best_hash: chain_info.best_proposal_block_hash,
- genesis_hash: chain_info.genesis_hash,
- }
- .rlp_bytes(),
- ),
- );
+ self.send_status(id);
+
let t = self.connected_nodes.insert(*id);
debug_assert!(t, "{} is already added to peer list", id);
@@ -272,10 +418,10 @@ impl NetworkExtension<Event> for Extension {
if let Ok(received_message) = Rlp::new(data).as_val() {
match received_message {
Message::Status {
- total_score,
+ nonce,
best_hash,
genesis_hash,
- } => self.on_peer_status(id, total_score, best_hash, genesis_hash),
+ } => self.on_peer_status(id, nonce, best_hash, genesis_hash),
Message::Request(request_id, request) => self.on_peer_request(id, request_id, request),
Message::Response(request_id, response) => self.on_peer_response(id, request_id, response),
}
@@ -287,27 +433,66 @@ impl NetworkExtension<Event> for Extension {
fn on_timeout(&mut self, token: TimerToken) {
match token {
SYNC_TIMER_TOKEN => {
- let best_proposal_score = self.client.chain_info().best_proposal_score;
let mut peer_ids: Vec<_> = self.header_downloaders.keys().cloned().collect();
peer_ids.shuffle(&mut thread_rng());
- for id in &peer_ids {
- let request = self.header_downloaders.get_mut(id).and_then(HeaderDownloader::create_request);
- if let Some(request) = request {
- self.send_header_request(id, request);
- break
+ match self.state {
+ State::SnapshotHeader(_, num) => {
+ for id in &peer_ids {
+ self.send_header_request(id, RequestMessage::Headers {
+ start_number: num - 1,
+ max_count: 2,
+ });
+ }
}
- }
-
- for id in peer_ids {
- let peer_score = if let Some(peer) = self.header_downloaders.get(&id) {
- peer.total_score()
- } else {
- U256::zero()
- };
+ State::SnapshotBody {
+ ref header,
+ ..
+ } => {
+ for id in &peer_ids {
+ if let Some(requests) = self.requests.get_mut(id) {
+ ctrace!(SYNC, "Send snapshot body request to {}", id);
+ let request = RequestMessage::Bodies(vec![header.hash()]);
+ let request_id = self.last_request;
+ self.last_request += 1;
+ requests.push((request_id, request.clone()));
+ self.api.send(id, Arc::new(Message::Request(request_id, request).rlp_bytes()));
+
+ let token = &self.tokens[id];
+ let token_info = self.tokens_info.get_mut(token).unwrap();
+
+ let _ = self.api.clear_timer(*token);
+ self.api
+ .set_timer_once(*token, Duration::from_millis(SYNC_EXPIRE_REQUEST_INTERVAL))
+ .expect("Timer set succeeds");
+ token_info.request_id = Some(request_id);
+ }
+ }
+ }
+ State::SnapshotChunk {
+ block,
+ ref mut restore,
+ } => {
+ if let Some(root) = restore.next_to_feed() {
+ self.send_chunk_request(&block, &root);
+ } else {
+ self.client.force_update_best_block(&block);
+ self.transition_to_full();
+ }
+ }
+ State::Full => {
+ for id in &peer_ids {
+ let request =
+ self.header_downloaders.get_mut(id).and_then(HeaderDownloader::create_request);
+ if let Some(request) = request {
+ self.send_header_request(id, request);
+ break
+ }
+ }
- if peer_score > best_proposal_score {
- self.send_body_request(&id);
+ for id in peer_ids {
+ self.send_body_request(&id);
+ }
}
}
}
@@ -386,61 +571,44 @@ pub enum Event {
impl Extension {
fn new_headers(&mut self, imported: Vec<BlockHash>, enacted: Vec<BlockHash>, retracted: Vec<BlockHash>) {
- let peer_ids: Vec<_> = self.header_downloaders.keys().cloned().collect();
- for id in peer_ids {
- if let Some(peer) = self.header_downloaders.get_mut(&id) {
+ if let State::Full = self.state {
+ for peer in self.header_downloaders.values_mut() {
peer.mark_as_imported(imported.clone());
}
+ let mut headers_to_download: Vec<_> = enacted
+ .into_iter()
+ .map(|hash| self.client.block_header(&BlockId::Hash(hash)).expect("Enacted header must exist"))
+ .collect();
+ headers_to_download.sort_unstable_by_key(EncodedHeader::number);
+ #[allow(clippy::redundant_closure)]
+ // False alarm. https://github.com/rust-lang/rust-clippy/issues/1439
+ headers_to_download.dedup_by_key(|h| h.hash());
+
+ let headers: Vec<_> = headers_to_download
+ .into_iter()
+ .filter(|header| self.client.block_body(&BlockId::Hash(header.hash())).is_none())
+ .collect(); // FIXME: No need to collect here if self is not borrowed.
+ for header in headers {
+ let parent = self
+ .client
+ .block_header(&BlockId::Hash(header.parent_hash()))
+ .expect("Enacted header must have parent");
+ let is_empty = header.transactions_root() == parent.transactions_root();
+ self.body_downloader.add_target(&header.decode(), is_empty);
+ }
+ self.body_downloader.remove_target(&retracted);
}
- let mut headers_to_download: Vec<_> = enacted
- .into_iter()
- .map(|hash| self.client.block_header(&BlockId::Hash(hash)).expect("Enacted header must exist"))
- .collect();
- headers_to_download.sort_unstable_by_key(EncodedHeader::number);
- #[allow(clippy::redundant_closure)]
- // False alarm. https://github.com/rust-lang/rust-clippy/issues/1439
- headers_to_download.dedup_by_key(|h| h.hash());
-
- let headers: Vec<_> = headers_to_download
- .into_iter()
- .filter(|header| self.client.block_body(&BlockId::Hash(header.hash())).is_none())
- .collect(); // FIXME: No need to collect here if self is not borrowed.
- for header in headers {
- let parent = self
- .client
- .block_header(&BlockId::Hash(header.parent_hash()))
- .expect("Enacted header must have parent");
- let is_empty = header.transactions_root() == parent.transactions_root();
- self.body_downloader.add_target(&header.decode(), is_empty);
- }
- self.body_downloader.remove_target(&retracted);
}
fn new_blocks(&mut self, imported: Vec<BlockHash>, invalid: Vec<BlockHash>) {
self.body_downloader.remove_target(&imported);
self.body_downloader.remove_target(&invalid);
-
-
- let chain_info = self.client.chain_info();
-
- for id in &self.connected_nodes {
- self.api.send(
- id,
- Arc::new(
- Message::Status {
- total_score: chain_info.best_proposal_score,
- best_hash: chain_info.best_proposal_block_hash,
- genesis_hash: chain_info.genesis_hash,
- }
- .rlp_bytes(),
- ),
- );
- }
+ self.send_status_broadcast();
}
}
impl Extension {
- fn on_peer_status(&mut self, from: &NodeId, total_score: U256, best_hash: BlockHash, genesis_hash: BlockHash) {
+ fn on_peer_status(&mut self, from: &NodeId, nonce: U256, best_hash: BlockHash, genesis_hash: BlockHash) {
// Validity check
if genesis_hash != self.client.chain_info().genesis_hash {
cinfo!(SYNC, "Genesis hash mismatch with peer {}", from);
@@ -449,21 +617,21 @@ impl Extension {
match self.header_downloaders.entry(*from) {
Entry::Occupied(mut peer) => {
- if !peer.get_mut().update(total_score, best_hash) {
+ if !peer.get_mut().update(nonce, best_hash) {
// FIXME: It should be an error level if the consensus is PoW.
cdebug!(SYNC, "Peer #{} status updated but score is less than before", from);
return
}
}
Entry::Vacant(e) => {
- e.insert(HeaderDownloader::new(self.client.clone(), total_score, best_hash));
+ e.insert(HeaderDownloader::new(self.client.clone(), nonce, best_hash));
}
}
- cinfo!(SYNC, "Peer #{} status update: total_score: {}, best_hash: {}", from, total_score, best_hash);
+ cinfo!(SYNC, "Peer #{} status update: nonce: {}, best_hash: {}", from, nonce, best_hash);
}
fn on_peer_request(&self, from: &NodeId, id: u64, request: RequestMessage) {
- if !self.header_downloaders.contains_key(from) {
+ if !self.connected_nodes.contains(from) {
cinfo!(SYNC, "Request from invalid peer #{} received", from);
return
}
@@ -485,11 +653,9 @@ impl Extension {
ctrace!(SYNC, "Received body request from {}", from);
self.create_bodies_response(hashes)
}
- RequestMessage::StateHead(hash) => self.create_state_head_response(hash),
- RequestMessage::StateChunk {
- block_hash,
- tree_root,
- } => self.create_state_chunk_response(block_hash, tree_root),
+ RequestMessage::StateChunk(block_hash, chunk_root) => {
+ self.create_state_chunk_response(block_hash, chunk_root)
+ }
};
self.api.send(from, Arc::new(Message::Response(id, response).rlp_bytes()));
@@ -501,21 +667,9 @@ impl Extension {
..
} => true,
RequestMessage::Bodies(hashes) => !hashes.is_empty(),
- RequestMessage::StateHead(hash) => match self.client.block_number(&BlockId::Hash(*hash)) {
- Some(number) if number % SNAPSHOT_PERIOD == 0 => true,
- _ => false,
- },
RequestMessage::StateChunk {
- block_hash,
..
- } => {
- let _is_checkpoint = match self.client.block_number(&BlockId::Hash(*block_hash)) {
- Some(number) if number % SNAPSHOT_PERIOD == 0 => true,
- _ => false,
- };
- // FIXME: check tree_root
- unimplemented!()
- }
+ } => true,
}
}
@@ -553,12 +707,18 @@ impl Extension {
ResponseMessage::Bodies(bodies)
}
- fn create_state_head_response(&self, _hash: BlockHash) -> ResponseMessage {
- unimplemented!()
- }
-
- fn create_state_chunk_response(&self, _hash: BlockHash, _tree_root: H256) -> ResponseMessage {
- unimplemented!()
+ fn create_state_chunk_response(&self, hash: BlockHash, chunk_roots: Vec<H256>) -> ResponseMessage {
+ let mut result = Vec::new();
+ for root in chunk_roots {
+ if let Some(dir) = &self.snapshot_dir {
+ let chunk_path = snapshot_path(&dir, &hash, &root);
+ match fs::read(chunk_path) {
+ Ok(chunk) => result.push(chunk),
+ _ => result.push(Vec::new()),
+ }
+ }
+ }
+ ResponseMessage::StateChunk(result)
}
fn on_peer_response(&mut self, from: &NodeId, id: u64, mut response: ResponseMessage) {
@@ -598,7 +758,24 @@ impl Extension {
self.on_body_response(hashes, bodies);
self.check_sync_variable();
}
- _ => unimplemented!(),
+ ResponseMessage::StateChunk(chunks) => {
+ let roots = match request {
+ RequestMessage::StateChunk(_, roots) => roots,
+ _ => unreachable!(),
+ };
+ if let Some(token) = self.tokens.get(from) {
+ if let Some(token_info) = self.tokens_info.get_mut(token) {
+ if token_info.request_id.is_none() {
+ ctrace!(SYNC, "Expired before handling response");
+ return
+ }
+ self.api.clear_timer(*token).expect("Timer clear succeed");
+ token_info.request_id = None;
+ }
+ }
+ self.dismiss_request(from, id);
+ self.on_chunk_response(from, &roots, &chunks);
+ }
}
}
}
@@ -652,13 +829,10 @@ impl Extension {
}
true
}
- (RequestMessage::StateHead(..), ResponseMessage::StateHead(..)) => unimplemented!(),
- (
- RequestMessage::StateChunk {
- ..
- },
- ResponseMessage::StateChunk(..),
- ) => unimplemented!(),
+ (RequestMessage::StateChunk(_, roots), ResponseMessage::StateChunk(chunks)) => {
+ // Check length
+ roots.len() == chunks.len()
+ }
_ => {
cwarn!(SYNC, "Invalid response type");
false
@@ -668,95 +842,228 @@ impl Extension {
fn on_header_response(&mut self, from: &NodeId, headers: &[Header]) {
ctrace!(SYNC, "Received header response from({}) with length({})", from, headers.len());
- let (mut completed, pivot_score_changed) = if let Some(peer) = self.header_downloaders.get_mut(from) {
- let before_pivot_score = peer.pivot_score();
- let encoded: Vec<_> = headers.iter().map(|h| EncodedHeader::new(h.rlp_bytes().to_vec())).collect();
- peer.import_headers(&encoded);
- let after_pivot_score = peer.pivot_score();
- (peer.downloaded(), before_pivot_score != after_pivot_score)
- } else {
- (Vec::new(), false)
- };
- completed.sort_unstable_by_key(EncodedHeader::number);
-
- let mut exists = Vec::new();
- let mut queued = Vec::new();
-
- for header in completed {
- let hash = header.hash();
- match self.client.import_header(header.clone().into_inner()) {
- Err(BlockImportError::Import(ImportError::AlreadyInChain)) => exists.push(hash),
- Err(BlockImportError::Import(ImportError::AlreadyQueued)) => queued.push(hash),
- // FIXME: handle import errors
- Err(err) => {
- cwarn!(SYNC, "Cannot import header({}): {:?}", header.hash(), err);
- break
+ match self.state {
+ State::SnapshotHeader(hash, _) => match headers {
+ [parent, header] if header.hash() == hash => {
+ match self.client.import_trusted_header(parent) {
+ Ok(_)
+ | Err(BlockImportError::Import(ImportError::AlreadyInChain))
+ | Err(BlockImportError::Import(ImportError::AlreadyQueued)) => {}
+ Err(err) => {
+ cwarn!(SYNC, "Cannot import header({}): {:?}", parent.hash(), err);
+ return
+ }
+ }
+ match self.client.import_trusted_header(header) {
+ Ok(_)
+ | Err(BlockImportError::Import(ImportError::AlreadyInChain))
+ | Err(BlockImportError::Import(ImportError::AlreadyQueued)) => {}
+ Err(err) => {
+ cwarn!(SYNC, "Cannot import header({}): {:?}", header.hash(), err);
+ return
+ }
+ }
+ self.state = State::SnapshotBody {
+ header: EncodedHeader::new(header.rlp_bytes().to_vec()),
+ prev_root: *parent.transactions_root(),
+ };
+ cdebug!(SYNC, "Transitioning state to {:?}", self.state);
}
- _ => {}
- }
- }
+ _ => cdebug!(
+ SYNC,
+ "Peer {} responded with a invalid response. requested hash: {}, response length: {}",
+ from,
+ hash,
+ headers.len()
+ ),
+ },
+ State::SnapshotBody {
+ ..
+ } => {}
+ State::SnapshotChunk {
+ ..
+ } => {}
+ State::Full => {
+ let (mut completed, peer_is_caught_up) = if let Some(peer) = self.header_downloaders.get_mut(from) {
+ let encoded: Vec<_> = headers.iter().map(|h| EncodedHeader::new(h.rlp_bytes().to_vec())).collect();
+ peer.import_headers(&encoded);
+ (peer.downloaded(), peer.is_caught_up())
+ } else {
+ (Vec::new(), true)
+ };
+ completed.sort_unstable_by_key(EncodedHeader::number);
- let request = self.header_downloaders.get_mut(from).and_then(|peer| {
- peer.mark_as_queued(queued);
- peer.mark_as_imported(exists);
- peer.create_request()
- });
- if pivot_score_changed {
- if let Some(request) = request {
- self.send_header_request(from, request);
+ let mut exists = Vec::new();
+ let mut queued = Vec::new();
+
+ for header in completed {
+ let hash = header.hash();
+ match self.client.import_header(header.clone().into_inner()) {
+ Err(BlockImportError::Import(ImportError::AlreadyInChain)) => exists.push(hash),
+ Err(BlockImportError::Import(ImportError::AlreadyQueued)) => queued.push(hash),
+ // FIXME: handle import errors
+ Err(err) => {
+ cwarn!(SYNC, "Cannot import header({}): {:?}", header.hash(), err);
+ break
+ }
+ _ => {}
+ }
+ }
+
+ let request = self.header_downloaders.get_mut(from).and_then(|peer| {
+ peer.mark_as_queued(queued);
+ peer.mark_as_imported(exists);
+ peer.create_request()
+ });
+ if !peer_is_caught_up {
+ if let Some(request) = request {
+ self.send_header_request(from, request);
+ }
+ }
}
}
}
fn on_body_response(&mut self, hashes: Vec<BlockHash>, bodies: Vec<Vec<UnverifiedTransaction>>) {
ctrace!(SYNC, "Received body response with lenth({}) {:?}", hashes.len(), hashes);
- {
- self.body_downloader.import_bodies(hashes, bodies);
- let completed = self.body_downloader.drain();
- for (hash, transactions) in completed {
- let header = self
- .client
- .block_header(&BlockId::Hash(hash))
- .expect("Downloaded body's header must exist")
- .decode();
- let block = Block {
- header,
- transactions,
- };
- cdebug!(SYNC, "Body download completed for #{}({})", block.header.number(), hash);
- match self.client.import_block(block.rlp_bytes(&Seal::With)) {
- Err(BlockImportError::Import(ImportError::AlreadyInChain)) => {
- cwarn!(SYNC, "Downloaded already existing block({})", hash)
- }
- Err(BlockImportError::Import(ImportError::AlreadyQueued)) => {
- cwarn!(SYNC, "Downloaded already queued in the verification queue({})", hash)
- }
- Err(err) => {
+
+ match self.state {
+ State::SnapshotBody {
+ ref header,
+ prev_root,
+ } => {
+ let body = bodies.first().expect("Body response in SnapshotBody state has only one body");
+ let new_root = skewed_merkle_root(prev_root, body.iter().map(Encodable::rlp_bytes));
+ if header.transactions_root() == new_root {
+ let block = Block {
+ header: header.decode(),
+ transactions: body.clone(),
+ };
+ match self.client.import_trusted_block(&block) {
+ Ok(_) | Err(BlockImportError::Import(ImportError::AlreadyInChain)) => {
+ self.state = State::SnapshotChunk {
+ block: header.hash(),
+ restore: SnapshotRestore::new(header.state_root()),
+ };
+ cdebug!(SYNC, "Transitioning state to {:?}", self.state);
+ }
+ Err(BlockImportError::Import(ImportError::AlreadyQueued)) => {}
// FIXME: handle import errors
- cwarn!(SYNC, "Cannot import block({}): {:?}", hash, err);
- break
+ Err(err) => {
+ cwarn!(SYNC, "Cannot import block({}): {:?}", header.hash(), err);
+ }
}
- _ => {}
}
}
+ State::Full => {
+ {
+ self.body_downloader.import_bodies(hashes, bodies);
+ let completed = self.body_downloader.drain();
+ for (hash, transactions) in completed {
+ let header = self
+ .client
+ .block_header(&BlockId::Hash(hash))
+ .expect("Downloaded body's header must exist")
+ .decode();
+ let block = Block {
+ header,
+ transactions,
+ };
+ cdebug!(SYNC, "Body download completed for #{}({})", block.header.number(), hash);
+ match self.client.import_block(block.rlp_bytes(&Seal::With)) {
+ Err(BlockImportError::Import(ImportError::AlreadyInChain)) => {
+ cwarn!(SYNC, "Downloaded already existing block({})", hash)
+ }
+ Err(BlockImportError::Import(ImportError::AlreadyQueued)) => {
+ cwarn!(SYNC, "Downloaded already queued in the verification queue({})", hash)
+ }
+ Err(err) => {
+ // FIXME: handle import errors
+ cwarn!(SYNC, "Cannot import block({}): {:?}", hash, err);
+ break
+ }
+ _ => {}
+ }
+ }
+ }
+
+ let mut peer_ids: Vec<_> = self.header_downloaders.keys().cloned().collect();
+ peer_ids.shuffle(&mut thread_rng());
+
+ for id in peer_ids {
+ self.send_body_request(&id);
+ }
+ }
+ _ => {}
}
+ }
+
+ fn on_chunk_response(&mut self, from: &NodeId, roots: &[H256], chunks: &[Vec<u8>]) {
+ if let State::SnapshotChunk {
+ block,
+ ref mut restore,
+ } = self.state
+ {
+ for (r, c) in roots.iter().zip(chunks) {
+ if c.is_empty() {
+ cdebug!(SYNC, "Peer {} sent empty response for chunk request {}", from, r);
+ continue
+ }
+ let decompressor = ChunkDecompressor::from_slice(c);
+ let raw_chunk = match decompressor.decompress() {
+ Ok(chunk) => chunk,
+ Err(e) => {
+ cwarn!(SYNC, "Decode failed for chunk response from peer {}: {}", from, e);
+ continue
+ }
+ };
+ let recovered = match raw_chunk.recover(*r) {
+ Ok(chunk) => chunk,
+ Err(e) => {
+ cwarn!(SYNC, "Invalid chunk response from peer {}: {}", from, e);
+ continue
+ }
+ };
- let total_score = self.client.chain_info().best_proposal_score;
- let mut peer_ids: Vec<_> = self.header_downloaders.keys().cloned().collect();
- peer_ids.shuffle(&mut thread_rng());
+ let batch = {
+ let mut state_db = self.client.state_db().write();
+ let hash_db = state_db.as_hashdb_mut();
+ restore.feed(hash_db, recovered);
+
+ let mut batch = DBTransaction::new();
+ match state_db.journal_under(&mut batch, 0, H256::zero()) {
+ Ok(_) => batch,
+ Err(e) => {
+ cwarn!(SYNC, "Failed to write state chunk to database: {}", e);
+ continue
+ }
+ }
+ };
+ self.client.db().write_buffered(batch);
+ match self.client.db().flush() {
+ Ok(_) => cdebug!(SYNC, "Wrote state chunk to database: {}", r),
+ Err(e) => cwarn!(SYNC, "Failed to flush database: {}", e),
+ }
+ }
- for id in peer_ids {
- let peer_score = if let Some(peer) = self.header_downloaders.get(&id) {
- peer.total_score()
+ if let Some(root) = restore.next_to_feed() {
+ self.send_chunk_request(&block, &root);
} else {
- U256::zero()
- };
-
- if peer_score > total_score {
- self.send_body_request(&id);
+ self.client.force_update_best_block(&block);
+ self.transition_to_full();
}
}
}
+
+ fn transition_to_full(&mut self) {
+ cdebug!(SYNC, "Transitioning state to {:?}", State::Full);
+ let best_hash = self.client.best_block_header().hash();
+ for downloader in self.header_downloaders.values_mut() {
+ downloader.update_pivot(best_hash);
+ }
+ self.state = State::Full;
+ self.send_status_broadcast();
+ }
}
pub struct BlockSyncSender(EventSender<Event>);
diff --git a/sync/src/block/message/mod.rs b/sync/src/block/message/mod.rs
index 42b9654981..04b3236f2a 100644
--- a/sync/src/block/message/mod.rs
+++ b/sync/src/block/message/mod.rs
@@ -29,15 +29,13 @@ const MESSAGE_ID_GET_HEADERS: u8 = 0x02;
const MESSAGE_ID_HEADERS: u8 = 0x03;
const MESSAGE_ID_GET_BODIES: u8 = 0x04;
const MESSAGE_ID_BODIES: u8 = 0x05;
-const MESSAGE_ID_GET_STATE_HEAD: u8 = 0x06;
-const MESSAGE_ID_STATE_HEAD: u8 = 0x07;
-const MESSAGE_ID_GET_STATE_CHUNK: u8 = 0x08;
-const MESSAGE_ID_STATE_CHUNK: u8 = 0x09;
+const MESSAGE_ID_GET_STATE_CHUNK: u8 = 0x0a;
+const MESSAGE_ID_STATE_CHUNK: u8 = 0x0b;
#[derive(Debug, PartialEq)]
pub enum Message {
Status {
- total_score: U256,
+ nonce: U256,
best_hash: BlockHash,
genesis_hash: BlockHash,
},
@@ -49,7 +47,7 @@ impl Encodable for Message {
fn rlp_append(&self, s: &mut RlpStream) {
match self {
Message::Status {
- total_score,
+ nonce,
best_hash,
genesis_hash,
} => {
@@ -57,7 +55,7 @@ impl Encodable for Message {
s.append(&MESSAGE_ID_STATUS);
s.begin_list(3);
- s.append(total_score);
+ s.append(nonce);
s.append(best_hash);
s.append(genesis_hash);
}
@@ -99,7 +97,7 @@ impl Decodable for Message {
}
Ok(Message::Status {
- total_score: message.val_at(0)?,
+ nonce: message.val_at(0)?,
best_hash: message.val_at(1)?,
genesis_hash: message.val_at(2)?,
})
@@ -114,11 +112,10 @@ impl Decodable for Message {
let request_id = rlp.val_at(1)?;
let message = rlp.at(2)?;
match id {
- MESSAGE_ID_GET_HEADERS
- | MESSAGE_ID_GET_BODIES
- | MESSAGE_ID_GET_STATE_HEAD
- | MESSAGE_ID_GET_STATE_CHUNK => Ok(Message::Request(request_id, RequestMessage::decode(id, &message)?)),
- MESSAGE_ID_HEADERS | MESSAGE_ID_BODIES | MESSAGE_ID_STATE_HEAD | MESSAGE_ID_STATE_CHUNK => {
+ MESSAGE_ID_GET_HEADERS | MESSAGE_ID_GET_BODIES | MESSAGE_ID_GET_STATE_CHUNK => {
+ Ok(Message::Request(request_id, RequestMessage::decode(id, &message)?))
+ }
+ MESSAGE_ID_HEADERS | MESSAGE_ID_BODIES | MESSAGE_ID_STATE_CHUNK => {
Ok(Message::Response(request_id, ResponseMessage::decode(id, &message)?))
}
_ => Err(DecoderError::Custom("Unknown message id detected")),
@@ -137,7 +134,7 @@ mod tests {
#[test]
fn status_message_rlp() {
rlp_encode_and_decode_test!(Message::Status {
- total_score: U256::default(),
+ nonce: U256::zero(),
best_hash: H256::default().into(),
genesis_hash: H256::default().into(),
});
@@ -148,10 +145,4 @@ mod tests {
let request_id = 10;
rlp_encode_and_decode_test!(Message::Request(request_id, RequestMessage::Bodies(vec![])));
}
-
- #[test]
- fn request_state_head_rlp() {
- let request_id = 10;
- rlp_encode_and_decode_test!(Message::Request(request_id, RequestMessage::StateHead(H256::random().into())));
- }
}
diff --git a/sync/src/block/message/request.rs b/sync/src/block/message/request.rs
index 6c7ef53010..ddc11ecf85 100644
--- a/sync/src/block/message/request.rs
+++ b/sync/src/block/message/request.rs
@@ -25,11 +25,7 @@ pub enum RequestMessage {
max_count: u64,
},
Bodies(Vec<BlockHash>),
- StateHead(BlockHash),
- StateChunk {
- block_hash: BlockHash,
- tree_root: H256,
- },
+ StateChunk(BlockHash, Vec<H256>),
}
impl Encodable for RequestMessage {
@@ -46,17 +42,10 @@ impl Encodable for RequestMessage {
RequestMessage::Bodies(hashes) => {
s.append_list(hashes);
}
- RequestMessage::StateHead(block_hash) => {
- s.begin_list(1);
- s.append(block_hash);
- }
- RequestMessage::StateChunk {
- block_hash,
- tree_root,
- } => {
+ RequestMessage::StateChunk(block_hash, merkle_roots) => {
s.begin_list(2);
s.append(block_hash);
- s.append(tree_root);
+ s.append_list(merkle_roots);
}
};
}
@@ -69,7 +58,6 @@ impl RequestMessage {
..
} => super::MESSAGE_ID_GET_HEADERS,
RequestMessage::Bodies(..) => super::MESSAGE_ID_GET_BODIES,
- RequestMessage::StateHead(..) => super::MESSAGE_ID_GET_STATE_HEAD,
RequestMessage::StateChunk {
..
} => super::MESSAGE_ID_GET_STATE_CHUNK,
@@ -92,16 +80,6 @@ impl RequestMessage {
}
}
super::MESSAGE_ID_GET_BODIES => RequestMessage::Bodies(rlp.as_list()?),
- super::MESSAGE_ID_GET_STATE_HEAD => {
- let item_count = rlp.item_count()?;
- if item_count != 1 {
- return Err(DecoderError::RlpIncorrectListLen {
- got: item_count,
- expected: 1,
- })
- }
- RequestMessage::StateHead(rlp.val_at(0)?)
- }
super::MESSAGE_ID_GET_STATE_CHUNK => {
let item_count = rlp.item_count()?;
if item_count != 2 {
@@ -110,10 +88,7 @@ impl RequestMessage {
expected: 2,
})
}
- RequestMessage::StateChunk {
- block_hash: rlp.val_at(0)?,
- tree_root: rlp.val_at(1)?,
- }
+ RequestMessage::StateChunk(rlp.val_at(0)?, rlp.list_at(1)?)
}
_ => return Err(DecoderError::Custom("Unknown message id detected")),
};
@@ -149,18 +124,9 @@ mod tests {
assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
}
- #[test]
- fn request_state_head_message_rlp() {
- let message = RequestMessage::StateHead(H256::default().into());
- assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
- }
-
#[test]
fn request_state_chunk_message_rlp() {
- let message = RequestMessage::StateChunk {
- block_hash: H256::default().into(),
- tree_root: H256::default(),
- };
+ let message = RequestMessage::StateChunk(H256::default().into(), vec![H256::default()]);
assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
}
}
diff --git a/sync/src/block/message/response.rs b/sync/src/block/message/response.rs
index f6a16f2f97..823f26d490 100644
--- a/sync/src/block/message/response.rs
+++ b/sync/src/block/message/response.rs
@@ -24,8 +24,7 @@ use ctypes::Header;
pub enum ResponseMessage {
Headers(Vec<Header>),
Bodies(Vec<Vec<UnverifiedTransaction>>),
- StateHead(Vec<u8>),
- StateChunk(Vec<u8>),
+ StateChunk(Vec<Vec<u8>>),
}
impl Encodable for ResponseMessage {
@@ -53,13 +52,8 @@ impl Encodable for ResponseMessage {
s.append(&compressed);
}
- ResponseMessage::StateHead(bytes) => {
- s.begin_list(1);
- s.append(bytes);
- }
- ResponseMessage::StateChunk(bytes) => {
- s.begin_list(1);
- s.append(bytes);
+ ResponseMessage::StateChunk(chunks) => {
+ s.append_list::<Vec<u8>, Vec<u8>>(chunks);
}
};
}
@@ -72,7 +66,6 @@ impl ResponseMessage {
..
} => super::MESSAGE_ID_HEADERS,
ResponseMessage::Bodies(..) => super::MESSAGE_ID_BODIES,
- ResponseMessage::StateHead(..) => super::MESSAGE_ID_STATE_HEAD,
ResponseMessage::StateChunk {
..
} => super::MESSAGE_ID_STATE_CHUNK,
@@ -109,26 +102,7 @@ impl ResponseMessage {
}
ResponseMessage::Bodies(bodies)
}
- super::MESSAGE_ID_STATE_HEAD => {
- let item_count = rlp.item_count()?;
- if item_count != 1 {
- return Err(DecoderError::RlpIncorrectListLen {
- got: item_count,
- expected: 1,
- })
- }
- ResponseMessage::StateHead(rlp.val_at(0)?)
- }
- super::MESSAGE_ID_STATE_CHUNK => {
- let item_count = rlp.item_count()?;
- if item_count != 1 {
- return Err(DecoderError::RlpIncorrectListLen {
- got: item_count,
- expected: 1,
- })
- }
- ResponseMessage::StateChunk(rlp.val_at(0)?)
- }
+ super::MESSAGE_ID_STATE_CHUNK => ResponseMessage::StateChunk(rlp.as_list()?),
_ => return Err(DecoderError::Custom("Unknown message id detected")),
};
@@ -184,12 +158,6 @@ mod tests {
assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
}
- #[test]
- fn state_head_message_rlp() {
- let message = ResponseMessage::StateHead(vec![]);
- assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
- }
-
#[test]
fn state_chunk_message_rlp() {
let message = ResponseMessage::StateChunk(vec![]);
diff --git a/sync/src/lib.rs b/sync/src/lib.rs
index ff2fc0ab84..5a0b1c5910 100644
--- a/sync/src/lib.rs
+++ b/sync/src/lib.rs
@@ -25,7 +25,6 @@ extern crate codechain_state as cstate;
extern crate codechain_timer as ctimer;
extern crate codechain_types as ctypes;
-#[cfg(test)]
extern crate hashdb;
extern crate journaldb;
extern crate kvdb;
@@ -47,11 +46,10 @@ extern crate trie_standardmap;
extern crate util_error;
mod block;
-mod snapshot;
+pub mod snapshot;
mod transaction;
pub use crate::block::{BlockSyncEvent, BlockSyncExtension, BlockSyncSender};
-pub use crate::snapshot::SnapshotService;
pub use crate::transaction::TransactionSyncExtension;
#[cfg(test)]
diff --git a/sync/src/snapshot/error.rs b/sync/src/snapshot/error.rs
deleted file mode 100644
index dba9adc024..0000000000
--- a/sync/src/snapshot/error.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2018 Kodebox, Inc.
-// This file is part of CodeChain.
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::fmt::{Display, Formatter, Result as FormatResult};
-use std::io::{Error as FileError, ErrorKind};
-
-use primitives::H256;
-use util_error::UtilError;
-
-#[derive(Debug)]
-pub enum Error {
- NodeNotFound(H256),
- SyncError(String),
- FileError(ErrorKind),
- UtilError(UtilError),
-}
-
-impl From<FileError> for Error {
- fn from(error: FileError) -> Self {
- Error::FileError(error.kind())
- }
-}
-
-impl From<UtilError> for Error {
- fn from(error: UtilError) -> Self {
- Error::UtilError(error)
- }
-}
-
-impl Display for Error {
- fn fmt(&self, f: &mut Formatter) -> FormatResult {
- match self {
- Error::NodeNotFound(key) => write!(f, "State node not found: {:x}", key),
- Error::SyncError(reason) => write!(f, "Sync error: {}", reason),
- Error::FileError(kind) => write!(f, "File system error: {:?}", kind),
- Error::UtilError(error) => write!(f, "Util error: {:?}", error),
- }
- }
-}
diff --git a/sync/src/snapshot/mod.rs b/sync/src/snapshot/mod.rs
index f0e8c6bedc..7a075284a2 100644
--- a/sync/src/snapshot/mod.rs
+++ b/sync/src/snapshot/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2018 Kodebox, Inc.
+// Copyright 2019 Kodebox, Inc.
// This file is part of CodeChain.
//
// This program is free software: you can redistribute it and/or modify
@@ -14,9 +14,173 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-mod error;
-mod service;
-#[cfg_attr(feature = "cargo-clippy", allow(clippy::module_inception))]
-mod snapshot;
-pub use self::service::Service as SnapshotService;
+use std::fs;
+use std::path::PathBuf;
+use std::str::FromStr;
+use std::sync::Arc;
+use std::thread::{spawn, JoinHandle};
+
+use ccore::snapshot_notify::{NotifyReceiverSource, ReceiverCanceller};
+use ccore::{BlockChainClient, BlockChainTrait, BlockId, Client};
+use cmerkle::snapshot::{ChunkCompressor, Error as SnapshotError, Snapshot};
+use ctypes::BlockHash;
+use hashdb::{AsHashDB, HashDB};
+use primitives::H256;
+
+pub struct Service {
+ join_handle: Option<JoinHandle<()>>,
+ canceller: Option<ReceiverCanceller>,
+}
+
+pub fn snapshot_dir(root_dir: &str, block: &BlockHash) -> PathBuf {
+ let mut path = PathBuf::new();
+ path.push(root_dir);
+ path.push(format!("{:x}", **block));
+ path
+}
+
+pub fn snapshot_path(root_dir: &str, block: &BlockHash, chunk_root: &H256) -> PathBuf {
+ let mut path = snapshot_dir(root_dir, block);
+ path.push(format!("{:x}", chunk_root));
+ path
+}
+
+impl Service {
+ pub fn new(
+ client: Arc<Client>,
+ notify_receiver_source: NotifyReceiverSource,
+ root_dir: String,
+ expiration: Option<u64>,
+ ) -> Self {
+ let NotifyReceiverSource(canceller, receiver) = notify_receiver_source;
+ let join_handle = spawn(move || {
+ cinfo!(SYNC, "Snapshot service is on");
+ while let Ok(block_hash) = receiver.recv() {
+ cinfo!(SYNC, "Snapshot is requested for block: {}", block_hash);
+ let state_root = if let Some(header) = client.block_header(&BlockId::Hash(block_hash)) {
+ header.state_root()
+ } else {
+ cerror!(SYNC, "There isn't corresponding header for the requested block hash: {}", block_hash,);
+ continue
+ };
+ {
+ let db_lock = client.state_db().read();
+ if let Err(err) = snapshot(db_lock.as_hashdb(), block_hash, state_root, &root_dir) {
+ cerror!(
+ SYNC,
+ "Snapshot request failed for block: {}, chunk_root: {}, err: {}",
+ block_hash,
+ state_root,
+ err
+ );
+ } else {
+ cinfo!(SYNC, "Snapshot is ready for block: {}", block_hash)
+ }
+ }
+
+ if let Some(expiration) = expiration {
+ if let Err(err) = cleanup_expired(&client, &root_dir, expiration) {
+ cerror!(SYNC, "Snapshot cleanup error after block hash {}, err: {}", block_hash, err);
+ }
+ }
+ }
+ cinfo!(SYNC, "Snapshot service is stopped")
+ });
+
+ Self {
+ canceller: Some(canceller),
+ join_handle: Some(join_handle),
+ }
+ }
+}
+
+fn snapshot(db: &dyn HashDB, block_hash: BlockHash, chunk_root: H256, root_dir: &str) -> Result<(), SnapshotError> {
+ let snapshot_dir = snapshot_dir(root_dir, &block_hash);
+ fs::create_dir_all(snapshot_dir)?;
+
+ for chunk in Snapshot::from_hashdb(db, chunk_root) {
+ let chunk_path = snapshot_path(root_dir, &block_hash, &chunk.root);
+ let chunk_file = fs::File::create(chunk_path)?;
+ let compressor = ChunkCompressor::new(chunk_file);
+ compressor.compress_chunk(&chunk)?;
+ }
+
+ Ok(())
+}
+
+fn cleanup_expired(client: &Client, root_dir: &str, expiration: u64) -> Result<(), SnapshotError> {
+ for entry in fs::read_dir(root_dir)? {
+ let entry = match entry {
+ Ok(entry) => entry,
+ Err(err) => {
+ cerror!(SYNC, "Snapshot cleanup can't retrieve entry. err: {}", err);
+ continue
+ }
+ };
+ let path = entry.path();
+
+ match entry.file_type().map(|x| x.is_dir()) {
+ Ok(true) => {}
+ Ok(false) => continue,
+ Err(err) => {
+ cerror!(SYNC, "Snapshot cleanup can't retrieve file info: {}, err: {}", path.to_string_lossy(), err);
+ continue
+ }
+ }
+
+ let name = match path.file_name().expect("Directories always have file name").to_str() {
+ Some(n) => n,
+ None => continue,
+ };
+ let hash = match H256::from_str(name) {
+ Ok(h) => BlockHash::from(h),
+ Err(_) => continue,
+ };
+ let number = if let Some(number) = client.block_number(&BlockId::Hash(hash)) {
+ number
+ } else {
+ cerror!(SYNC, "Snapshot cleanup can't retrieve block number for block_hash: {}", hash);
+ continue
+ };
+
+ if number + expiration < client.best_block_header().number() {
+ cleanup_snapshot(root_dir, hash)
+ }
+ }
+ Ok(())
+}
+
+/// Remove all files in `root_dir/block_hash`
+fn cleanup_snapshot(root_dir: &str, block_hash: BlockHash) {
+ let path = snapshot_dir(root_dir, &block_hash);
+ let rename_to = PathBuf::from(root_dir).join(format!("{:x}.old", *block_hash));
+ // It is okay to ignore errors. We just wanted them to be removed.
+ match fs::rename(path, &rename_to) {
+ Ok(()) => {}
+ Err(err) => {
+ cerror!(SYNC, "Snapshot cleanup: renaming {} failed, reason: {}", block_hash, err);
+ }
+ }
+ // Ignore the error. Cleanup failure is not a critical error.
+ match fs::remove_dir_all(rename_to) {
+ Ok(()) => {}
+ Err(err) => {
+ cerror!(SYNC, "Snapshot cleanup: removing {} failed, reason: {}", block_hash, err);
+ }
+ }
+}
+
+impl Drop for Service {
+ fn drop(&mut self) {
+ if let Some(canceller) = self.canceller.take() {
+ // The thread corresponding to the `self.join_handle` waits for the `self.canceller` is dropped.
+ // It must be dropped first not to make deadlock at `handle.join()`.
+ drop(canceller);
+ }
+
+ if let Some(handle) = self.join_handle.take() {
+ handle.join().expect("Snapshot service thread shouldn't panic");
+ }
+ }
+}
diff --git a/sync/src/snapshot/service.rs b/sync/src/snapshot/service.rs
deleted file mode 100644
index e076dc3661..0000000000
--- a/sync/src/snapshot/service.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright 2018 Kodebox, Inc.
-// This file is part of CodeChain.
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::io::ErrorKind;
-use std::path::PathBuf;
-use std::sync::Arc;
-use std::thread::spawn;
-
-use ccore::{BlockChainClient, BlockChainTrait, BlockId, ChainNotify, Client, DatabaseClient};
-use ctypes::BlockHash;
-
-use super::error::Error;
-use super::snapshot::{Snapshot, WriteSnapshot};
-
-pub struct Service {
- client: Arc<Client>,
- /// Snapshot root directory
- root_dir: String,
- /// Snapshot creation period in unit of block numbers
- period: u64,
-}
-
-impl Service {
- pub fn new(client: Arc<Client>, root_dir: String, period: u64) -> Arc<Self> {
- Arc::new(Self {
- client,
- root_dir,
- period,
- })
- }
-}
-
-impl ChainNotify for Service {
- /// fires when chain has new blocks.
- fn new_blocks(
- &self,
- _imported: Vec<BlockHash>,
- _invalid: Vec<BlockHash>,
- enacted: Vec<BlockHash>,
- _retracted: Vec<BlockHash>,
- _sealed: Vec<BlockHash>,
- ) {
- let best_number = self.client.chain_info().best_block_number;
- let is_checkpoint = enacted
- .iter()
- .map(|hash| self.client.block_number(&BlockId::Hash(*hash)).expect("Enacted block must exist"))
- .any(|number| number % self.period == 0);
- if is_checkpoint && best_number > self.period {
- let number = (best_number / self.period - 1) * self.period;
- let header = self.client.block_header(&BlockId::Number(number)).expect("Snapshot target must exist");
-
- let db = self.client.database();
- let path: PathBuf = [self.root_dir.clone(), format!("{:x}", *header.hash())].iter().collect();
- let root = header.state_root();
- // FIXME: The db can be corrupted because the CodeChain doesn't wait child threads end on exit.
- spawn(move || match Snapshot::try_new(path).map(|s| s.write_snapshot(db.as_ref(), &root)) {
- Ok(_) => {}
- Err(Error::FileError(ErrorKind::AlreadyExists)) => {}
- Err(e) => cerror!(SNAPSHOT, "{}", e),
- });
- }
- }
-}
diff --git a/sync/src/snapshot/snapshot.rs b/sync/src/snapshot/snapshot.rs
deleted file mode 100644
index f5a363540f..0000000000
--- a/sync/src/snapshot/snapshot.rs
+++ /dev/null
@@ -1,393 +0,0 @@
-// Copyright 2018-2019 Kodebox, Inc.
-// This file is part of CodeChain.
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::collections::HashSet;
-use std::convert::AsRef;
-use std::fs::{create_dir_all, File};
-use std::io::{Read, Write};
-use std::iter::once;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-use ccore::COL_STATE;
-use cmerkle::Node;
-use journaldb::{self, Algorithm, JournalDB};
-use kvdb::KeyValueDB;
-use primitives::H256;
-use rlp::{Rlp, RlpStream};
-use snap;
-
-use super::error::Error;
-
-pub struct Snapshot {
- path: PathBuf,
-}
-
-impl Snapshot {
- pub fn try_new<P>(path: P) -> Result<Self, Error>
- where
- P: AsRef<Path>, {
- create_dir_all(&path)?;
- Ok(Snapshot {
- path: path.as_ref().to_owned(),
- })
- }
-}
-
-impl Snapshot {
- fn file_for(&self, root: &H256) -> PathBuf {
- self.path.join(format!("{:x}", root))
- }
-
- fn write_nodes<'a, I>(&self, root: &H256, iter: I) -> Result<(), Error>
- where
- I: IntoIterator<Item = &'a (H256, Vec<u8>)>, {
- let file = File::create(self.file_for(root))?;
- let mut snappy = snap::Writer::new(file);
-
- let mut stream = RlpStream::new();
- stream.begin_unbounded_list();
- for (key, value) in iter {
- stream.begin_list(2);
- stream.append(key);
- stream.append(value);
- }
- stream.complete_unbounded_list();
-
- snappy.write_all(&stream.drain())?;
- Ok(())
- }
-
- fn read_chunk(&self, backing: Arc<dyn KeyValueDB>, root: &H256) -> Result<Chunk, Error> {
- let file = File::open(self.file_for(root))?;
- let mut buf = Vec::new();
- let mut snappy = snap::Reader::new(file);
- snappy.read_to_end(&mut buf)?;
-
- let rlp = Rlp::new(&buf);
- let mut journal = journaldb::new(backing, Algorithm::Archive, COL_STATE);
- let mut inserted_keys = HashSet::new();
- let mut referenced_keys = HashSet::new();
- referenced_keys.insert(*root);
- for rlp_pair in rlp.iter() {
- if rlp_pair.item_count().unwrap() != 2 {
- return Err(Error::SyncError("Chunk contains invalid size of pair".to_string()))
- }
-
- let key = rlp_pair.val_at(0).unwrap();
- let value: Vec<_> = rlp_pair.val_at(1).unwrap();
-
- let node =
- Node::decoded(&value).ok_or_else(|| Error::SyncError("Chunk condtains an invalid node".to_string()))?;
-
- if journal.contains(&key) {
- cwarn!(SNAPSHOT, "Chunk contains duplicated key: {}", key);
- }
-
- if let Node::Branch(_, childs) = node {
- for child in childs.iter() {
- if let Some(child) = child {
- referenced_keys.insert(*child);
- }
- }
- }
-
- let hash_key = journal.insert(&value);
- if hash_key != key {
- return Err(Error::SyncError("Chunk contains an invalid key for a value".to_string()))
- }
- inserted_keys.insert(hash_key);
- }
-
- let never_referenced_keys: Vec<H256> =
- inserted_keys.iter().filter(|key| !referenced_keys.contains(key)).cloned().collect();
-
- Ok(Chunk {
- journal,
- never_referenced_keys,
- })
- }
-}
-
-struct Chunk {
- journal: Box<dyn JournalDB>,
- never_referenced_keys: Vec<H256>,
-}
-
-impl Chunk {
- fn purge(&mut self) -> bool {
- if self.never_referenced_keys.is_empty() {
- return false
- }
- for key in &self.never_referenced_keys {
- self.journal.remove(key);
- }
- self.never_referenced_keys.clear();
- true
- }
-
- fn is_deeper_than(&self, root: &H256, max_depth: usize) -> bool {
- let mut stack = Vec::new();
- stack.push((*root, 0));
- while let Some((key, depth)) = stack.pop() {
- match self.journal.get(&key) {
- None => continue,
- Some(_) if depth >= max_depth => return false,
- Some(value) => {
- if let Some(Node::Branch(_, childs)) = Node::decoded(&value) {
- for child in childs.iter() {
- if let Some(child) = child {
- stack.push((*child, depth + 1));
- }
- }
- }
- }
- }
- }
- false
- }
-
- fn missing_keys(&self, root: &H256) -> Vec<H256> {
- let mut result = Vec::new();
- let mut stack = Vec::new();
- stack.push(*root);
- while let Some(key) = stack.pop() {
- match self.journal.get(&key) {
- None => {
- result.push(key);
- }
- Some(value) => {
- if let Some(Node::Branch(_, childs)) = Node::decoded(&value) {
- for child in childs.iter() {
- if let Some(child) = child {
- stack.push(*child);
- }
- }
- }
- }
- }
- }
- result
- }
-}
-
-pub trait WriteSnapshot {
- fn write_snapshot(&self, db: &dyn KeyValueDB, root: &H256) -> Result<(), Error>;
-}
-
-pub trait ReadSnapshot {
- fn read_snapshot(&self, db: Arc<dyn KeyValueDB>, root: &H256) -> Result<(), Error>;
-}
-
-impl WriteSnapshot for Snapshot {
- fn write_snapshot(&self, db: &dyn KeyValueDB, root: &H256) -> Result<(), Error> {
- let root_val = match db.get(COL_STATE, root) {
- Ok(Some(value)) => value.to_vec(),
- Ok(None) => return Err(Error::SyncError("Invalid state root, or the database is empty".to_string())),
- Err(e) => return Err(e.into()),
- };
-
- let children = children_of(db, &root_val)?;
- let mut grandchildren = Vec::new();
- for (_, value) in &children {
- grandchildren.extend(children_of(db, value)?);
- }
-
- self.write_nodes(root, once(&(*root, root_val)).chain(&children))?;
- for (grandchild, _) in &grandchildren {
- let nodes = enumerate_subtree(db, grandchild)?;
- self.write_nodes(grandchild, &nodes)?;
- }
-
- Ok(())
- }
-}
-
-impl ReadSnapshot for Snapshot {
- fn read_snapshot(&self, db: Arc<dyn KeyValueDB>, root: &H256) -> Result<(), Error> {
- let head = {
- let mut head = self.read_chunk(db.clone(), root)?;
- if head.purge() {
- cinfo!(SNAPSHOT, "Head chunk contains garbages");
- }
-
- if head.is_deeper_than(root, 2) {
- return Err(Error::SyncError("Head chunk has an invalid shape".to_string()))
- }
-
- let mut transaction = db.transaction();
- head.journal.inject(&mut transaction)?;
- db.write_buffered(transaction);
- head
- };
-
- for chunk_root in head.missing_keys(root) {
- let mut chunk = self.read_chunk(db.clone(), &chunk_root)?;
- if chunk.purge() {
- cinfo!(SNAPSHOT, "Chunk contains garbages");
- }
-
- if !chunk.missing_keys(&chunk_root).is_empty() {
- return Err(Error::SyncError("Chunk is an incomplete trie".to_string()))
- }
-
- let mut transaction = db.transaction();
- chunk.journal.inject(&mut transaction)?;
- db.write_buffered(transaction);
- }
-
- Ok(())
- }
-}
-
-fn get_node(db: &dyn KeyValueDB, key: &H256) -> Result<Vec<u8>, Error> {
- match db.get(COL_STATE, key) {
- Ok(Some(value)) => Ok(value.to_vec()),
- Ok(None) => Err(Error::NodeNotFound(*key)),
- Err(e) => Err(e.into()),
- }
-}
-
-fn children_of(db: &dyn KeyValueDB, node: &[u8]) -> Result<Vec<(H256, Vec<u8>)>, Error> {
- let keys = match Node::decoded(node) {
- None => Vec::new(),
- Some(Node::Leaf(..)) => Vec::new(),
- Some(Node::Branch(_, children)) => children.iter().filter_map(|child| *child).collect(),
- };
-
- let mut result = Vec::new();
- for key in keys {
- result.push((key, get_node(db, &key)?));
- }
- Ok(result)
-}
-
-fn enumerate_subtree(db: &dyn KeyValueDB, root: &H256) -> Result<Vec<(H256, Vec<u8>)>, Error> {
- let node = get_node(db, root)?;
- let children = match Node::decoded(&node) {
- None => Vec::new(),
- Some(Node::Leaf(..)) => Vec::new(),
- Some(Node::Branch(_, children)) => children.iter().filter_map(|child| *child).collect(),
- };
- let mut result: Vec<_> = vec![(*root, node)];
- for child in children {
- result.extend(enumerate_subtree(db, &child)?);
- }
- Ok(result)
-}
-
-#[cfg(test)]
-mod tests {
- use std::collections::HashSet;
- use std::sync::Arc;
-
- use ccore::COL_STATE;
-
- use cmerkle::{Trie, TrieDB, TrieDBMut, TrieMut};
- use journaldb;
- use journaldb::Algorithm;
- use kvdb_memorydb;
- use primitives::H256;
- use tempfile::tempdir;
- use trie_standardmap::{Alphabet, StandardMap, ValueMode};
-
- use super::{ReadSnapshot, Snapshot, WriteSnapshot};
-
- #[test]
- fn init() {
- let snapshot_dir = tempdir().unwrap();
- let snapshot = Snapshot::try_new(&snapshot_dir).unwrap();
- let mut root = H256::new();
-
- let kvdb = Arc::new(kvdb_memorydb::create(1));
- let mut jdb = journaldb::new(kvdb.clone(), Algorithm::Archive, COL_STATE);
- {
- let _ = TrieDBMut::new(jdb.as_hashdb_mut(), &mut root);
- }
- /* do nothing */
- let result = snapshot.write_snapshot(kvdb.as_ref(), &root);
-
- assert!(result.is_err());
- }
-
- fn random_insert_and_restore_with_count(count: usize) {
- let mut seed = H256::new();
- let x = StandardMap {
- alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()),
- min_key: 5,
- journal_key: 0,
- value_mode: ValueMode::Index,
- count,
- }
- .make_with(&mut seed);
-
- let snapshot_dir = tempdir().unwrap();
- let snapshot = Snapshot::try_new(&snapshot_dir).unwrap();
- let mut root = H256::new();
- {
- let kvdb = Arc::new(kvdb_memorydb::create(1));
- let mut jdb = journaldb::new(kvdb.clone(), Algorithm::Archive, COL_STATE);
- {
- let mut t = TrieDBMut::new(jdb.as_hashdb_mut(), &mut root);
- let mut inserted_keys = HashSet::new();
- for &(ref key, ref value) in &x {
- if !inserted_keys.insert(key) {
- continue
- }
- assert!(t.insert(key, value).unwrap().is_none());
- assert_eq!(t.insert(key, value).unwrap(), Some(value.to_vec()));
- }
- }
- {
- let mut batch = jdb.backing().transaction();
- let _ = jdb.inject(&mut batch).unwrap();
- jdb.backing().write(batch).unwrap();
- }
-
- snapshot.write_snapshot(kvdb.as_ref(), &root).unwrap();
- }
-
- {
- let kvdb = Arc::new(kvdb_memorydb::create(1));
- snapshot.read_snapshot(kvdb.clone(), &root).unwrap();
-
- let mut jdb = journaldb::new(kvdb, Algorithm::Archive, COL_STATE);
- let t = TrieDB::try_new(jdb.as_hashdb_mut(), &root).unwrap();
- let mut inserted_keys = HashSet::new();
- for &(ref key, ref value) in &x {
- if !inserted_keys.insert(key) {
- continue
- }
- assert_eq!(t.get(key).unwrap(), Some(value.to_vec()));
- }
- }
- }
-
- #[test]
- fn random_insert_and_restore_1() {
- random_insert_and_restore_with_count(1);
- }
-
- #[test]
- fn random_insert_and_restore_100() {
- random_insert_and_restore_with_count(100);
- }
-
- #[test]
- fn random_insert_and_restore_10000() {
- random_insert_and_restore_with_count(10000);
- }
-}
diff --git a/test/src/e2e.dynval/2/snapshot.test.ts b/test/src/e2e.dynval/2/snapshot.test.ts
new file mode 100644
index 0000000000..bb706a2c45
--- /dev/null
+++ b/test/src/e2e.dynval/2/snapshot.test.ts
@@ -0,0 +1,187 @@
+// Copyright 2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import * as chai from "chai";
+import { expect } from "chai";
+import * as chaiAsPromised from "chai-as-promised";
+import * as stake from "codechain-stakeholder-sdk";
+import * as fs from "fs";
+import "mocha";
+import * as path from "path";
+
+import mkdirp = require("mkdirp");
+import { validators } from "../../../tendermint.dynval/constants";
+import { faucetAddress, faucetSecret } from "../../helper/constants";
+import { PromiseExpect } from "../../helper/promise";
+import CodeChain, { Signer } from "../../helper/spawn";
+import { setTermTestTimeout, withNodes } from "../setup";
+
+chai.use(chaiAsPromised);
+
+const SNAPSHOT_CONFIG = `${__dirname}/../../../tendermint.dynval/snapshot-config.yml`;
+const SNAPSHOT_PATH = `${__dirname}/../../../../snapshot/`;
+
+describe("Snapshot for Tendermint with Dynamic Validator", function() {
+ const promiseExpect = new PromiseExpect();
+ const snapshotValidators = validators.slice(0, 3);
+ const freshNodeValidator = validators[3];
+ const { nodes } = withNodes(this, {
+ promiseExpect,
+ overrideParams: {
+ maxNumOfValidators: 3,
+ era: 1
+ },
+ validators: snapshotValidators.map((signer, index) => ({
+ signer,
+ delegation: 5000,
+ deposit: 10_000_000 - index // tie-breaker
+ })),
+ modify: () => {
+ mkdirp.sync(SNAPSHOT_PATH);
+ const snapshotPath = fs.mkdtempSync(SNAPSHOT_PATH);
+ return {
+ additionalArgv: [
+ "--snapshot-path",
+ snapshotPath,
+ "--config",
+ SNAPSHOT_CONFIG
+ ],
+ nodeAdditionalProperties: {
+ snapshotPath
+ }
+ };
+ }
+ });
+
+ it("should be exist after some time", async function() {
+ const termWaiter = setTermTestTimeout(this, {
+ terms: 2
+ });
+ const termMetadata = await termWaiter.waitNodeUntilTerm(nodes[0], {
+ target: 2,
+ termPeriods: 1
+ });
+ const snapshotBlock = await getSnapshotBlock(nodes[0], termMetadata);
+ expect(
+ path.join(
+ nodes[0].snapshotPath,
+ snapshotBlock.hash.toString(),
+ snapshotBlock.stateRoot.toString()
+ )
+ ).to.satisfy(fs.existsSync);
+ });
+
+ it("should be able to boot with the snapshot", async function() {
+ const termWaiter = setTermTestTimeout(this, {
+ terms: 3
+ });
+ const termMetadata1 = await termWaiter.waitNodeUntilTerm(nodes[0], {
+ target: 2,
+ termPeriods: 1
+ });
+ const snapshotBlock = await getSnapshotBlock(nodes[0], termMetadata1);
+ await makeItValidator(nodes[0], freshNodeValidator);
+ const snapshotPath = fs.mkdtempSync(SNAPSHOT_PATH);
+ const node = new CodeChain({
+ chain: `${__dirname}/../../scheme/tendermint-dynval.json`,
+ argv: [
+ "--engine-signer",
+ freshNodeValidator.platformAddress.toString(),
+ "--password-path",
+ `test/tendermint.dynval/${freshNodeValidator.platformAddress.value}/password.json`,
+ "--force-sealing",
+ "--snapshot-path",
+ snapshotPath,
+ "--config",
+ SNAPSHOT_CONFIG,
+ "--snapshot-hash",
+ snapshotBlock.hash.toString(),
+ "--snapshot-number",
+ snapshotBlock.number.toString()
+ ],
+ additionalKeysPath: `tendermint.dynval/${freshNodeValidator.platformAddress.value}/keys`
+ });
+ try {
+ await node.start();
+ await node.connect(nodes[0]);
+ await termWaiter.waitNodeUntilTerm(node, {
+ target: 4,
+ termPeriods: 2
+ });
+
+ // Check that the freshNodeValidator is still a validator & make sure it doesn't have a block/header before termMetadata1.
+ } catch (e) {
+ node.keepLogs();
+ throw e;
+ } finally {
+ await node.clean();
+ }
+ });
+
+ afterEach(async function() {
+ promiseExpect.checkFulfilled();
+ });
+});
+
+async function getSnapshotBlock(
+ node: CodeChain,
+ termMetadata: stake.TermMetadata
+) {
+ const blockNumber = termMetadata.lastTermFinishedBlockNumber + 1;
+ await node.waitBlockNumber(blockNumber);
+ return (await node.sdk.rpc.chain.getBlock(blockNumber))!;
+}
+
+async function makeItValidator(node: CodeChain, freshNodeValidator: Signer) {
+ const faucetSeq = await node.sdk.rpc.chain.getSeq(faucetAddress);
+ const payTx = node.sdk.core
+ .createPayTransaction({
+ recipient: freshNodeValidator.platformAddress,
+ quantity: 200000000
+ })
+ .sign({
+ secret: faucetSecret,
+ seq: faucetSeq,
+ fee: 10
+ });
+ await node.waitForTx(await node.sdk.rpc.chain.sendSignedTransaction(payTx));
+ const selfNominateTx = stake
+ .createSelfNominateTransaction(node.sdk, 10000000, "")
+ .sign({
+ secret: freshNodeValidator.privateKey,
+ seq: await node.sdk.rpc.chain.getSeq(
+ freshNodeValidator.platformAddress
+ ),
+ fee: 10
+ });
+ await node.waitForTx(
+ await node.sdk.rpc.chain.sendSignedTransaction(selfNominateTx)
+ );
+ const delegateTx = stake
+ .createDelegateCCSTransaction(
+ node.sdk,
+ freshNodeValidator.platformAddress,
+ 5000
+ )
+ .sign({
+ secret: faucetSecret,
+ seq: faucetSeq + 1,
+ fee: 10
+ });
+ await node.waitForTx(
+ await node.sdk.rpc.chain.sendSignedTransaction(delegateTx)
+ );
+}
diff --git a/test/src/e2e.dynval/setup.ts b/test/src/e2e.dynval/setup.ts
index 493f6ef602..6cc695e2fb 100644
--- a/test/src/e2e.dynval/setup.ts
+++ b/test/src/e2e.dynval/setup.ts
@@ -39,17 +39,29 @@ interface ValidatorConfig {
delegation?: U64Value;
}
-export function withNodes(
+interface NodePropertyModifier<T> {
+ additionalArgv: string[];
+ nodeAdditionalProperties: T;
+}
+
+export function withNodes<T>(
suite: Suite,
options: {
promiseExpect: PromiseExpect;
validators: ValidatorConfig[];
overrideParams?: Partial<CommonParams>;
onBeforeEnable?: (nodes: CodeChain[]) => Promise<void>;
+ modify?: (signer: Signer, index: number) => NodePropertyModifier<T>;
}
) {
- const nodes: CodeChain[] = [];
- const { overrideParams = {} } = options;
+ const nodes: (CodeChain & T)[] = [];
+ const {
+ overrideParams = {},
+ modify = () => ({
+ additionalArgv: [],
+ nodeAdditionalProperties: {} as T
+ })
+ } = options;
const initialParams = {
...defaultParams,
...overrideParams
@@ -62,7 +74,8 @@ export function withNodes(
nodes.length = 0;
const newNodes = await createNodes({
...options,
- initialParams
+ initialParams,
+ modify
});
nodes.push(...newNodes);
});
@@ -95,14 +108,15 @@ export function findNode(nodes: CodeChain[], signer: Signer) {
);
}
-async function createNodes(options: {
+async function createNodes<T>(options: {
promiseExpect: PromiseExpect;
validators: ValidatorConfig[];
initialParams: CommonParams;
onBeforeEnable?: (nodes: CodeChain[]) => Promise<void>;
-}): Promise<CodeChain[]> {
+ modify: (signer: Signer, index: number) => NodePropertyModifier<T>;
+}): Promise<(CodeChain & T)[]> {
const chain = `${__dirname}/../scheme/tendermint-dynval.json`;
- const { promiseExpect, validators, initialParams } = options;
+ const { promiseExpect, validators, initialParams, modify } = options;
const initialNodes: CodeChain[] = [];
const initialValidators = [
@@ -124,20 +138,23 @@ async function createNodes(options: {
});
}
- const nodes: CodeChain[] = [];
+ const nodes: (CodeChain & T)[] = [];
for (let i = 0; i < validators.length; i++) {
const { signer: validator } = validators[i];
- nodes[i] = new CodeChain({
+ const modifier = modify(validator, i);
+ const node = new CodeChain({
chain,
argv: [
"--engine-signer",
validator.platformAddress.value,
"--password-path",
`test/tendermint.dynval/${validator.platformAddress.value}/password.json`,
- "--force-sealing"
+ "--force-sealing",
+ ...modifier.additionalArgv
],
additionalKeysPath: `tendermint.dynval/${validator.platformAddress.value}/keys`
});
+ nodes[i] = Object.assign(node, modifier.nodeAdditionalProperties);
nodes[i].signer = validator;
}
let bootstrapFailed = false;
@@ -451,7 +468,7 @@ interface TermWaiter {
target: number;
termPeriods: number;
}
- ): Promise<void>;
+ ): Promise<stake.TermMetadata>;
}
export function setTermTestTimeout(
@@ -483,7 +500,7 @@ export function setTermTestTimeout(
termPeriods: number;
}
) {
- await node.waitForTermChange(
+ return await node.waitForTermChange(
waiterParams.target,
termPeriodsToTime(waiterParams.termPeriods, 0.5)
);
diff --git a/test/src/e2e/snapshot.test.ts b/test/src/e2e/snapshot.test.ts
new file mode 100644
index 0000000000..195cf00c8a
--- /dev/null
+++ b/test/src/e2e/snapshot.test.ts
@@ -0,0 +1,120 @@
+// Copyright 2018-2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import { expect } from "chai";
+import * as fs from "fs";
+import "mocha";
+import * as path from "path";
+
+import { aliceAddress } from "../helper/constants";
+import CodeChain from "../helper/spawn";
+
+const SNAPSHOT_PATH = `${__dirname}/../../../snapshot/`;
+
+describe("Snapshot", async function() {
+ let node: CodeChain;
+ before(async function() {
+ node = new CodeChain({
+ argv: ["--snapshot-path", SNAPSHOT_PATH]
+ });
+ await node.start();
+ });
+
+ it("can make a snapshot when it is requsted with devel rpc", async function() {
+ const pay = await node.sendPayTx({
+ quantity: 100,
+ recipient: aliceAddress
+ });
+
+ const blockHash = (await node.sdk.rpc.chain.getTransaction(pay.hash()))!
+ .blockHash!;
+ await node.sdk.rpc.sendRpcRequest("devel_snapshot", [
+ blockHash.toJSON()
+ ]);
+ // Wait for 1 secs
+ await new Promise(resolve => setTimeout(resolve, 1000));
+
+ const stateRoot = (await node.sdk.rpc.chain.getBlock(blockHash))!
+ .stateRoot;
+ expect(
+ path.join(SNAPSHOT_PATH, blockHash.toString(), stateRoot.toString())
+ ).to.satisfies(fs.existsSync);
+ });
+
+ it("can restore from snapshot", async function() {
+ for (let i = 0; i < 10; i++) {
+ const tx = await node.sendPayTx({
+ quantity: 100,
+ recipient: aliceAddress
+ });
+ await node.waitForTx(tx.hash());
+ }
+
+ const pay = await node.sendPayTx({
+ quantity: 100,
+ recipient: aliceAddress
+ });
+
+ const blockHash = (await node.sdk.rpc.chain.getTransaction(pay.hash()))!
+ .blockHash!;
+
+ const block = (await node.sdk.rpc.chain.getBlock(blockHash))!;
+ await node.sdk.rpc.sendRpcRequest("devel_snapshot", [
+ blockHash.toJSON()
+ ]);
+ // Wait for 1 secs
+ await new Promise(resolve => setTimeout(resolve, 1000));
+
+ const newNode = new CodeChain({
+ argv: [
+ "--snapshot-hash",
+ block.hash.toString(),
+ "--snapshot-number",
+ block.number.toString()
+ ]
+ });
+
+ try {
+ await newNode.start();
+ await newNode.connect(node);
+ await newNode.waitBlockNumber(block.number);
+ await node.sdk.rpc.devel.stopSealing();
+ // New node creates block
+ const newPay = await newNode.sendPayTx({
+ quantity: 100,
+ recipient: aliceAddress
+ });
+ await newNode.waitForTx(newPay.hash());
+ await node.sdk.rpc.devel.startSealing();
+ await node.waitForTx(newPay.hash());
+ } catch (e) {
+ newNode.keepLogs();
+ throw e;
+ } finally {
+ await newNode.clean();
+ }
+ });
+
+ afterEach(function() {
+ if (this.currentTest!.state === "failed") {
+ node.keepLogs();
+ }
+ });
+
+ after(async function() {
+ await node.clean();
+ });
+});
diff --git a/test/src/helper/mock/blockSyncMessage.ts b/test/src/helper/mock/blockSyncMessage.ts
index d064d825b4..19385edd0d 100644
--- a/test/src/helper/mock/blockSyncMessage.ts
+++ b/test/src/helper/mock/blockSyncMessage.ts
@@ -64,7 +64,9 @@ export class BlockSyncMessage {
if (msgId === MessageType.MESSAGE_ID_STATUS) {
Emitter.emit("status");
const msg = decodedmsg[1];
- const totalScore = new U256(parseInt(msg[0].toString("hex"), 16));
+ const totalScore = new U256(
+ parseInt(msg[0].toString("hex"), 16) || 0
+ );
const bestHash = new H256(msg[1].toString("hex"));
const genesisHash = new H256(msg[2].toString("hex"));
return new BlockSyncMessage({
diff --git a/test/src/helper/spawn.ts b/test/src/helper/spawn.ts
index 4b2eb4bc95..f2d1cf396b 100644
--- a/test/src/helper/spawn.ts
+++ b/test/src/helper/spawn.ts
@@ -859,14 +859,16 @@ export default class CodeChain {
public async waitForTermChange(target: number, timeout?: number) {
const start = Date.now();
while (true) {
- const termMetadata = await stake.getTermMetadata(this.sdk);
+ const termMetadata = (await stake.getTermMetadata(this.sdk))!;
if (termMetadata && termMetadata.currentTermId >= target) {
- break;
+ return termMetadata;
}
await wait(1000);
if (timeout) {
if (Date.now() - start > timeout * 1000) {
- throw new Error(`Term didn't changed in ${timeout} s`);
+ throw new Error(
+ `Term didn't changed to ${target} in ${timeout} s. It is ${termMetadata.currentTermId} now`
+ );
}
}
}
diff --git a/test/tendermint.dynval/snapshot-config.yml b/test/tendermint.dynval/snapshot-config.yml
new file mode 100644
index 0000000000..9f5c890280
--- /dev/null
+++ b/test/tendermint.dynval/snapshot-config.yml
@@ -0,0 +1,19 @@
+[codechain]
+
+[mining]
+
+[network]
+
+[rpc]
+
+[ipc]
+
+[ws]
+
+[snapshot]
+disable = false
+
+[stratum]
+
+[email_alarm]
+
diff --git a/util/merkle/Cargo.toml b/util/merkle/Cargo.toml
index deb92626de..f95f3f8005 100644
--- a/util/merkle/Cargo.toml
+++ b/util/merkle/Cargo.toml
@@ -8,8 +8,11 @@ edition = "2018"
rand = "0.6.1"
hashdb = {path = "../hashdb" }
codechain-crypto = { git = "https://github.com/CodeChain-io/rust-codechain-crypto.git", version = "0.1" }
+memorydb = { path = "../memorydb" }
primitives = { git = "https://github.com/CodeChain-io/rust-codechain-primitives.git", version = "0.4" }
rlp = { git = "https://github.com/CodeChain-io/rlp.git", version = "0.4" }
+rlp_derive = { path = "../rlp_derive" }
+snap = "0.2"
[dev-dependencies]
journaldb = { path = "../journaldb" }
diff --git a/util/merkle/src/lib.rs b/util/merkle/src/lib.rs
index 80acf9399a..cf40cc9e46 100644
--- a/util/merkle/src/lib.rs
+++ b/util/merkle/src/lib.rs
@@ -20,6 +20,9 @@ extern crate hashdb;
extern crate memorydb;
extern crate primitives;
extern crate rlp;
+#[macro_use]
+extern crate rlp_derive;
+extern crate snap;
#[cfg(test)]
extern crate trie_standardmap as standardmap;
@@ -33,6 +36,8 @@ use primitives::H256;
mod nibbleslice;
pub mod node;
mod skewed;
+#[allow(dead_code)]
+pub mod snapshot;
pub mod triedb;
pub mod triedbmut;
pub mod triehash;
diff --git a/util/merkle/src/nibbleslice.rs b/util/merkle/src/nibbleslice.rs
index d39714c21d..d979860809 100644
--- a/util/merkle/src/nibbleslice.rs
+++ b/util/merkle/src/nibbleslice.rs
@@ -17,7 +17,7 @@ use std::cmp::*;
use std::fmt;
-#[derive(Eq, Ord)]
+#[derive(Eq, Ord, Copy, Clone)]
pub struct NibbleSlice<'a> {
pub data: &'a [u8],
pub offset: usize,
diff --git a/util/merkle/src/node.rs b/util/merkle/src/node.rs
index 66f2704808..4d556860d6 100644
--- a/util/merkle/src/node.rs
+++ b/util/merkle/src/node.rs
@@ -112,4 +112,11 @@ impl<'a> Node<'a> {
}
}
}
+
+ pub fn mid(self, offset: usize) -> Self {
+ match self {
+ Node::Leaf(partial, value) => Node::Leaf(partial.mid(offset), value),
+ Node::Branch(partial, child) => Node::Branch(partial.mid(offset), child),
+ }
+ }
}
diff --git a/util/merkle/src/snapshot/chunk.rs b/util/merkle/src/snapshot/chunk.rs
new file mode 100644
index 0000000000..40ee320c13
--- /dev/null
+++ b/util/merkle/src/snapshot/chunk.rs
@@ -0,0 +1,284 @@
+// Copyright 2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::collections::VecDeque;
+use std::convert::From;
+
+use ccrypto::BLAKE_NULL_RLP;
+use hashdb::{DBValue, HashDB};
+use primitives::H256;
+
+use super::error::{ChunkError, Error};
+use super::{DecodedPathSlice, PathSlice, CHUNK_HEIGHT};
+use crate::nibbleslice::NibbleSlice;
+use crate::{Node, TrieDBMut};
+
+#[derive(RlpEncodable, RlpDecodable, Eq, PartialEq)]
+pub struct TerminalNode {
+ // Relative path from the chunk root.
+ pub path_slice: PathSlice,
+ pub node_rlp: Vec<u8>,
+}
+
+impl std::fmt::Debug for TerminalNode {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+ let path_slice = NibbleSlice::from_encoded(&self.path_slice);
+ f.debug_struct("TerminalNode")
+ .field("path_slice", &path_slice)
+ .field("node_rlp", &NodeDebugAdaptor {
+ rlp: &self.node_rlp,
+ })
+ .finish()
+ }
+}
+
+struct NodeDebugAdaptor<'a> {
+ rlp: &'a [u8],
+}
+
+impl<'a> std::fmt::Debug for NodeDebugAdaptor<'a> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+ match Node::decoded(&self.rlp) {
+ Some(node) => write!(f, "{:?}", &node),
+ None => write!(f, "{:?}", self.rlp),
+ }
+ }
+}
+
+/// An unverified chunk from the network
+#[derive(Debug)]
+pub struct RawChunk {
+ pub nodes: Vec<TerminalNode>,
+}
+
+/// Fully recovered, and re-hydrated chunk.
+pub struct RecoveredChunk {
+ pub(crate) root: H256,
+ /// contains all nodes including non-terminal nodes and terminal nodes.
+ /// You can blindly pour all items in `nodes` into `HashDB`.
+ pub(crate) nodes: Vec<(H256, DBValue)>,
+ /// Their path slices are relative to this chunk root.
+ pub(crate) unresolved_chunks: Vec<UnresolvedChunk>,
+}
+
+impl RawChunk {
+ /// Verify and recover the chunk
+ pub fn recover(&self, expected_chunk_root: H256) -> Result<RecoveredChunk, Error> {
+ let mut memorydb = memorydb::MemoryDB::new();
+ let mut chunk_root = H256::new();
+
+ {
+ let mut trie = TrieDBMut::new(&mut memorydb, &mut chunk_root);
+ for node in self.nodes.iter() {
+ let old_val = match Node::decoded(&node.node_rlp) {
+ Some(Node::Branch(slice, child)) => {
+ let encoded = DecodedPathSlice::from_encoded(&node.path_slice).with_slice(slice).encode();
+ trie.insert_raw(Node::Branch(NibbleSlice::from_encoded(&encoded), child))?
+ }
+ Some(Node::Leaf(slice, data)) => {
+ let encoded = DecodedPathSlice::from_encoded(&node.path_slice).with_slice(slice).encode();
+ trie.insert_raw(Node::Leaf(NibbleSlice::from_encoded(&encoded), data))?
+ }
+ None => return Err(ChunkError::InvalidContent.into()),
+ };
+
+ if let Some(old_val) = old_val {
+ if old_val != node.node_rlp.as_slice() {
+ return Err(ChunkError::InvalidContent.into())
+ }
+ }
+ }
+ }
+
+ // Some nodes in the chunk is different from the expected.
+ if chunk_root != expected_chunk_root {
+ return Err(ChunkError::ChunkRootMismatch {
+ expected: expected_chunk_root,
+ actual: chunk_root,
+ }
+ .into())
+ }
+
+ let mut nodes = Vec::new();
+ let mut unresolved_chunks = Vec::new();
+ let mut queue: VecDeque<NodePath> = VecDeque::from(vec![NodePath::new(chunk_root)]);
+ while let Some(path) = queue.pop_front() {
+ let node = match memorydb.get(&path.key) {
+ Some(x) => x,
+ None => {
+ // all unresolved should depth == CHUNK_HEIGHT + 1
+ if path.depth != CHUNK_HEIGHT + 1 {
+ return Err(ChunkError::InvalidHeight.into())
+ }
+
+ unresolved_chunks.push(UnresolvedChunk::from(path));
+ continue
+ }
+ };
+
+ if path.depth > CHUNK_HEIGHT {
+ return Err(ChunkError::InvalidHeight.into())
+ }
+ nodes.push((path.key, node.clone()));
+
+ let node = Node::decoded(&node).expect("Chunk root was verified; Node can't be wrong");
+ if let Node::Branch(slice, children) = node {
+ for (index, child) in children.iter().enumerate() {
+ if let Some(child) = child {
+ queue.push_back(path.with_slice_and_index(slice, index, *child));
+ }
+ }
+ }
+ }
+
+ Ok(RecoveredChunk {
+ root: expected_chunk_root,
+ nodes,
+ unresolved_chunks,
+ })
+ }
+}
+
+impl std::fmt::Debug for RecoveredChunk {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+ struct Adapter<'a>(&'a [(H256, DBValue)]);
+ impl<'a> std::fmt::Debug for Adapter<'a> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+ f.debug_list()
+ .entries(self.0.iter().map(|(hash, rlp)| {
+ (hash, NodeDebugAdaptor {
+ rlp,
+ })
+ }))
+ .finish()
+ }
+ }
+
+ f.debug_struct("RecoveredChunk")
+ .field("root", &self.root)
+ .field("nodes", &Adapter(&self.nodes))
+ .field("unresolved_chunks", &self.unresolved_chunks)
+ .finish()
+ }
+}
+
+/// Chunk obtained from the state db.
+#[derive(Debug)]
+pub struct Chunk {
+ pub root: H256,
+ pub terminal_nodes: Vec<TerminalNode>,
+}
+
+impl Chunk {
+ pub(crate) fn from_chunk_root(db: &dyn HashDB, chunk_root: H256) -> Chunk {
+ let mut unresolved: VecDeque<NodePath> = VecDeque::from(vec![NodePath::new(chunk_root)]);
+ let mut terminal_nodes: Vec<TerminalNode> = Vec::new();
+ while let Some(path) = unresolved.pop_front() {
+ assert!(path.key != BLAKE_NULL_RLP, "Empty DB");
+ assert!(path.depth <= CHUNK_HEIGHT);
+ let node = db.get(&path.key).expect("Can't find the node in a db. DB is inconsistent");
+ let node_decoded = Node::decoded(&node).expect("Node cannot be decoded. DB is inconsistent");
+
+ match node_decoded {
+ // Continue to BFS
+ Node::Branch(slice, ref children) if path.depth < CHUNK_HEIGHT => {
+ for (i, hash) in children.iter().enumerate() {
+ if let Some(hash) = hash {
+ unresolved.push_back(path.with_slice_and_index(slice, i, *hash));
+ }
+ }
+ }
+ // Reached the terminal node. Branch at path.depth == CHUNK_HEIGHT || Leaf
+ _ => terminal_nodes.push(TerminalNode {
+ path_slice: path.path_slice.encode(),
+ node_rlp: node.to_vec(),
+ }),
+ };
+ }
+ Chunk {
+ root: chunk_root,
+ terminal_nodes,
+ }
+ }
+
+ // Returns path slices to unresolved chunk roots relative to this chunk root
+ pub(crate) fn unresolved_chunks(&self) -> Vec<UnresolvedChunk> {
+ let mut result = Vec::new();
+ for node in self.terminal_nodes.iter() {
+ let decoded = Node::decoded(&node.node_rlp).expect("All terminal nodes should be valid");
+ if let Node::Branch(slice, children) = decoded {
+ for (i, child) in children.iter().enumerate() {
+ if let Some(child) = child {
+ result.push(UnresolvedChunk {
+ path_slice: DecodedPathSlice::from_encoded(&node.path_slice).with_slice_and_index(slice, i),
+ chunk_root: *child,
+ })
+ }
+ }
+ }
+ }
+ result
+ }
+
+ #[cfg(test)]
+ pub(crate) fn into_raw_chunk(self) -> RawChunk {
+ RawChunk {
+ nodes: self.terminal_nodes,
+ }
+ }
+}
+
+/// path slice to `chunk_root` is relative to the root of originating chunk.
+#[derive(Debug)]
+pub(crate) struct UnresolvedChunk {
+ pub path_slice: DecodedPathSlice,
+ pub chunk_root: H256,
+}
+
+impl From<NodePath> for UnresolvedChunk {
+ fn from(path: NodePath) -> Self {
+ Self {
+ path_slice: path.path_slice,
+ chunk_root: path.key,
+ }
+ }
+}
+
+#[derive(Debug)]
+struct NodePath {
+ // path slice to the node relative to chunk_root
+ path_slice: DecodedPathSlice,
+ depth: usize,
+ key: H256,
+}
+
+impl NodePath {
+ fn new(key: H256) -> NodePath {
+ NodePath {
+ path_slice: DecodedPathSlice::new(),
+ depth: 1,
+ key,
+ }
+ }
+
+ fn with_slice_and_index(&self, slice: NibbleSlice, index: usize, key: H256) -> NodePath {
+ NodePath {
+ path_slice: self.path_slice.with_slice_and_index(slice, index),
+ depth: self.depth + 1,
+ key,
+ }
+ }
+}
diff --git a/util/merkle/src/snapshot/compress.rs b/util/merkle/src/snapshot/compress.rs
new file mode 100644
index 0000000000..e5733baf2a
--- /dev/null
+++ b/util/merkle/src/snapshot/compress.rs
@@ -0,0 +1,119 @@
+// Copyright 2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::io::{Cursor, Read, Write};
+
+use rlp::{Rlp, RlpStream};
+
+use super::chunk::{Chunk, RawChunk};
+use super::error::{ChunkError, Error};
+use super::CHUNK_MAX_NODES;
+
+pub struct ChunkDecompressor<R> {
+ read: R,
+}
+
+impl<R> ChunkDecompressor<R> {
+ pub fn new(read: R) -> Self {
+ ChunkDecompressor {
+ read,
+ }
+ }
+}
+
+impl<'a> ChunkDecompressor<Cursor<&'a [u8]>> {
+ pub fn from_slice(slice: &'a [u8]) -> Self {
+ ChunkDecompressor::new(Cursor::new(slice))
+ }
+}
+
+impl<R> ChunkDecompressor<R>
+where
+ R: Read + Clone,
+{
+ pub fn decompress(self) -> Result<RawChunk, Error> {
+ let mut buf = Vec::new();
+
+ let mut snappy = snap::Reader::new(self.read);
+ snappy.read_to_end(&mut buf)?;
+
+ let rlp = Rlp::new(&buf);
+ let len = rlp.item_count()?;
+ if len > CHUNK_MAX_NODES {
+ return Err(ChunkError::TooBig.into())
+ }
+
+ Ok(RawChunk {
+ nodes: rlp.as_list()?,
+ })
+ }
+}
+
+pub struct ChunkCompressor<W> {
+ write: W,
+}
+
+impl<W> ChunkCompressor<W> {
+ pub fn new(write: W) -> Self {
+ ChunkCompressor {
+ write,
+ }
+ }
+}
+
+impl<W> ChunkCompressor<W>
+where
+ W: Write,
+{
+ pub fn compress_chunk(self, chunk: &Chunk) -> Result<(), Error> {
+ let mut rlp = RlpStream::new_list(chunk.terminal_nodes.len());
+ for node in chunk.terminal_nodes.iter() {
+ rlp.append(node);
+ }
+ let mut snappy = snap::Writer::new(self.write);
+ snappy.write_all(rlp.as_raw())?;
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+ use crate::snapshot::chunk::{Chunk, TerminalNode};
+
+ #[test]
+ fn test_compress_decompress() {
+ let chunk = Chunk {
+ root: Default::default(),
+ terminal_nodes: vec![
+ (TerminalNode {
+ path_slice: b"12345".to_vec(),
+ node_rlp: b"45678".to_vec(),
+ }),
+ (TerminalNode {
+ path_slice: b"56789".to_vec(),
+ node_rlp: b"123abc".to_vec(),
+ }),
+ ],
+ };
+
+ let mut buffer = Vec::new();
+ ChunkCompressor::new(&mut buffer).compress_chunk(&chunk).unwrap();
+ let decompressed = ChunkDecompressor::from_slice(&buffer).decompress().unwrap();
+
+ assert_eq!(chunk.terminal_nodes, decompressed.nodes);
+ }
+}
diff --git a/util/merkle/src/snapshot/error.rs b/util/merkle/src/snapshot/error.rs
new file mode 100644
index 0000000000..5077312818
--- /dev/null
+++ b/util/merkle/src/snapshot/error.rs
@@ -0,0 +1,91 @@
+// Copyright 2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::io::Error as IoError;
+
+use primitives::H256;
+use rlp::DecoderError as RlpDecoderError;
+
+use crate::TrieError;
+use std::fmt::{Display, Formatter};
+
+#[derive(Debug)]
+pub enum Error {
+ IoError(IoError),
+ RlpDecoderError(RlpDecoderError),
+ TrieError(TrieError),
+ ChunkError(ChunkError),
+}
+
+impl From<IoError> for Error {
+ fn from(err: IoError) -> Self {
+ Error::IoError(err)
+ }
+}
+
+impl From<RlpDecoderError> for Error {
+ fn from(err: RlpDecoderError) -> Self {
+ Error::RlpDecoderError(err)
+ }
+}
+
+impl From<TrieError> for Error {
+ fn from(err: TrieError) -> Self {
+ Error::TrieError(err)
+ }
+}
+
+impl From<ChunkError> for Error {
+ fn from(err: ChunkError) -> Self {
+ Error::ChunkError(err)
+ }
+}
+
+impl Display for Error {
+ fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> {
+ match self {
+ Error::IoError(err) => write!(f, "IoError: {}", err),
+ Error::RlpDecoderError(err) => write!(f, "RlpDecoderError: {}", err),
+ Error::TrieError(err) => write!(f, "TrieError: {}", err),
+ Error::ChunkError(err) => write!(f, "ChunkError: {}", err),
+ }
+ }
+}
+
+#[derive(Debug)]
+pub enum ChunkError {
+ TooBig,
+ InvalidHeight,
+ ChunkRootMismatch {
+ expected: H256,
+ actual: H256,
+ },
+ InvalidContent,
+}
+
+impl Display for ChunkError {
+ fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> {
+ match self {
+ ChunkError::TooBig => write!(f, "Chunk has too many elements"),
+ ChunkError::InvalidHeight => write!(f, "Chunk height is unexpected height"),
+ ChunkError::ChunkRootMismatch {
+ expected,
+ actual,
+ } => write!(f, "Chunk root is different from expected. expected: {}, actual: {}", expected, actual),
+ ChunkError::InvalidContent => write!(f, "Chunk content is invalid"),
+ }
+ }
+}
diff --git a/util/merkle/src/snapshot/mod.rs b/util/merkle/src/snapshot/mod.rs
new file mode 100644
index 0000000000..860fc71d94
--- /dev/null
+++ b/util/merkle/src/snapshot/mod.rs
@@ -0,0 +1,343 @@
+// Copyright 2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub mod chunk;
+mod compress;
+mod error;
+mod ordered_heap;
+
+use std::cmp::Ordering;
+
+use ccrypto::BLAKE_NULL_RLP;
+use hashdb::HashDB;
+use primitives::H256;
+
+use self::chunk::{Chunk, RecoveredChunk, UnresolvedChunk};
+pub use self::compress::{ChunkCompressor, ChunkDecompressor};
+pub use self::error::Error;
+use self::ordered_heap::OrderedHeap;
+use crate::nibbleslice::NibbleSlice;
+
+const CHUNK_HEIGHT: usize = 3;
+const CHUNK_MAX_NODES: usize = 256; // 16 ^ (CHUNK_HEIGHT-1)
+
+/// Example:
+/// use codechain_merkle::snapshot::Restore;
+/// let mut rm = Restore::new(root);
+/// while let Some(root) = rm.next_to_feed() {
+/// let raw_chunk = request(block_hash, root)?;
+/// let chunk = raw_chunk.recover(root)?;
+/// rm.feed(db, chunk);
+/// }
+pub struct Restore {
+ pending: Option<ChunkPathPrefix>,
+ unresolved: OrderedHeap<DepthFirst<ChunkPathPrefix>>,
+}
+
+impl Restore {
+ pub fn new(merkle_root: H256) -> Self {
+ let mut result = Restore {
+ pending: None,
+ unresolved: OrderedHeap::new(),
+ };
+ if merkle_root != BLAKE_NULL_RLP {
+ result.unresolved.push(ChunkPathPrefix::new(merkle_root).into());
+ }
+ result
+ }
+
+ pub fn feed(&mut self, db: &mut dyn HashDB, chunk: RecoveredChunk) {
+ let pending_path = self.pending.take().expect("feed() should be called after next()");
+ assert_eq!(pending_path.chunk_root, chunk.root, "Unexpected chunk");
+
+ // Pour nodes into the DB
+ for (key, value) in chunk.nodes {
+ db.emplace(key, value);
+ }
+
+ // Extend search paths
+ for unresolved in chunk.unresolved_chunks {
+ self.unresolved.push(pending_path.with_unresolved_chunk(&unresolved).into());
+ }
+
+ self.pending = None;
+ }
+
+ pub fn next_to_feed(&mut self) -> Option<H256> {
+ if let Some(pending) = &self.pending {
+ Some(pending.chunk_root)
+ } else if let Some(path) = self.unresolved.pop() {
+ let chunk_root = path.chunk_root;
+ self.pending = Some(path.0);
+
+ Some(chunk_root)
+ } else {
+ None
+ }
+ }
+}
+
+impl std::fmt::Debug for Restore {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+ f.debug_struct("Restore").field("pending", &self.pending).field("unresolved", &"<...>".to_string()).finish()
+ }
+}
+
+/// Example:
+/// use std::fs::File;
+/// use codechain_merkle::snapshot::Snapshot;
+///
+/// for chunk in Snapshot::from_hashdb(db, root) {
+/// let mut file = File::create(format!("{}/{}", block_id, chunk.root))?;
+/// let mut compressor = ChunkCompressor::new(&mut file);
+/// compressor.compress(chunk);
+/// }
+pub struct Snapshot<'a> {
+ db: &'a dyn HashDB,
+ remaining: OrderedHeap<DepthFirst<ChunkPathPrefix>>,
+}
+
+impl<'a> Snapshot<'a> {
+ pub fn from_hashdb(db: &'a dyn HashDB, chunk_root: H256) -> Self {
+ let mut result = Snapshot {
+ db,
+ remaining: OrderedHeap::new(),
+ };
+ if chunk_root != BLAKE_NULL_RLP {
+ result.remaining.push(ChunkPathPrefix::new(chunk_root).into());
+ }
+ result
+ }
+}
+
+impl<'a> Iterator for Snapshot<'a> {
+ type Item = Chunk;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if let Some(path) = self.remaining.pop() {
+ let chunk = Chunk::from_chunk_root(self.db, path.chunk_root);
+ for unresolved in chunk.unresolved_chunks() {
+ self.remaining.push(path.with_unresolved_chunk(&unresolved).into());
+ }
+ Some(chunk)
+ } else {
+ None
+ }
+ }
+}
+
+
+#[derive(Debug)]
+struct ChunkPathPrefix {
+ // Absolute path prefix of the chunk root
+ path_prefix: DecodedPathSlice,
+ depth: usize,
+ chunk_root: H256,
+}
+
+impl ChunkPathPrefix {
+ fn new(chunk_root: H256) -> ChunkPathPrefix {
+ ChunkPathPrefix {
+ path_prefix: DecodedPathSlice::new(),
+ depth: 1,
+ chunk_root,
+ }
+ }
+
+ fn with_unresolved_chunk(&self, unresolved: &UnresolvedChunk) -> ChunkPathPrefix {
+ ChunkPathPrefix {
+ path_prefix: self.path_prefix.with_path_slice(&unresolved.path_slice),
+ depth: self.depth + 1,
+ chunk_root: unresolved.chunk_root,
+ }
+ }
+}
+
+impl Ord for DepthFirst<ChunkPathPrefix> {
+ fn cmp(&self, other: &Self) -> Ordering {
+ self.0.depth.cmp(&other.0.depth)
+ }
+}
+
+impl From<ChunkPathPrefix> for DepthFirst<ChunkPathPrefix> {
+ fn from(path: ChunkPathPrefix) -> Self {
+ DepthFirst(path)
+ }
+}
+
+/// Encoded value by NibbleSlice::encoded()
+pub type PathSlice = Vec<u8>;
+
+/// for item i, i in 0..16
+pub(crate) struct DecodedPathSlice(Vec<u8>);
+
+impl DecodedPathSlice {
+ fn new() -> DecodedPathSlice {
+ DecodedPathSlice(Vec::new())
+ }
+
+ fn from_encoded(slice: &[u8]) -> DecodedPathSlice {
+ DecodedPathSlice(NibbleSlice::from_encoded(slice).to_vec())
+ }
+
+ fn with_slice_and_index(&self, slice: NibbleSlice, i: usize) -> DecodedPathSlice {
+ assert!(i < 16);
+ let mut v = self.0.clone();
+ v.append(&mut slice.to_vec());
+ v.push(i as u8);
+ DecodedPathSlice(v)
+ }
+
+ fn with_slice(&self, slice: NibbleSlice) -> DecodedPathSlice {
+ let mut v = self.0.clone();
+ v.append(&mut slice.to_vec());
+ DecodedPathSlice(v)
+ }
+
+ fn with_path_slice(&self, path_slice: &DecodedPathSlice) -> DecodedPathSlice {
+ let mut v = self.0.clone();
+ v.extend(path_slice.0.as_slice());
+ DecodedPathSlice(v)
+ }
+
+ fn encode(&self) -> PathSlice {
+ let (encoded, _) = NibbleSlice::from_vec(&self.0);
+ encoded.to_vec()
+ }
+}
+
+impl std::fmt::Debug for DecodedPathSlice {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+ let (encoded, _) = NibbleSlice::from_vec(&self.0);
+ let nibble_slice = NibbleSlice::from_encoded(&encoded);
+ writeln!(f, "{:?}", nibble_slice)
+ }
+}
+
+#[derive(Debug)]
+struct DepthFirst<T>(T);
+
+impl<T> PartialOrd for DepthFirst<T>
+where
+ Self: Ord,
+{
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+ Some(self.cmp(&other))
+ }
+}
+
+impl<T> PartialEq for DepthFirst<T>
+where
+ Self: Ord,
+{
+ fn eq(&self, other: &Self) -> bool {
+ self.cmp(other) == Ordering::Equal
+ }
+}
+
+impl<T> Eq for DepthFirst<T> where Self: Ord {}
+
+impl<T> std::ops::Deref for DepthFirst<T> {
+ type Target = T;
+
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ use std::collections::HashMap;
+ use std::iter::FromIterator;
+
+ use memorydb::MemoryDB;
+ use primitives::{Bytes, H256};
+ use standardmap::{Alphabet, StandardMap, ValueMode};
+
+ use super::chunk::RawChunk;
+ use crate::{Trie, TrieDB, TrieDBMut, TrieMut};
+
+ fn random_insert_and_restore_with_count(count: usize) {
+ let standard_map = StandardMap {
+ alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()),
+ min_key: 5,
+ journal_key: 0,
+ value_mode: ValueMode::Index,
+ count,
+ }
+ .make_with(&mut H256::new());
+ // Unique standard map
+ let unique_map: HashMap<Bytes, Bytes> = HashMap::from_iter(standard_map.into_iter());
+
+ let mut root = H256::new();
+ let chunks: HashMap<H256, RawChunk> = {
+ // We will throw out `db` after snapshot.
+ let mut db = MemoryDB::new();
+ let mut trie = TrieDBMut::new(&mut db, &mut root);
+ for (key, value) in &unique_map {
+ trie.insert(key, value).unwrap();
+ }
+
+ Snapshot::from_hashdb(&db, root).map(|chunk| (chunk.root, chunk.into_raw_chunk())).collect()
+ };
+ dbg!(chunks.len());
+
+ let mut db = MemoryDB::new();
+ let mut recover = Restore::new(root);
+ while let Some(chunk_root) = recover.next_to_feed() {
+ let recovered = chunks[&chunk_root].recover(chunk_root).unwrap();
+ recover.feed(&mut db, recovered);
+ }
+
+ let trie = TrieDB::try_new(&db, &root).unwrap();
+ for (key, value) in &unique_map {
+ assert_eq!(trie.get(key).unwrap().as_ref(), Some(value));
+ }
+ }
+
+ #[test]
+ fn random_insert_and_restore_0() {
+ random_insert_and_restore_with_count(0);
+ }
+
+ #[test]
+ fn random_insert_and_restore_1() {
+ random_insert_and_restore_with_count(1);
+ }
+
+ #[test]
+ fn random_insert_and_restore_2() {
+ random_insert_and_restore_with_count(2);
+ }
+
+ #[test]
+ fn random_insert_and_restore_100() {
+ random_insert_and_restore_with_count(100);
+ }
+
+ #[test]
+ fn random_insert_and_restore_10000() {
+ random_insert_and_restore_with_count(10_000);
+ }
+
+ #[test]
+ #[ignore]
+ fn random_insert_and_restore_100000() {
+ random_insert_and_restore_with_count(100_000);
+ }
+}
diff --git a/util/merkle/src/snapshot/ordered_heap.rs b/util/merkle/src/snapshot/ordered_heap.rs
new file mode 100644
index 0000000000..d83efd77c1
--- /dev/null
+++ b/util/merkle/src/snapshot/ordered_heap.rs
@@ -0,0 +1,76 @@
+// Copyright 2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use std::collections::BinaryHeap;
+
+pub struct OrderedHeap<T> {
+ heap: BinaryHeap<OrderedHeapEntry<T>>,
+ seq: usize,
+}
+
+impl<T: Ord> OrderedHeap<T> {
+ pub fn new() -> OrderedHeap<T> {
+ OrderedHeap {
+ heap: BinaryHeap::new(),
+ seq: 0,
+ }
+ }
+
+ pub fn push(&mut self, value: T) {
+ self.heap.push(OrderedHeapEntry {
+ seq: self.seq,
+ value,
+ });
+ self.seq += 1;
+ }
+
+ pub fn pop(&mut self) -> Option<T> {
+ self.heap.pop().map(|x| x.value)
+ }
+}
+
+#[derive(Debug, Clone)]
+struct OrderedHeapEntry<T> {
+ seq: usize,
+ value: T,
+}
+
+impl<T: Ord> Ord for OrderedHeapEntry<T> {
+ fn cmp(&self, other: &Self) -> Ordering {
+ self.value.cmp(&other.value).then(self.seq.cmp(&other.seq).reverse())
+ }
+}
+
+impl<T> PartialOrd for OrderedHeapEntry<T>
+where
+ Self: Ord,
+{
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+ Some(self.cmp(&other))
+ }
+}
+
+impl<T> PartialEq for OrderedHeapEntry<T>
+where
+ Self: Ord,
+{
+ fn eq(&self, other: &Self) -> bool {
+ self.cmp(other) == Ordering::Equal
+ }
+}
+
+impl<T> Eq for OrderedHeapEntry<T> where Self: Ord {}
diff --git a/util/merkle/src/triedb.rs b/util/merkle/src/triedb.rs
index d10a4f00ca..a456db9004 100644
--- a/util/merkle/src/triedb.rs
+++ b/util/merkle/src/triedb.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use ccrypto::blake256;
+use ccrypto::{blake256, BLAKE_NULL_RLP};
use hashdb::HashDB;
use primitives::H256;
@@ -105,6 +105,26 @@ impl<'db> TrieDB<'db> {
None => Ok(None),
}
}
+
+ /// Check if every leaf of the trie exists
+ pub fn is_complete(&self) -> bool {
+ *self.root == BLAKE_NULL_RLP || self.is_complete_aux(self.root)
+ }
+
+ /// Check if every leaf of the trie starting from `hash` exists
+ pub fn is_complete_aux(&self, hash: &H256) -> bool {
+ if let Some(node_rlp) = self.db.get(hash) {
+ match RlpNode::decoded(node_rlp.as_ref()) {
+ Some(RlpNode::Branch(.., children)) => {
+ children.iter().flatten().all(|child| self.is_complete_aux(child))
+ }
+ Some(RlpNode::Leaf(..)) => true,
+ None => false,
+ }
+ } else {
+ false
+ }
+ }
}
impl<'db> Trie for TrieDB<'db> {
@@ -126,6 +146,19 @@ mod tests {
use crate::*;
use memorydb::*;
+ fn delete_any_child(db: &mut MemoryDB, root: &H256) {
+ let node_rlp = db.get(root).unwrap();
+ match RlpNode::decoded(&node_rlp).unwrap() {
+ RlpNode::Leaf(..) => {
+ db.remove(root);
+ }
+ RlpNode::Branch(.., children) => {
+ let first_child = children.iter().find(|c| c.is_some()).unwrap().unwrap();
+ db.remove(&first_child);
+ }
+ }
+ }
+
#[test]
fn get() {
let mut memdb = MemoryDB::new();
@@ -141,4 +174,33 @@ mod tests {
assert_eq!(t.get(b"B"), Ok(Some(b"ABCBA".to_vec())));
assert_eq!(t.get(b"C"), Ok(None));
}
+
+ #[test]
+ fn is_complete_success() {
+ let mut memdb = MemoryDB::new();
+ let mut root = H256::new();
+ {
+ let mut t = TrieDBMut::new(&mut memdb, &mut root);
+ t.insert(b"A", b"ABC").unwrap();
+ t.insert(b"B", b"ABCBA").unwrap();
+ }
+
+ let t = TrieDB::try_new(&memdb, &root).unwrap();
+ assert!(t.is_complete());
+ }
+
+ #[test]
+ fn is_complete_fail() {
+ let mut memdb = MemoryDB::new();
+ let mut root = H256::new();
+ {
+ let mut t = TrieDBMut::new(&mut memdb, &mut root);
+ t.insert(b"A", b"ABC").unwrap();
+ t.insert(b"B", b"ABCBA").unwrap();
+ }
+ delete_any_child(&mut memdb, &root);
+
+ let t = TrieDB::try_new(&memdb, &root).unwrap();
+ assert!(!t.is_complete());
+ }
}
diff --git a/util/merkle/src/triedbmut.rs b/util/merkle/src/triedbmut.rs
index 5684ae0556..6bb89adedc 100644
--- a/util/merkle/src/triedbmut.rs
+++ b/util/merkle/src/triedbmut.rs
@@ -170,6 +170,113 @@ impl<'a> TrieDBMut<'a> {
}
}
+ pub(crate) fn insert_raw(&mut self, node: RlpNode) -> crate::Result<Option<DBValue>> {
+ let mut old_val = None;
+ let cur_hash = *self.root;
+ *self.root = self.insert_raw_aux(node, Some(cur_hash), &mut old_val)?;
+
+ Ok(old_val)
+ }
+
+ fn insert_raw_aux(
+ &mut self,
+ node: RlpNode,
+ cur_node_hash: Option<H256>,
+ old_val: &mut Option<DBValue>,
+ ) -> crate::Result<H256> {
+ let path = match &node {
+ RlpNode::Leaf(slice, _) | RlpNode::Branch(slice, _) => slice,
+ };
+
+ match cur_node_hash {
+ Some(hash) => {
+ let existing_node_rlp = self.db.get(&hash).ok_or_else(|| TrieError::IncompleteDatabase(hash))?;
+ match RlpNode::decoded(&existing_node_rlp) {
+ Some(RlpNode::Leaf(partial, value)) => {
+ // Renew the Leaf
+ if &partial == path {
+ let hash = self.db.insert(&RlpNode::encoded(node));
+ *old_val = Some(existing_node_rlp);
+ Ok(hash)
+ } else {
+ // Make branch node and insert Leaves
+ let common = partial.common_prefix(&path);
+ let mut new_child = empty_children();
+ let new_partial = partial.mid(common);
+ let new_path = path.mid(common);
+ new_child[new_partial.at(0) as usize] = Some(self.insert_aux(
+ new_partial.mid(1),
+ value,
+ new_child[new_partial.at(0) as usize],
+ old_val,
+ )?);
+ new_child[new_path.at(0) as usize] = Some(self.insert_raw_aux(
+ node.mid(common + 1),
+ new_child[new_path.at(0) as usize],
+ old_val,
+ )?);
+
+ let hash = self
+ .db
+ .insert(&RlpNode::encoded_until(RlpNode::Branch(partial, new_child.into()), common));
+
+ Ok(hash)
+ }
+ }
+ Some(RlpNode::Branch(partial, mut children)) => {
+ let common = partial.common_prefix(&path);
+
+ // Make new branch node and insert leaf and branch with new path
+ if common < partial.len() {
+ let mut new_child = empty_children();
+ let new_partial = partial.mid(common);
+ let new_path = path.mid(common);
+ let o_branch = RlpNode::Branch(new_partial.mid(1), children);
+
+ let b_hash = self.db.insert(&RlpNode::encoded(o_branch));
+
+ new_child[new_partial.at(0) as usize] = Some(b_hash);
+ new_child[new_path.at(0) as usize] = Some(self.insert_raw_aux(
+ node.mid(common + 1),
+ new_child[new_path.at(0) as usize],
+ old_val,
+ )?);
+
+ let hash = self
+ .db
+ .insert(&RlpNode::encoded_until(RlpNode::Branch(partial, new_child.into()), common));
+
+ Ok(hash)
+ } else {
+ // Insert leaf into the branch node
+ let new_path = path.mid(common);
+
+ children[new_path.at(0) as usize] = Some(self.insert_raw_aux(
+ node.mid(common + 1),
+ children[new_path.at(0) as usize],
+ old_val,
+ )?);
+
+ let new_branch = RlpNode::Branch(partial, children);
+ let node_rlp = RlpNode::encoded(new_branch);
+ let hash = self.db.insert(&node_rlp);
+
+ Ok(hash)
+ }
+ }
+ None => {
+ let hash = self.db.insert(&RlpNode::encoded(node));
+ Ok(hash)
+ }
+ }
+ }
+ None => {
+ let hash = self.db.insert(&RlpNode::encoded(node));
+ Ok(hash)
+ }
+ }
+ }
+
/// Remove auxiliary
fn remove_aux(
&mut self,