diff --git a/Cargo.lock b/Cargo.lock index f5369cce2d5..9087f27d538 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -403,15 +403,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "autocfg" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" -dependencies = [ - "autocfg 1.1.0", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -1285,7 +1276,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" dependencies = [ - "autocfg 1.1.0", + "autocfg", "cfg-if 1.0.0", "crossbeam-utils 0.8.8", "lazy_static", @@ -1309,7 +1300,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ - "autocfg 1.1.0", + "autocfg", "cfg-if 0.1.10", "lazy_static", ] @@ -2285,7 +2276,7 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ - "autocfg 1.1.0", + "autocfg", "hashbrown 0.11.2", "serde", ] @@ -2593,7 +2584,7 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ - "autocfg 1.1.0", + "autocfg", "scopeguard", "serde", ] @@ -2708,7 +2699,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -2879,6 +2870,7 @@ dependencies = [ "num-rational", "once_cell", "rand 0.8.5", + "rand_chacha 0.3.1", "rayon", "strum", "thiserror", @@ -3048,8 +3040,8 @@ dependencies = [ "near-store", "num-rational", "primitive-types", - "rand 0.6.5", "rand 0.8.5", + "rand_hc 0.3.1", "serde_json", "smart-default", "tracing", @@ -3243,7 +3235,7 @@ dependencies = [ "protobuf 3.0.2", "protobuf-codegen", "rand 0.8.5", - "rand_xorshift 0.3.0", + "rand_xorshift", "rayon", "serde", "smart-default", @@ -3772,7 +3764,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-integer", "num-traits", ] @@ -3783,7 +3775,7 @@ version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-traits", ] @@ -3793,7 +3785,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-bigint", "num-integer", "num-traits", @@ -3806,7 +3798,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -3911,7 +3903,7 @@ version = "0.9.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" dependencies = [ - "autocfg 1.1.0", + "autocfg", "cc", "libc", "openssl-src", @@ -4476,24 +4468,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.8", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.2", - "rand_hc 0.1.0", - "rand_isaac", - "rand_jitter", - "rand_pcg", - "rand_xorshift 0.1.1", - "winapi", -] - [[package]] name = "rand" version = "0.7.3" @@ -4518,16 +4492,6 @@ dependencies = [ "rand_core 0.6.3", ] -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.3.1", -] - [[package]] name = "rand_chacha" version = "0.2.2" @@ -4548,21 +4512,6 @@ dependencies = [ "rand_core 0.6.3", ] -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.5.1" @@ -4581,15 +4530,6 @@ dependencies = [ "getrandom 0.2.6", ] -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "rand_hc" version = "0.2.0" @@ -4600,42 +4540,12 @@ dependencies = [ ] [[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.4.2", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" +name = "rand_hc" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" dependencies = [ - "rand_core 0.3.1", + "rand_core 0.6.3", ] [[package]] @@ -4662,7 +4572,7 @@ version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" dependencies = [ - "autocfg 1.1.0", + "autocfg", "crossbeam-deque", "either", "rayon-core", @@ -4940,7 +4850,7 @@ dependencies = [ "num-rational", "num-traits", "rand 0.8.5", - "rand_xorshift 0.3.0", + "rand_xorshift", "rocksdb", "serde_json", "tempfile", @@ -5411,7 +5321,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" dependencies = [ - "autocfg 1.1.0", + "autocfg", "static_assertions", "version_check", ] diff --git a/Cargo.toml b/Cargo.toml index 466a77d690b..88f09e37853 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,6 +149,8 @@ protobuf = "3.0.1" protobuf-codegen = "3.0.1" quote = "1.0" rand = "0.8.5" +rand_chacha = "0.3.1" +rand_hc = "0.3.1" rand_xorshift = "0.3" rayon = "1.5" redis = "0.21.5" diff --git a/chain/chain/Cargo.toml b/chain/chain/Cargo.toml index a34082b3925..98cb6cde8c2 100644 --- a/chain/chain/Cargo.toml +++ b/chain/chain/Cargo.toml @@ -19,6 +19,7 @@ lru.workspace = true num-rational.workspace = true once_cell.workspace = true rand.workspace = true +rand_chacha.workspace = true rayon.workspace = true strum.workspace = true thiserror.workspace = true diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index e12a2eb4c76..d9ade4cdfc6 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -9,9 +9,9 @@ use itertools::Itertools; use near_o11y::log_assert; use near_primitives::sandbox::state_patch::SandboxStatePatch; use near_primitives::time::Clock; -use rand::rngs::StdRng; use rand::seq::SliceRandom; use rand::SeedableRng; +use rand_chacha::ChaCha20Rng; use tracing::{debug, error, info, warn, Span}; use near_chain_primitives::error::{BlockKnownError, Error, LogTransientStorageError}; @@ -1536,15 +1536,22 @@ impl Chain { } // sort the receipts deterministically so the order that they will be processed is deterministic for (_, receipt_proofs) in receipt_proofs_by_shard_id.iter_mut() { - let mut slice = [0u8; 32]; - slice.copy_from_slice(block.hash().as_ref()); - let mut rng: StdRng = SeedableRng::from_seed(slice); - receipt_proofs.shuffle(&mut rng); + Self::shuffle_receipt_proofs(receipt_proofs, block.hash()); } Ok(receipt_proofs_by_shard_id) } + fn shuffle_receipt_proofs( + receipt_proofs: &mut Vec, + block_hash: &CryptoHash, + ) { + let mut slice = [0u8; 32]; + slice.copy_from_slice(block_hash.as_ref()); + let mut rng: ChaCha20Rng = SeedableRng::from_seed(slice); + receipt_proofs.shuffle(&mut rng); + } + #[cfg(test)] pub(crate) fn mark_block_as_challenged( &mut self, @@ -5358,3 +5365,19 @@ impl Chain { .collect() } } + +#[cfg(test)] +mod tests { + use near_primitives::hash::CryptoHash; + + #[test] + pub fn receipt_randomness_reproducibility() { + // Sanity check that the receipt shuffling implementation does not change. + let mut receipt_proofs = vec![0, 1, 2, 3, 4, 5, 6]; + crate::Chain::shuffle_receipt_proofs( + &mut receipt_proofs, + &CryptoHash::hash_bytes(&[1, 2, 3, 4, 5]), + ); + assert_eq!(receipt_proofs, vec![2, 3, 1, 4, 0, 5, 6],); + } +} diff --git a/chain/epoch-manager/Cargo.toml b/chain/epoch-manager/Cargo.toml index 0bb2e87a0fd..ff1677ef76b 100644 --- a/chain/epoch-manager/Cargo.toml +++ b/chain/epoch-manager/Cargo.toml @@ -9,14 +9,12 @@ edition.workspace = true [dependencies] -# Changing this version will lead to change to the protocol, as will change how validators get shuffled. -protocol_defining_rand = { package = "rand", version = "0.6.5", default-features = false } - borsh.workspace = true chrono = { workspace = true, optional = true } num-rational.workspace = true primitive-types.workspace = true rand.workspace = true +rand_hc.workspace = true serde_json.workspace = true smart-default.workspace = true tracing.workspace = true diff --git a/chain/epoch-manager/src/proposals.rs b/chain/epoch-manager/src/proposals.rs index 21ac376698e..682934ad49b 100644 --- a/chain/epoch-manager/src/proposals.rs +++ b/chain/epoch-manager/src/proposals.rs @@ -86,6 +86,9 @@ mod old_validator_selection { AccountId, Balance, NumSeats, ValidatorId, ValidatorKickoutReason, }; use near_primitives::version::ProtocolVersion; + use rand::seq::SliceRandom; + use rand::SeedableRng; + use rand_hc::Hc128Rng; use crate::proposals::find_threshold; use crate::types::RngSeed; @@ -183,13 +186,7 @@ mod old_validator_selection { .collect::>(); assert!(dup_proposals.len() >= num_total_seats as usize, "bug in find_threshold"); - { - use protocol_defining_rand::seq::SliceRandom; - use protocol_defining_rand::{rngs::StdRng, SeedableRng}; - // Shuffle duplicate proposals. - let mut rng: StdRng = SeedableRng::from_seed(rng_seed); - dup_proposals.shuffle(&mut rng); - } + shuffle_duplicate_proposals(&mut dup_proposals, rng_seed); // Block producers are first `num_block_producer_seats` proposals. let mut block_producers_settlement = @@ -270,4 +267,27 @@ mod old_validator_selection { rng_seed, )) } + + fn shuffle_duplicate_proposals(dup_proposals: &mut Vec, rng_seed: RngSeed) { + let mut rng: Hc128Rng = SeedableRng::from_seed(rng_seed); + dup_proposals.shuffle(&mut rng); + } + + #[cfg(test)] + mod tests { + use near_primitives::hash::CryptoHash; + + use crate::proposals::old_validator_selection::shuffle_duplicate_proposals; + + #[test] + pub fn proposal_randomness_reproducibility() { + // Sanity check that the proposal shuffling implementation does not change. + let mut dup_proposals = vec![0, 1, 2, 3, 4, 5, 6]; + shuffle_duplicate_proposals( + &mut dup_proposals, + CryptoHash::hash_bytes(&[1, 2, 3, 4, 5]).as_bytes().clone(), + ); + assert_eq!(dup_proposals, vec![3, 1, 0, 4, 5, 6, 2]); + } + } } diff --git a/deny.toml b/deny.toml index ca260b8b353..e38d39842a3 100644 --- a/deny.toml +++ b/deny.toml @@ -26,19 +26,8 @@ skip = [ # criterion uses clap=2.34.0 which relies on an older textwrap { name = "textwrap", version = "=0.11.0" }, - # near-epoch-manager fixed the rand version to ensure protocol - # stability. This pulls in other ancient crates. - { name = "rand", version = "=0.6.5" }, - { name = "rand_xorshift", version = "=0.1.1" }, - # rand 0.6.5 uses two versions of rand_core due to weird dependencies mismatch with rand_chacha - { name = "rand_core", version = "=0.3.1" }, - { name = "rand_chacha", version = "=0.1.1" }, - - { name = "autocfg", version = "=0.1.8" }, - # wasmer 0.17 and wasmtime 0.17 use conflicting versions of those { name = "wasmparser", version = "=0.51.4" }, - { name = "rand_core", version = "=0.4.2" }, # wasmer 0.17 and wasmtime 0.17 uses older versions of some crates { name = "generic-array", version = "=0.12.4" },