From 2eb70b3b915e2fcc72ec19b1d0e4ad0a5d332f77 Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Aug 2023 11:18:37 -0600 Subject: [PATCH 1/7] setup benches binary --- .gitignore | 2 +- Cargo.toml | 8 +-- benches/Cargo.toml | 12 ++++ benches/src/main.rs | 142 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 benches/Cargo.toml create mode 100644 benches/src/main.rs diff --git a/.gitignore b/.gitignore index f0434d32..3f2816ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Generated by Cargo # will have compiled files and executables /target/ +/benches/target/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html @@ -15,7 +16,6 @@ Cargo.lock *.csv .DS_Store -# /contracts/* arbiter/ .vscode \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index e17646d9..eb7581dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [workspace] - # List of crates included in this workspace -members = [ - "arbiter-core", -] +members = ["arbiter-core"] + +# List of crates excluded from this workspace +exclude = ["benches"] # Package configuration [package] diff --git a/benches/Cargo.toml b/benches/Cargo.toml new file mode 100644 index 00000000..44fe09ab --- /dev/null +++ b/benches/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "benches" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = "1.32.0" +arbiter-core = { path = "../arbiter-core" } +ethers = { git = "https://github.com/primitivefinance/ethers-rs.git"} # Custom fork for middleware integration +anyhow = "1.0.75" \ No newline at end of file diff --git a/benches/src/main.rs b/benches/src/main.rs new file mode 100644 index 00000000..051de633 --- /dev/null +++ b/benches/src/main.rs @@ -0,0 +1,142 @@ +// use ethers_providers::{Middleware, Provider, Http}; +// use ethers_signers::LocalWallet; +// use ethers_middleware::SignerMiddleware; +// use ethers_core::types::{Address, TransactionRequest}; +use std::convert::TryFrom; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use anyhow::Result; +use arbiter_core::{ + bindings::{ + arbiter_math::ArbiterMath, + arbiter_token::{self, ArbiterToken}, + }, + environment::EnvironmentParameters, + manager::Manager, + middleware::RevmMiddleware, +}; +use ethers::types::Address; +use ethers::{ + core::{utils::Anvil, k256::{Secp256k1, ecdsa::SigningKey}}, + middleware::SignerMiddleware, + providers::{Http, Middleware, Provider}, + signers::{LocalWallet, Signer, Wallet}, + types::{I256, U256}, +}; + +const NUM_STEPS: usize = 1000; + +#[tokio::main] +async fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + let args = args.get(1).unwrap().as_str(); + match args { + label @ "anvil" => { + let client = anvil_startup().await?; + let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await?; + stateless_call_loop(arbiter_math, label).await?; + stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label).await?; + } + label @ "arbiter" => { + let client = arbiter_startup().await?; + let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await?; + stateless_call_loop(arbiter_math, label).await?; + stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label).await?; + }, + _ => panic!("Invalid argument"), + }; + + Ok(()) +} + +async fn anvil_startup( +) -> Result, Wallet>>> { + // Create an anvil instance + // No blocktime mines a new block for each tx. + let anvil = Anvil::new().spawn(); + + // Create a client + let provider = Provider::::try_from(anvil.endpoint()) + .unwrap() + .interval(Duration::ZERO); + + let wallet: LocalWallet = anvil.keys()[0].clone().into(); + let client = Arc::new(SignerMiddleware::new( + provider, + wallet.with_chain_id(anvil.chain_id()), + )); + + Ok(client) +} + +async fn arbiter_startup() -> Result> { + let mut manager = Manager::new(); + let params = EnvironmentParameters { + block_rate: 10.0, + seed: 0, + }; + manager.add_environment("env", params)?; + + let client = Arc::new(RevmMiddleware::new( + manager.environments.get("env").unwrap(), + Some("name".to_string()), + )); + + manager.start_environment("env")?; + + Ok(client) +} + +async fn deployments( + client: Arc, + label: &str, +) -> Result<(ArbiterMath, ArbiterToken)> { + let start = Instant::now(); + let arbiter_math = ArbiterMath::deploy(client.clone(), ())?.send().await?; + let arbiter_token = arbiter_token::ArbiterToken::deploy( + client.clone(), + ("Arbiter Token".to_string(), "ARBT".to_string(), 18_u8), + )? + .send() + .await?; + let duration = start.elapsed(); + println!("Time elapsed in {} deployment is: {:?}", label, duration); + Ok((arbiter_math, arbiter_token)) +} + +async fn stateless_call_loop(arbiter_math: ArbiterMath, label: &str) -> Result<()> { + let iwad = I256::from(10_u128.pow(18)); + let start = Instant::now(); + for _ in 0..NUM_STEPS { + arbiter_math.cdf(iwad).call().await?; + } + let duration = start.elapsed(); + + println!("Time elapsed in {} cdf loop is: {:?}", label, duration); + Ok(()) +} + +async fn stateful_call_loop(arbiter_token: arbiter_token::ArbiterToken, mint_address: Address, label: &str) -> Result<()> { + let wad = U256::from(10_u128.pow(18)); + let start = Instant::now(); + for _ in 0..NUM_STEPS { + arbiter_token + .mint(mint_address, wad) + .send() + .await? + .await?; + } + let duration = start.elapsed(); + println!("Time elapsed in {} mint loop is: {:?}", label, duration); + Ok(()) +} + +async fn mixture_loop( + arbiter_math: ArbiterMath, + arbiter_token: arbiter_token::ArbiterToken, +) -> Result<()> { + Ok(()) +} From 86e470b0150b4075050715bd955113916c356e4b Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Aug 2023 11:30:29 -0600 Subject: [PATCH 2/7] working benches --- benches/src/main.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/benches/src/main.rs b/benches/src/main.rs index 051de633..2fe09ed3 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -19,8 +19,9 @@ use arbiter_core::{ middleware::RevmMiddleware, }; use ethers::types::Address; +use ethers::utils::AnvilInstance; use ethers::{ - core::{utils::Anvil, k256::{Secp256k1, ecdsa::SigningKey}}, + core::{utils::Anvil, k256::ecdsa::SigningKey}, middleware::SignerMiddleware, providers::{Http, Middleware, Provider}, signers::{LocalWallet, Signer, Wallet}, @@ -35,7 +36,7 @@ async fn main() -> Result<()> { let args = args.get(1).unwrap().as_str(); match args { label @ "anvil" => { - let client = anvil_startup().await?; + let (client, _anvil_instance) = anvil_startup().await?; let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await?; stateless_call_loop(arbiter_math, label).await?; stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label).await?; @@ -53,10 +54,11 @@ async fn main() -> Result<()> { } async fn anvil_startup( -) -> Result, Wallet>>> { +) -> Result<(Arc, Wallet>>, AnvilInstance)> { // Create an anvil instance // No blocktime mines a new block for each tx. let anvil = Anvil::new().spawn(); + println!("Anvil endpoint: {}", anvil.endpoint()); // Create a client let provider = Provider::::try_from(anvil.endpoint()) @@ -69,7 +71,7 @@ async fn anvil_startup( wallet.with_chain_id(anvil.chain_id()), )); - Ok(client) + Ok((client, anvil)) } async fn arbiter_startup() -> Result> { @@ -134,9 +136,9 @@ async fn stateful_call_loop(arbiter_token: arbiter_toke Ok(()) } -async fn mixture_loop( - arbiter_math: ArbiterMath, - arbiter_token: arbiter_token::ArbiterToken, +async fn _mixture_loop( + _arbiter_math: ArbiterMath, + _arbiter_token: arbiter_token::ArbiterToken, ) -> Result<()> { Ok(()) } From dd9101b67f4b62402b194f4e604c258ac37bb409 Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Aug 2023 11:50:03 -0600 Subject: [PATCH 3/7] added bench cfg --- benches/src/main.rs | 71 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/benches/src/main.rs b/benches/src/main.rs index 2fe09ed3..41dcee2a 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -2,8 +2,8 @@ // use ethers_signers::LocalWallet; // use ethers_middleware::SignerMiddleware; // use ethers_core::types::{Address, TransactionRequest}; -use std::convert::TryFrom; use std::{ + convert::TryFrom, sync::Arc, time::{Duration, Instant}, }; @@ -18,14 +18,13 @@ use arbiter_core::{ manager::Manager, middleware::RevmMiddleware, }; -use ethers::types::Address; -use ethers::utils::AnvilInstance; use ethers::{ - core::{utils::Anvil, k256::ecdsa::SigningKey}, + core::{k256::ecdsa::SigningKey, utils::Anvil}, middleware::SignerMiddleware, providers::{Http, Middleware, Provider}, signers::{LocalWallet, Signer, Wallet}, - types::{I256, U256}, + types::{Address, I256, U256}, + utils::AnvilInstance, }; const NUM_STEPS: usize = 1000; @@ -46,15 +45,54 @@ async fn main() -> Result<()> { let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await?; stateless_call_loop(arbiter_math, label).await?; stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label).await?; - }, + } _ => panic!("Invalid argument"), }; Ok(()) } -async fn anvil_startup( -) -> Result<(Arc, Wallet>>, AnvilInstance)> { +#[cfg(bench)] +use test::Bencher; +#[cfg(bench)] +use std::process::Termination; + +#[cfg(bench)] +fn anvil(b: &mut Bencher) -> impl Termination { + let runtime = tokio::runtime::Runtime::new().unwrap(); + b.iter(|| { + runtime.block_on(async { + let label = "anvil"; + let (client, _anvil) = anvil_startup().await.unwrap(); + let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await.unwrap(); + stateless_call_loop(arbiter_math, label).await.unwrap(); + stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label) + .await + .unwrap(); + }) + }); +} + +#[cfg(bench)] +fn arbiter(b: &mut Bencher) -> impl Termination { + let runtime = tokio::runtime::Runtime::new().unwrap(); + b.iter(|| { + runtime.block_on(async { + let label = "arbiter"; + let client = arbiter_startup().await.unwrap(); + let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await.unwrap(); + stateless_call_loop(arbiter_math, label).await.unwrap(); + stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label) + .await + .unwrap(); + }) + }); +} + +async fn anvil_startup() -> Result<( + Arc, Wallet>>, + AnvilInstance, +)> { // Create an anvil instance // No blocktime mines a new block for each tx. let anvil = Anvil::new().spawn(); @@ -109,7 +147,10 @@ async fn deployments( Ok((arbiter_math, arbiter_token)) } -async fn stateless_call_loop(arbiter_math: ArbiterMath, label: &str) -> Result<()> { +async fn stateless_call_loop( + arbiter_math: ArbiterMath, + label: &str, +) -> Result<()> { let iwad = I256::from(10_u128.pow(18)); let start = Instant::now(); for _ in 0..NUM_STEPS { @@ -121,15 +162,15 @@ async fn stateless_call_loop(arbiter_math: ArbiterMath< Ok(()) } -async fn stateful_call_loop(arbiter_token: arbiter_token::ArbiterToken, mint_address: Address, label: &str) -> Result<()> { +async fn stateful_call_loop( + arbiter_token: arbiter_token::ArbiterToken, + mint_address: Address, + label: &str, +) -> Result<()> { let wad = U256::from(10_u128.pow(18)); let start = Instant::now(); for _ in 0..NUM_STEPS { - arbiter_token - .mint(mint_address, wad) - .send() - .await? - .await?; + arbiter_token.mint(mint_address, wad).send().await?.await?; } let duration = start.elapsed(); println!("Time elapsed in {} mint loop is: {:?}", label, duration); From a3fd897db5f280c63f4921510c1e71e67a08f23e Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Aug 2023 12:40:44 -0600 Subject: [PATCH 4/7] finalize benchmarks and report --- README.md | 27 ++++++ benches/Cargo.toml | 4 +- benches/src/main.rs | 211 ++++++++++++++++++++++++++++++-------------- 3 files changed, 176 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 199a88fd..3b45f63e 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,33 @@ cargo doc --workspace --no-deps --open This will generate and open the docs in your browser. From there, you can look at the documentation for each crate in the Arbiter workspace. We will post both crates to crates.io once we have removed any and all Github linked crates. +## Benchmarks + +The closest benchmark we have to Arbiter is running [Anvil](https://github.com/foundry-rs/foundry/tree/master/crates/anvil) and streaming transactions there. The biggest reasons why we chose to build Arbiter was to gain more control over the EVM environment and to have a more robust simulation framework, but we also wanted to gain in speed. Preliminary benchmarks against Anvil are given in the following table. + +| Operation | Arbiter | Anvil | Relative Difference | +|-----------------|-----------|-------------|---------------------| +| Deploy | 282.38µs | 8.82159ms | ~31x | +| Stateless Call | 3.0696ms | 15.17205ms | ~5x | +| Stateful Call | 1.63895ms | 161.18949ms | ~98x | + +The above can be described by: +- Deploy: Deploying a contract to the EVM. We deployed `ArbiterToken` and `ArbiterMath` in this call. +- Stateless Call: Calling a contract that does not mutate state. We called `ArbiterMath`'s `cdf` function 100 times in this call. +- Stateful Call: Calling a contract that mutates state. We called `ArbiterToken`'s `mint` function 100 times in this call. + +All the times were achieved with the release profile and averaged over 100 runs. Anvil was set to mine blocks for each transaction as opposed to setting an enforced block time. + +The benchmarking code can be found in the `benches/` directory. The above was achieved running `cargo install --path ./benches` to install the release profile binary, then running: +- `benches arbiter` +- `benches anvil` +Improvements could be made with a `cfg(bench)`, but bugs were found in compilation there with nightly rust at time of testing. + +Times were achieved on an Apple Macbook Pro M1 Max with 8 performance and 2 efficiency cores, and with 32GB of RAM. + +Please let us know if you find any issues with these benchmarks or if you have any suggestions on how to improve them! + + ## Contributing See our [Contributing Guidelines](https://github.com/primitivefinance/arbiter/blob/main/.github/CONTRIBUTING.md) diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 44fe09ab..64c9dc4e 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -9,4 +9,6 @@ edition = "2021" tokio = "1.32.0" arbiter-core = { path = "../arbiter-core" } ethers = { git = "https://github.com/primitivefinance/ethers-rs.git"} # Custom fork for middleware integration -anyhow = "1.0.75" \ No newline at end of file +anyhow = "1.0.75" +log = "0.4.20" +env_logger = "0.10.0" \ No newline at end of file diff --git a/benches/src/main.rs b/benches/src/main.rs index 41dcee2a..69851240 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -26,67 +26,109 @@ use ethers::{ types::{Address, I256, U256}, utils::AnvilInstance, }; +use log::info; -const NUM_STEPS: usize = 1000; +const ENV_LABEL: &str = "env"; + +const NUM_BENCH_ITERATIONS: usize = 100; +const NUM_LOOP_STEPS: usize = 100; + +#[derive(Debug)] +struct BenchDurations { + deploy: Duration, + stateless_call: Duration, + stateful_call: Duration, +} #[tokio::main] async fn main() -> Result<()> { + // Set up logging. + if std::env::var("RUST_LOG").is_err() { + std::env::set_var("RUST_LOG", "error"); + } + env_logger::init(); + + // Get input argument. let args: Vec = std::env::args().collect(); let args = args.get(1).unwrap().as_str(); - match args { - label @ "anvil" => { - let (client, _anvil_instance) = anvil_startup().await?; - let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await?; - stateless_call_loop(arbiter_math, label).await?; - stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label).await?; - } - label @ "arbiter" => { - let client = arbiter_startup().await?; - let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await?; - stateless_call_loop(arbiter_math, label).await?; - stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label).await?; + let mut durations = vec![]; + + let ten_percent = NUM_BENCH_ITERATIONS / 10; + + for index in 0..NUM_BENCH_ITERATIONS { + durations.push(match args { + label @ "anvil" => { + // Start up Anvil with a client. + let (client, _anvil_instance) = anvil_startup().await?; + let duration = bencher(client, label).await?; + drop(_anvil_instance); + duration + } + label @ "arbiter" => { + let (client, mut manager) = arbiter_startup().await?; + let duration = bencher(client, label).await?; + manager.stop_environment(ENV_LABEL)?; + duration + } + _ => panic!("Invalid argument"), + }); + if index % ten_percent == 0 { + println!("{index}% complete"); } - _ => panic!("Invalid argument"), + } + + let sum_durations = durations.iter().fold( + BenchDurations { + deploy: Duration::default(), + stateless_call: Duration::default(), + stateful_call: Duration::default(), + }, + |acc, duration| BenchDurations { + deploy: acc.deploy + duration.deploy, + stateless_call: acc.stateless_call + duration.stateless_call, + stateful_call: acc.stateful_call + duration.stateful_call, + }, + ); + + // let len = durations.len() as u32; + let average_durations = BenchDurations { + deploy: sum_durations.deploy / NUM_LOOP_STEPS as u32, + stateless_call: sum_durations.stateless_call / NUM_LOOP_STEPS as u32, + stateful_call: sum_durations.stateful_call / NUM_LOOP_STEPS as u32, }; + println!("Average durations: {:?}", average_durations); + Ok(()) } -#[cfg(bench)] -use test::Bencher; -#[cfg(bench)] -use std::process::Termination; +async fn bencher(client: Arc, label: &str) -> Result { + // Track the duration for each part of the benchmark. + let mut total_deploy_duration = 0; + let mut total_stateless_call_duration = 0; + let mut total_stateful_call_duration = 0; -#[cfg(bench)] -fn anvil(b: &mut Bencher) -> impl Termination { - let runtime = tokio::runtime::Runtime::new().unwrap(); - b.iter(|| { - runtime.block_on(async { - let label = "anvil"; - let (client, _anvil) = anvil_startup().await.unwrap(); - let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await.unwrap(); - stateless_call_loop(arbiter_math, label).await.unwrap(); - stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label) - .await - .unwrap(); - }) - }); -} + // Deploy `ArbiterMath` and `ArbiterToken` contracts and tally up how long this + // takes. + let (arbiter_math, arbiter_token, deploy_duration) = deployments(client.clone(), label).await?; + total_deploy_duration += deploy_duration.as_micros(); -#[cfg(bench)] -fn arbiter(b: &mut Bencher) -> impl Termination { - let runtime = tokio::runtime::Runtime::new().unwrap(); - b.iter(|| { - runtime.block_on(async { - let label = "arbiter"; - let client = arbiter_startup().await.unwrap(); - let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await.unwrap(); - stateless_call_loop(arbiter_math, label).await.unwrap(); - stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label) - .await - .unwrap(); - }) - }); + // Call `cdf` `NUM_LOOP_STEPS` times on `ArbiterMath` and tally up how long this + // takes. + let stateless_call_duration = stateless_call_loop(arbiter_math, label).await?; + total_stateless_call_duration += stateless_call_duration.as_micros(); + + // Call `mint` `NUM_LOOP_STEPS` times on `ArbiterToken` and tally up how long + // this takes. + let statefull_call_duration = + stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label).await?; + total_stateful_call_duration += statefull_call_duration.as_micros(); + + Ok(BenchDurations { + deploy: Duration::from_micros(total_deploy_duration as u64), + stateless_call: Duration::from_micros(total_stateless_call_duration as u64), + stateful_call: Duration::from_micros(total_stateful_call_duration as u64), + }) } async fn anvil_startup() -> Result<( @@ -96,7 +138,6 @@ async fn anvil_startup() -> Result<( // Create an anvil instance // No blocktime mines a new block for each tx. let anvil = Anvil::new().spawn(); - println!("Anvil endpoint: {}", anvil.endpoint()); // Create a client let provider = Provider::::try_from(anvil.endpoint()) @@ -112,28 +153,28 @@ async fn anvil_startup() -> Result<( Ok((client, anvil)) } -async fn arbiter_startup() -> Result> { +async fn arbiter_startup() -> Result<(Arc, Manager)> { let mut manager = Manager::new(); let params = EnvironmentParameters { block_rate: 10.0, seed: 0, }; - manager.add_environment("env", params)?; + manager.add_environment(ENV_LABEL, params)?; let client = Arc::new(RevmMiddleware::new( - manager.environments.get("env").unwrap(), + manager.environments.get(ENV_LABEL).unwrap(), Some("name".to_string()), )); - manager.start_environment("env")?; + manager.start_environment(ENV_LABEL)?; - Ok(client) + Ok((client, manager)) } async fn deployments( client: Arc, label: &str, -) -> Result<(ArbiterMath, ArbiterToken)> { +) -> Result<(ArbiterMath, ArbiterToken, Duration)> { let start = Instant::now(); let arbiter_math = ArbiterMath::deploy(client.clone(), ())?.send().await?; let arbiter_token = arbiter_token::ArbiterToken::deploy( @@ -143,43 +184,83 @@ async fn deployments( .send() .await?; let duration = start.elapsed(); - println!("Time elapsed in {} deployment is: {:?}", label, duration); - Ok((arbiter_math, arbiter_token)) + info!("Time elapsed in {} deployment is: {:?}", label, duration); + + Ok((arbiter_math, arbiter_token, duration)) } async fn stateless_call_loop( arbiter_math: ArbiterMath, label: &str, -) -> Result<()> { +) -> Result { let iwad = I256::from(10_u128.pow(18)); let start = Instant::now(); - for _ in 0..NUM_STEPS { + for _ in 0..NUM_LOOP_STEPS { arbiter_math.cdf(iwad).call().await?; } let duration = start.elapsed(); + info!("Time elapsed in {} cdf loop is: {:?}", label, duration); - println!("Time elapsed in {} cdf loop is: {:?}", label, duration); - Ok(()) + Ok(duration) } async fn stateful_call_loop( arbiter_token: arbiter_token::ArbiterToken, mint_address: Address, label: &str, -) -> Result<()> { +) -> Result { let wad = U256::from(10_u128.pow(18)); let start = Instant::now(); - for _ in 0..NUM_STEPS { + for _ in 0..NUM_LOOP_STEPS { arbiter_token.mint(mint_address, wad).send().await?.await?; } let duration = start.elapsed(); - println!("Time elapsed in {} mint loop is: {:?}", label, duration); - Ok(()) + info!("Time elapsed in {} mint loop is: {:?}", label, duration); + + Ok(duration) } async fn _mixture_loop( _arbiter_math: ArbiterMath, _arbiter_token: arbiter_token::ArbiterToken, ) -> Result<()> { - Ok(()) + todo!("Have a loop here that takes a few different types of calls at once.") +} + +#[cfg(bench)] +use std::process::Termination; + +#[cfg(bench)] +use test::Bencher; + +#[cfg(bench)] +fn anvil(b: &mut Bencher) -> impl Termination { + let runtime = tokio::runtime::Runtime::new().unwrap(); + b.iter(|| { + runtime.block_on(async { + let label = "anvil"; + let (client, _anvil) = anvil_startup().await.unwrap(); + let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await.unwrap(); + stateless_call_loop(arbiter_math, label).await.unwrap(); + stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label) + .await + .unwrap(); + }) + }); +} + +#[cfg(bench)] +fn arbiter(b: &mut Bencher) -> impl Termination { + let runtime = tokio::runtime::Runtime::new().unwrap(); + b.iter(|| { + runtime.block_on(async { + let label = "arbiter"; + let client = arbiter_startup().await.unwrap(); + let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await.unwrap(); + stateless_call_loop(arbiter_math, label).await.unwrap(); + stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label) + .await + .unwrap(); + }) + }); } From f0d2369eeb31ab3e3f150b6a5a2c68fb7391fd2c Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Aug 2023 13:49:16 -0600 Subject: [PATCH 5/7] fix linter issues and CI --- .github/workflows/lint.yaml | 6 +++--- arbiter-core/src/middleware.rs | 17 +++++++++-------- arbiter-core/src/tests/management.rs | 10 +++++++--- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index a0888fc7..310a9309 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -17,11 +17,11 @@ jobs: - name: install rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: nightly components: rustfmt - name: cargo fmt - run: cargo fmt --all -- --check + run: cargo +nightly fmt --all -- --check clippy: name: clippy @@ -37,4 +37,4 @@ jobs: components: clippy - name: cargo clippy - run: cargo clippy --all-targets --all-features -- -D warnings + run: cargo clippy --all --all-features -- -D warnings diff --git a/arbiter-core/src/middleware.rs b/arbiter-core/src/middleware.rs index f3e3189b..e65f0854 100644 --- a/arbiter-core/src/middleware.rs +++ b/arbiter-core/src/middleware.rs @@ -550,13 +550,13 @@ impl JsonRpcClient for Connection { let value = serde_json::to_value(¶ms)?; // Take this value as an array then cast it to a string - let str = value.as_array().ok_or(ProviderError::CustomError(format!( + let str = value.as_array().ok_or(ProviderError::CustomError( "The params value passed to the `Connection` via a `request` was empty. - This is likely due to not specifying a specific `Filter` ID!" - )))?[0] - .as_str().ok_or(ProviderError::CustomError(format!( - "The params value passed to the `Connection` via a `request` could not be later cast to `str`!" - )))?; + This is likely due to not specifying a specific `Filter` ID!".to_string() + ))?[0] + .as_str().ok_or(ProviderError::CustomError( + "The params value passed to the `Connection` via a `request` could not be later cast to `str`!".to_string() + ))?; // Now get the `U256` ID via the string decoded from hex radix. let id = ethers::types::U256::from_str_radix(str, 16) @@ -569,9 +569,10 @@ impl JsonRpcClient for Connection { let filter_receiver = filter_receivers .get_mut(&id) - .ok_or(ProviderError::CustomError(format!( + .ok_or(ProviderError::CustomError( "The filter ID does not seem to match any that this client owns!" - )))?; + .to_string(), + ))?; let mut logs = vec![]; let filtered_params = FilteredParams::new(Some(filter_receiver.filter.clone())); if let Ok(received_logs) = filter_receiver.receiver.recv() { diff --git a/arbiter-core/src/tests/management.rs b/arbiter-core/src/tests/management.rs index 17bee7ae..da3a9860 100644 --- a/arbiter-core/src/tests/management.rs +++ b/arbiter-core/src/tests/management.rs @@ -106,8 +106,12 @@ async fn stop_environment_after_transactions() -> Result<()> { manager.add_environment(TEST_ENV_LABEL, params).unwrap(); manager.start_environment(TEST_ENV_LABEL).unwrap(); - // Send some transactions (e.g., deploy `ArbiterMath` which is easy and has no args) - let client = Arc::new(RevmMiddleware::new(manager.environments.get(TEST_ENV_LABEL).unwrap(), Some(TEST_SIGNER_SEED_AND_LABEL.to_string()))); + // Send some transactions (e.g., deploy `ArbiterMath` which is easy and has no + // args) + let client = Arc::new(RevmMiddleware::new( + manager.environments.get(TEST_ENV_LABEL).unwrap(), + Some(TEST_SIGNER_SEED_AND_LABEL.to_string()), + )); ArbiterMath::deploy(client, ())?.send().await?; manager.stop_environment(TEST_ENV_LABEL).unwrap(); @@ -121,4 +125,4 @@ async fn stop_environment_after_transactions() -> Result<()> { State::Stopped ); Ok(()) -} \ No newline at end of file +} From f8be46d161749ae1277f1fc134ff114c30bc47a4 Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Aug 2023 16:06:25 -0600 Subject: [PATCH 6/7] refactor and fix a few errors --- README.md | 17 +- arbiter-core/Cargo.toml | 5 + .../main.rs => arbiter-core/benches/bench.rs | 151 ++++++------------ benches/Cargo.toml | 14 -- 4 files changed, 61 insertions(+), 126 deletions(-) rename benches/src/main.rs => arbiter-core/benches/bench.rs (56%) delete mode 100644 benches/Cargo.toml diff --git a/README.md b/README.md index 3b45f63e..e1086142 100644 --- a/README.md +++ b/README.md @@ -88,23 +88,20 @@ We will post both crates to crates.io once we have removed any and all Github li The closest benchmark we have to Arbiter is running [Anvil](https://github.com/foundry-rs/foundry/tree/master/crates/anvil) and streaming transactions there. The biggest reasons why we chose to build Arbiter was to gain more control over the EVM environment and to have a more robust simulation framework, but we also wanted to gain in speed. Preliminary benchmarks against Anvil are given in the following table. -| Operation | Arbiter | Anvil | Relative Difference | -|-----------------|-----------|-------------|---------------------| -| Deploy | 282.38µs | 8.82159ms | ~31x | -| Stateless Call | 3.0696ms | 15.17205ms | ~5x | -| Stateful Call | 1.63895ms | 161.18949ms | ~98x | +| Operation | Arbiter | Anvil | Relative Difference | +|-----------------|------------|--------------|---------------------| +| Deploy | 254.873µs | 8.525ms | ~33.44x | +| Stateless Call | 4.657507ms | 14.605913ms | ~3.14x | +| Stateful Call | 921.762µs | 160.975985ms | ~174.64x | The above can be described by: - Deploy: Deploying a contract to the EVM. We deployed `ArbiterToken` and `ArbiterMath` in this call. - Stateless Call: Calling a contract that does not mutate state. We called `ArbiterMath`'s `cdf` function 100 times in this call. - Stateful Call: Calling a contract that mutates state. We called `ArbiterToken`'s `mint` function 100 times in this call. -All the times were achieved with the release profile and averaged over 100 runs. Anvil was set to mine blocks for each transaction as opposed to setting an enforced block time. +All the times were achieved with the release profile and averaged over 1000 runs. Anvil was set to mine blocks for each transaction as opposed to setting an enforced block time. -The benchmarking code can be found in the `benches/` directory. The above was achieved running `cargo install --path ./benches` to install the release profile binary, then running: -- `benches arbiter` -- `benches anvil` -Improvements could be made with a `cfg(bench)`, but bugs were found in compilation there with nightly rust at time of testing. +The benchmarking code can be found in the `arbiter-core/benches/` directory. The above was achieved running `cargo bench --package arbiter-core` which will automatically run with the release profile. Times were achieved on an Apple Macbook Pro M1 Max with 8 performance and 2 efficiency cores, and with 32GB of RAM. diff --git a/arbiter-core/Cargo.toml b/arbiter-core/Cargo.toml index 3c08f73c..82782906 100644 --- a/arbiter-core/Cargo.toml +++ b/arbiter-core/Cargo.toml @@ -41,3 +41,8 @@ env_logger = "0.10.0" test-log = "0.2.12" futures = "0.3.28" assert_matches = "1.5" + +[[bench]] +name = "bench" +path = "benches/bench.rs" +harness = false \ No newline at end of file diff --git a/benches/src/main.rs b/arbiter-core/benches/bench.rs similarity index 56% rename from benches/src/main.rs rename to arbiter-core/benches/bench.rs index 69851240..621700fd 100644 --- a/benches/src/main.rs +++ b/arbiter-core/benches/bench.rs @@ -1,7 +1,3 @@ -// use ethers_providers::{Middleware, Provider, Http}; -// use ethers_signers::LocalWallet; -// use ethers_middleware::SignerMiddleware; -// use ethers_core::types::{Address, TransactionRequest}; use std::{ convert::TryFrom, sync::Arc, @@ -30,7 +26,7 @@ use log::info; const ENV_LABEL: &str = "env"; -const NUM_BENCH_ITERATIONS: usize = 100; +const NUM_BENCH_ITERATIONS: usize = 1000; const NUM_LOOP_STEPS: usize = 100; #[derive(Debug)] @@ -42,63 +38,59 @@ struct BenchDurations { #[tokio::main] async fn main() -> Result<()> { - // Set up logging. - if std::env::var("RUST_LOG").is_err() { - std::env::set_var("RUST_LOG", "error"); - } - env_logger::init(); - - // Get input argument. - let args: Vec = std::env::args().collect(); - let args = args.get(1).unwrap().as_str(); - let mut durations = vec![]; + // Choose the benchmark group items by label. + let group = ["anvil", "arbiter"]; + // Set up for showing percentage done. let ten_percent = NUM_BENCH_ITERATIONS / 10; - for index in 0..NUM_BENCH_ITERATIONS { - durations.push(match args { - label @ "anvil" => { - // Start up Anvil with a client. - let (client, _anvil_instance) = anvil_startup().await?; - let duration = bencher(client, label).await?; - drop(_anvil_instance); - duration + for item in group { + // Count up total durations for each part of the benchmark. + let mut durations = Vec::with_capacity(NUM_BENCH_ITERATIONS); + println!("Running {item} benchmark"); + + for index in 0..NUM_BENCH_ITERATIONS { + durations.push(match item { + label @ "anvil" => { + let (client, _anvil_instance) = anvil_startup().await?; + let duration = bencher(client, label).await?; + drop(_anvil_instance); + duration + } + label @ "arbiter" => { + let (client, mut manager) = arbiter_startup().await?; + let duration = bencher(client, label).await?; + manager.stop_environment(ENV_LABEL)?; + duration + } + _ => panic!("Invalid argument"), + }); + if index % ten_percent == 0 { + println!("{index} out of {NUM_BENCH_ITERATIONS} complete"); } - label @ "arbiter" => { - let (client, mut manager) = arbiter_startup().await?; - let duration = bencher(client, label).await?; - manager.stop_environment(ENV_LABEL)?; - duration - } - _ => panic!("Invalid argument"), - }); - if index % ten_percent == 0 { - println!("{index}% complete"); } + let sum_durations = durations.iter().fold( + BenchDurations { + deploy: Duration::default(), + stateless_call: Duration::default(), + stateful_call: Duration::default(), + }, + |acc, duration| BenchDurations { + deploy: acc.deploy + duration.deploy, + stateless_call: acc.stateless_call + duration.stateless_call, + stateful_call: acc.stateful_call + duration.stateful_call, + }, + ); + + let average_durations = BenchDurations { + deploy: sum_durations.deploy / NUM_BENCH_ITERATIONS as u32, + stateless_call: sum_durations.stateless_call / NUM_BENCH_ITERATIONS as u32, + stateful_call: sum_durations.stateful_call / NUM_BENCH_ITERATIONS as u32, + }; + + println!("Average durations for {item}: {:?}", average_durations); } - let sum_durations = durations.iter().fold( - BenchDurations { - deploy: Duration::default(), - stateless_call: Duration::default(), - stateful_call: Duration::default(), - }, - |acc, duration| BenchDurations { - deploy: acc.deploy + duration.deploy, - stateless_call: acc.stateless_call + duration.stateless_call, - stateful_call: acc.stateful_call + duration.stateful_call, - }, - ); - - // let len = durations.len() as u32; - let average_durations = BenchDurations { - deploy: sum_durations.deploy / NUM_LOOP_STEPS as u32, - stateless_call: sum_durations.stateless_call / NUM_LOOP_STEPS as u32, - stateful_call: sum_durations.stateful_call / NUM_LOOP_STEPS as u32, - }; - - println!("Average durations: {:?}", average_durations); - Ok(()) } @@ -135,8 +127,8 @@ async fn anvil_startup() -> Result<( Arc, Wallet>>, AnvilInstance, )> { - // Create an anvil instance - // No blocktime mines a new block for each tx. + // Create an Anvil instance + // No blocktime mines a new block for each tx, which is fastest. let anvil = Anvil::new().spawn(); // Create a client @@ -219,48 +211,3 @@ async fn stateful_call_loop( Ok(duration) } - -async fn _mixture_loop( - _arbiter_math: ArbiterMath, - _arbiter_token: arbiter_token::ArbiterToken, -) -> Result<()> { - todo!("Have a loop here that takes a few different types of calls at once.") -} - -#[cfg(bench)] -use std::process::Termination; - -#[cfg(bench)] -use test::Bencher; - -#[cfg(bench)] -fn anvil(b: &mut Bencher) -> impl Termination { - let runtime = tokio::runtime::Runtime::new().unwrap(); - b.iter(|| { - runtime.block_on(async { - let label = "anvil"; - let (client, _anvil) = anvil_startup().await.unwrap(); - let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await.unwrap(); - stateless_call_loop(arbiter_math, label).await.unwrap(); - stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label) - .await - .unwrap(); - }) - }); -} - -#[cfg(bench)] -fn arbiter(b: &mut Bencher) -> impl Termination { - let runtime = tokio::runtime::Runtime::new().unwrap(); - b.iter(|| { - runtime.block_on(async { - let label = "arbiter"; - let client = arbiter_startup().await.unwrap(); - let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await.unwrap(); - stateless_call_loop(arbiter_math, label).await.unwrap(); - stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label) - .await - .unwrap(); - }) - }); -} diff --git a/benches/Cargo.toml b/benches/Cargo.toml deleted file mode 100644 index 64c9dc4e..00000000 --- a/benches/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "benches" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -tokio = "1.32.0" -arbiter-core = { path = "../arbiter-core" } -ethers = { git = "https://github.com/primitivefinance/ethers-rs.git"} # Custom fork for middleware integration -anyhow = "1.0.75" -log = "0.4.20" -env_logger = "0.10.0" \ No newline at end of file From dd4111cde2ccfcc0072220565c3cf987ed936e2f Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Aug 2023 16:29:00 -0600 Subject: [PATCH 7/7] added one last bench --- README.md | 8 +++++--- arbiter-core/benches/bench.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e1086142..2da9cc90 100644 --- a/README.md +++ b/README.md @@ -90,12 +90,14 @@ The closest benchmark we have to Arbiter is running [Anvil](https://github.com/f | Operation | Arbiter | Anvil | Relative Difference | |-----------------|------------|--------------|---------------------| -| Deploy | 254.873µs | 8.525ms | ~33.44x | -| Stateless Call | 4.657507ms | 14.605913ms | ~3.14x | -| Stateful Call | 921.762µs | 160.975985ms | ~174.64x | +| Deploy | 241.819µs | 8.215446ms | ~33.97x | +| Lookup | 480.319µs | 13.052063ms | ~27.17x | +| Stateless Call | 4.03235ms | 10.238771ms | ~2.53x | +| Stateful Call | 843.296µs | 153.102478ms | ~181.55x | The above can be described by: - Deploy: Deploying a contract to the EVM. We deployed `ArbiterToken` and `ArbiterMath` in this call. +- Lookup: Looking up a the `balanceOf` for a client's address for `ArbiterToken`. - Stateless Call: Calling a contract that does not mutate state. We called `ArbiterMath`'s `cdf` function 100 times in this call. - Stateful Call: Calling a contract that mutates state. We called `ArbiterToken`'s `mint` function 100 times in this call. diff --git a/arbiter-core/benches/bench.rs b/arbiter-core/benches/bench.rs index 621700fd..3ae0475c 100644 --- a/arbiter-core/benches/bench.rs +++ b/arbiter-core/benches/bench.rs @@ -32,6 +32,7 @@ const NUM_LOOP_STEPS: usize = 100; #[derive(Debug)] struct BenchDurations { deploy: Duration, + lookup: Duration, stateless_call: Duration, stateful_call: Duration, } @@ -72,11 +73,13 @@ async fn main() -> Result<()> { let sum_durations = durations.iter().fold( BenchDurations { deploy: Duration::default(), + lookup: Duration::default(), stateless_call: Duration::default(), stateful_call: Duration::default(), }, |acc, duration| BenchDurations { deploy: acc.deploy + duration.deploy, + lookup: acc.lookup + duration.lookup, stateless_call: acc.stateless_call + duration.stateless_call, stateful_call: acc.stateful_call + duration.stateful_call, }, @@ -84,6 +87,7 @@ async fn main() -> Result<()> { let average_durations = BenchDurations { deploy: sum_durations.deploy / NUM_BENCH_ITERATIONS as u32, + lookup: sum_durations.lookup / NUM_BENCH_ITERATIONS as u32, stateless_call: sum_durations.stateless_call / NUM_BENCH_ITERATIONS as u32, stateful_call: sum_durations.stateful_call / NUM_BENCH_ITERATIONS as u32, }; @@ -97,6 +101,7 @@ async fn main() -> Result<()> { async fn bencher(client: Arc, label: &str) -> Result { // Track the duration for each part of the benchmark. let mut total_deploy_duration = 0; + let mut total_lookup_duration = 0; let mut total_stateless_call_duration = 0; let mut total_stateful_call_duration = 0; @@ -105,6 +110,11 @@ async fn bencher(client: Arc, label: &str) -> Result let (arbiter_math, arbiter_token, deploy_duration) = deployments(client.clone(), label).await?; total_deploy_duration += deploy_duration.as_micros(); + // Call `balance_of` `NUM_LOOP_STEPS` times on `ArbiterToken` and tally up how + // long basic lookups take. + let lookup_duration = lookup(arbiter_token.clone(), label).await?; + total_lookup_duration += lookup_duration.as_micros(); + // Call `cdf` `NUM_LOOP_STEPS` times on `ArbiterMath` and tally up how long this // takes. let stateless_call_duration = stateless_call_loop(arbiter_math, label).await?; @@ -118,6 +128,7 @@ async fn bencher(client: Arc, label: &str) -> Result Ok(BenchDurations { deploy: Duration::from_micros(total_deploy_duration as u64), + lookup: Duration::from_micros(total_lookup_duration as u64), stateless_call: Duration::from_micros(total_stateless_call_duration as u64), stateful_call: Duration::from_micros(total_stateful_call_duration as u64), }) @@ -181,6 +192,21 @@ async fn deployments( Ok((arbiter_math, arbiter_token, duration)) } +async fn lookup( + arbiter_token: ArbiterToken, + label: &str, +) -> Result { + let address = arbiter_token.client().default_sender().unwrap(); + let start = Instant::now(); + for _ in 0..NUM_LOOP_STEPS { + arbiter_token.balance_of(address).call().await?; + } + let duration = start.elapsed(); + info!("Time elapsed in {} cdf loop is: {:?}", label, duration); + + Ok(duration) +} + async fn stateless_call_loop( arbiter_math: ArbiterMath, label: &str,