refactor and fix a few errors

README.md

@@ -88,23 +88,20 @@ We will post both crates to crates.io once we have removed any and all Github li
 
 The closest benchmark we have to Arbiter is running [Anvil](https://github.com/foundry-rs/foundry/tree/master/crates/anvil) and streaming transactions there. The biggest reasons why we chose to build Arbiter was to gain more control over the EVM environment and to have a more robust simulation framework, but we also wanted to gain in speed. Preliminary benchmarks against Anvil are given in the following table.
 
-| Operation       |  Arbiter  |    Anvil    | Relative Difference |
-|-----------------|-----------|-------------|---------------------|
-| Deploy          | 282.38µs  | 8.82159ms   | ~31x                |
-| Stateless Call  | 3.0696ms  | 15.17205ms  | ~5x                 |
-| Stateful Call   | 1.63895ms | 161.18949ms | ~98x                |
+| Operation       |  Arbiter   |    Anvil     | Relative Difference |
+|-----------------|------------|--------------|---------------------|
+| Deploy          | 254.873µs  | 8.525ms      | ~33.44x             |
+| Stateless Call  | 4.657507ms | 14.605913ms  | ~3.14x              |
+| Stateful Call   | 921.762µs  | 160.975985ms | ~174.64x            |
 
 The above can be described by:
 - Deploy: Deploying a contract to the EVM. We deployed `ArbiterToken` and `ArbiterMath` in this call.
 - Stateless Call: Calling a contract that does not mutate state. We called `ArbiterMath`'s `cdf` function 100 times in this call.
 - Stateful Call: Calling a contract that mutates state. We called `ArbiterToken`'s `mint` function 100 times in this call.
 
-All the times were achieved with the release profile and averaged over 100 runs. Anvil was set to mine blocks for each transaction as opposed to setting an enforced block time. 
+All the times were achieved with the release profile and averaged over 1000 runs. Anvil was set to mine blocks for each transaction as opposed to setting an enforced block time. 
 
-The benchmarking code can be found in the `benches/` directory. The above was achieved running `cargo install --path ./benches` to install the release profile binary, then running:
-- `benches arbiter`
-- `benches anvil`
-Improvements could be made with a `cfg(bench)`, but bugs were found in compilation there with nightly rust at time of testing. 
+The benchmarking code can be found in the `arbiter-core/benches/` directory. The above was achieved running `cargo bench --package arbiter-core` which will automatically run with the release profile.
 
 Times were achieved on an Apple Macbook Pro M1 Max with 8 performance and 2 efficiency cores, and with 32GB of RAM.
 

arbiter-core/Cargo.toml

@@ -41,3 +41,8 @@ env_logger = "0.10.0"
 test-log = "0.2.12"
 futures = "0.3.28"
 assert_matches = "1.5"
+
+[[bench]]
+name = "bench"
+path = "benches/bench.rs"
+harness = false

benches/src/main.rs → arbiter-core/benches/bench.rs

@@ -1,7 +1,3 @@
-// use ethers_providers::{Middleware, Provider, Http};
-// use ethers_signers::LocalWallet;
-// use ethers_middleware::SignerMiddleware;
-// use ethers_core::types::{Address, TransactionRequest};
 use std::{
     convert::TryFrom,
     sync::Arc,
@@ -30,7 +26,7 @@ use log::info;
 
 const ENV_LABEL: &str = "env";
 
-const NUM_BENCH_ITERATIONS: usize = 100;
+const NUM_BENCH_ITERATIONS: usize = 1000;
 const NUM_LOOP_STEPS: usize = 100;
 
 #[derive(Debug)]
@@ -42,63 +38,59 @@ struct BenchDurations {
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    // Set up logging.
-    if std::env::var("RUST_LOG").is_err() {
-        std::env::set_var("RUST_LOG", "error");
-    }
-    env_logger::init();
-
-    // Get input argument.
-    let args: Vec<String> = std::env::args().collect();
-    let args = args.get(1).unwrap().as_str();
-    let mut durations = vec![];
+    // Choose the benchmark group items by label.
+    let group = ["anvil", "arbiter"];
 
+    // Set up for showing percentage done.
     let ten_percent = NUM_BENCH_ITERATIONS / 10;
 
-    for index in 0..NUM_BENCH_ITERATIONS {
-        durations.push(match args {
-            label @ "anvil" => {
-                // Start up Anvil with a client.
-                let (client, _anvil_instance) = anvil_startup().await?;
-                let duration = bencher(client, label).await?;
-                drop(_anvil_instance);
-                duration
+    for item in group {
+        // Count up total durations for each part of the benchmark.
+        let mut durations = Vec::with_capacity(NUM_BENCH_ITERATIONS);
+        println!("Running {item} benchmark");
+
+        for index in 0..NUM_BENCH_ITERATIONS {
+            durations.push(match item {
+                label @ "anvil" => {
+                    let (client, _anvil_instance) = anvil_startup().await?;
+                    let duration = bencher(client, label).await?;
+                    drop(_anvil_instance);
+                    duration
+                }
+                label @ "arbiter" => {
+                    let (client, mut manager) = arbiter_startup().await?;
+                    let duration = bencher(client, label).await?;
+                    manager.stop_environment(ENV_LABEL)?;
+                    duration
+                }
+                _ => panic!("Invalid argument"),
+            });
+            if index % ten_percent == 0 {
+                println!("{index} out of {NUM_BENCH_ITERATIONS} complete");
             }
-            label @ "arbiter" => {
-                let (client, mut manager) = arbiter_startup().await?;
-                let duration = bencher(client, label).await?;
-                manager.stop_environment(ENV_LABEL)?;
-                duration
-            }
-            _ => panic!("Invalid argument"),
-        });
-        if index % ten_percent == 0 {
-            println!("{index}% complete");
         }
+        let sum_durations = durations.iter().fold(
+            BenchDurations {
+                deploy: Duration::default(),
+                stateless_call: Duration::default(),
+                stateful_call: Duration::default(),
+            },
+            |acc, duration| BenchDurations {
+                deploy: acc.deploy + duration.deploy,
+                stateless_call: acc.stateless_call + duration.stateless_call,
+                stateful_call: acc.stateful_call + duration.stateful_call,
+            },
+        );
+
+        let average_durations = BenchDurations {
+            deploy: sum_durations.deploy / NUM_BENCH_ITERATIONS as u32,
+            stateless_call: sum_durations.stateless_call / NUM_BENCH_ITERATIONS as u32,
+            stateful_call: sum_durations.stateful_call / NUM_BENCH_ITERATIONS as u32,
+        };
+
+        println!("Average durations for {item}: {:?}", average_durations);
     }
 
-    let sum_durations = durations.iter().fold(
-        BenchDurations {
-            deploy: Duration::default(),
-            stateless_call: Duration::default(),
-            stateful_call: Duration::default(),
-        },
-        |acc, duration| BenchDurations {
-            deploy: acc.deploy + duration.deploy,
-            stateless_call: acc.stateless_call + duration.stateless_call,
-            stateful_call: acc.stateful_call + duration.stateful_call,
-        },
-    );
-
-    // let len = durations.len() as u32;
-    let average_durations = BenchDurations {
-        deploy: sum_durations.deploy / NUM_LOOP_STEPS as u32,
-        stateless_call: sum_durations.stateless_call / NUM_LOOP_STEPS as u32,
-        stateful_call: sum_durations.stateful_call / NUM_LOOP_STEPS as u32,
-    };
-
-    println!("Average durations: {:?}", average_durations);
-
     Ok(())
 }
 
@@ -135,8 +127,8 @@ async fn anvil_startup() -> Result<(
     Arc<SignerMiddleware<Provider<Http>, Wallet<SigningKey>>>,
     AnvilInstance,
 )> {
-    // Create an anvil instance
-    // No blocktime mines a new block for each tx.
+    // Create an Anvil instance
+    // No blocktime mines a new block for each tx, which is fastest.
     let anvil = Anvil::new().spawn();
 
     // Create a client
@@ -219,48 +211,3 @@ async fn stateful_call_loop<M: Middleware + 'static>(
 
     Ok(duration)
 }
-
-async fn _mixture_loop<M>(
-    _arbiter_math: ArbiterMath<M>,
-    _arbiter_token: arbiter_token::ArbiterToken<M>,
-) -> Result<()> {
-    todo!("Have a loop here that takes a few different types of calls at once.")
-}
-
-#[cfg(bench)]
-use std::process::Termination;
-
-#[cfg(bench)]
-use test::Bencher;
-
-#[cfg(bench)]
-fn anvil(b: &mut Bencher) -> impl Termination {
-    let runtime = tokio::runtime::Runtime::new().unwrap();
-    b.iter(|| {
-        runtime.block_on(async {
-            let label = "anvil";
-            let (client, _anvil) = anvil_startup().await.unwrap();
-            let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await.unwrap();
-            stateless_call_loop(arbiter_math, label).await.unwrap();
-            stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label)
-                .await
-                .unwrap();
-        })
-    });
-}
-
-#[cfg(bench)]
-fn arbiter(b: &mut Bencher) -> impl Termination {
-    let runtime = tokio::runtime::Runtime::new().unwrap();
-    b.iter(|| {
-        runtime.block_on(async {
-            let label = "arbiter";
-            let client = arbiter_startup().await.unwrap();
-            let (arbiter_math, arbiter_token) = deployments(client.clone(), label).await.unwrap();
-            stateless_call_loop(arbiter_math, label).await.unwrap();
-            stateful_call_loop(arbiter_token, client.default_sender().unwrap(), label)
-                .await
-                .unwrap();
-        })
-    });
-}