Migrate benchmarks to the JSON State Test format

Author: miles170

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "evmc"]
 	path = evmc
 	url = https://github.com/ethereum/evmc
+[submodule "test/benchmarks"]
+	path = test/benchmarks
+	url = https://github.com/ipsilon/evm-benchmarks

CMakeLists.txt

@@ -4,7 +4,7 @@
 
 cmake_minimum_required(VERSION 3.16...3.23)
 
-if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/evmc/.git)
+if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/evmc/.git OR NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/test/benchmarks/.git)
     message(FATAL_ERROR "Git submodules not initialized, execute:\n  git submodule update --init")
 endif()
 

test/bench/CMakeLists.txt

@@ -4,7 +4,7 @@
 
 add_executable(evmone-bench)
 target_include_directories(evmone-bench PRIVATE ${evmone_private_include_dir})
-target_link_libraries(evmone-bench PRIVATE evmone testutils evmc::loader benchmark::benchmark)
+target_link_libraries(evmone-bench PRIVATE evmone testutils evmc::loader evmone::statetestutils benchmark::benchmark)
 target_sources(
     evmone-bench PRIVATE
     bench.cpp

test/bench/bench.cpp

@@ -2,6 +2,7 @@
 // Copyright 2019 The evmone Authors.
 // SPDX-License-Identifier: Apache-2.0
 
+#include "../statetest/statetest.hpp"
 #include "helpers.hpp"
 #include "synthetic_benchmarks.hpp"
 #include <benchmark/benchmark.h>
@@ -12,6 +13,7 @@
 #include <fstream>
 #include <iostream>
 #include <span>
+#include <string>
 
 namespace fs = std::filesystem;
 
@@ -48,49 +50,17 @@ struct BenchmarkCase
     {}
 };
 
-
-constexpr auto runtime_code_extension = ".bin-runtime";
-constexpr auto inputs_extension = ".inputs";
-
 /// Loads the benchmark case's inputs from the inputs file at the given path.
-std::vector<BenchmarkCase::Input> load_inputs(const fs::path& path)
+std::vector<BenchmarkCase::Input> load_inputs(const StateTransitionTest& state_test)
 {
-    enum class state
-    {
-        name,
-        input,
-        expected_output
-    };
-
-    auto inputs_file = std::ifstream{path};
-
     std::vector<BenchmarkCase::Input> inputs;
-    auto st = state::name;
-    std::string input_name;
-    bytes input;
-    for (std::string l; std::getline(inputs_file, l);)
+    for (size_t i = 0; i < state_test.multi_tx.inputs.size(); ++i)
     {
-        switch (st)
-        {
-        case state::name:
-            if (l.empty())
-                continue;  // Skip any empty line.
-            input_name = std::move(l);
-            st = state::input;
-            break;
-
-        case state::input:
-            input = from_hexx(l);
-            st = state::expected_output;
-            break;
-
-        case state::expected_output:
-            inputs.emplace_back(std::move(input_name), std::move(input), from_hexx(l));
-            input_name = {};
-            input = {};
-            st = state::name;
-            break;
-        }
+        const auto input_name = state_test.labels.at(i);
+        const auto input = state_test.multi_tx.inputs[i];
+        const bytes expected_output;
+
+        inputs.emplace_back(std::move(input_name), std::move(input), std::move(expected_output));
     }
 
     return inputs;
@@ -99,16 +69,14 @@ std::vector<BenchmarkCase::Input> load_inputs(const fs::path& path)
 /// Loads a benchmark case from a file at `path` and all its inputs from the matching inputs file.
 BenchmarkCase load_benchmark(const fs::path& path, const std::string& name_prefix)
 {
-    const auto name = name_prefix + path.stem().string();
+    auto state_test = evmone::test::load_state_test(path);
 
-    std::ifstream file{path};
-    std::string code_hexx{std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{}};
-    BenchmarkCase b{name, from_hexx(code_hexx)};
+    const auto name = name_prefix + path.stem().string();
+    const auto code = state_test.pre_state.get(state_test.multi_tx.to.value()).code;
+    const auto inputs = load_inputs(state_test);
 
-    auto inputs_path = path;
-    inputs_path.replace_extension(inputs_extension);
-    if (fs::exists(inputs_path))
-        b.inputs = load_inputs(inputs_path);
+    BenchmarkCase b{name, code};
+    b.inputs = inputs;
 
     return b;
 }
@@ -124,7 +92,7 @@ std::vector<BenchmarkCase> load_benchmarks_from_dir(  // NOLINT(misc-no-recursio
     {
         if (e.is_directory())
             subdirs.emplace_back(e);
-        else if (e.path().extension() == runtime_code_extension)
+        else if (e.path().extension() == ".json")
             code_files.emplace_back(e);
     }