Migrate benchmarks to the JSON State Test format

Author: miles170

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "evmc"]
 	path = evmc
 	url = https://github.com/ethereum/evmc
+[submodule "test/benchmarks"]
+	path = test/benchmarks
+	url = https://github.com/ipsilon/evm-benchmarks

CMakeLists.txt

@@ -4,7 +4,7 @@
 
 cmake_minimum_required(VERSION 3.16...3.23)
 
-if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/evmc/.git)
+if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/evmc/.git OR NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/test/benchmarks/.git)
     message(FATAL_ERROR "Git submodules not initialized, execute:\n  git submodule update --init")
 endif()
 

test/bench/CMakeLists.txt

@@ -12,6 +12,7 @@ target_sources(
     bench.cpp
     helpers.hpp
     synthetic_benchmarks.cpp synthetic_benchmarks.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../statetest/statetest_loader.cpp
 )
 
 # Tests

test/bench/bench.cpp

@@ -2,6 +2,7 @@
 // Copyright 2019 The evmone Authors.
 // SPDX-License-Identifier: Apache-2.0
 
+#include "../statetest/statetest.hpp"
 #include "helpers.hpp"
 #include "synthetic_benchmarks.hpp"
 #include <benchmark/benchmark.h>
@@ -47,49 +48,17 @@ struct BenchmarkCase
     {}
 };
 
-
-constexpr auto runtime_code_extension = ".bin-runtime";
-constexpr auto inputs_extension = ".inputs";
-
 /// Loads the benchmark case's inputs from the inputs file at the given path.
-std::vector<BenchmarkCase::Input> load_inputs(const fs::path& path)
+std::vector<BenchmarkCase::Input> load_inputs(const StateTransitionTest& state_test)
 {
-    enum class state
-    {
-        name,
-        input,
-        expected_output
-    };
-
-    auto inputs_file = std::ifstream{path};
-
     std::vector<BenchmarkCase::Input> inputs;
-    auto st = state::name;
-    std::string input_name;
-    bytes input;
-    for (std::string l; std::getline(inputs_file, l);)
+    for (size_t i = 0; i < state_test.multi_tx.inputs.size(); ++i)
     {
-        switch (st)
-        {
-        case state::name:
-            if (l.empty())
-                continue;  // Skip any empty line.
-            input_name = std::move(l);
-            st = state::input;
-            break;
-
-        case state::input:
-            input = from_hexx(l);
-            st = state::expected_output;
-            break;
-
-        case state::expected_output:
-            inputs.emplace_back(std::move(input_name), std::move(input), from_hexx(l));
-            input_name = {};
-            input = {};
-            st = state::name;
-            break;
-        }
+        const auto input_name = state_test.info.labels.at(i);
+        const auto input = state_test.multi_tx.inputs[i];
+        bytes expected_output;
+
+        inputs.emplace_back(std::move(input_name), std::move(input), std::move(expected_output));
     }
 
     return inputs;
@@ -98,16 +67,14 @@ std::vector<BenchmarkCase::Input> load_inputs(const fs::path& path)
 /// Loads a benchmark case from a file at `path` and all its inputs from the matching inputs file.
 BenchmarkCase load_benchmark(const fs::path& path, const std::string& name_prefix)
 {
-    const auto name = name_prefix + path.stem().string();
+    auto state_test = evmone::test::load_state_test(path);
 
-    std::ifstream file{path};
-    std::string code_hexx{std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{}};
-    BenchmarkCase b{name, from_hexx(code_hexx)};
+    const auto name = name_prefix + path.stem().string();
+    const auto code = state_test.pre_state.get(state_test.multi_tx.to.value()).code;
+    const auto inputs = load_inputs(state_test);
 
-    auto inputs_path = path;
-    inputs_path.replace_extension(inputs_extension);
-    if (fs::exists(inputs_path))
-        b.inputs = load_inputs(inputs_path);
+    BenchmarkCase b{name, code};
+    b.inputs = inputs;
 
     return b;
 }
@@ -123,7 +90,7 @@ std::vector<BenchmarkCase> load_benchmarks_from_dir(  // NOLINT(misc-no-recursio
     {
         if (e.is_directory())
             subdirs.emplace_back(e);
-        else if (e.path().extension() == runtime_code_extension)
+        else if (e.path().extension() == ".json")
             code_files.emplace_back(e);
     }