statetest: Run all tests from a JSON file (#863)

The ethereum/tests has a convention that there is a single test in a JSON file. However, execution-spec-tests don't follow this convention and put multiple tests in a single file. This change loads and executes all tests from a JSON state test file. Fixes #862.
ethereum · Apr 17, 2024 · ed5562f · ed5562f
1 parent 21d7839
commit ed5562f
Show file tree

Hide file tree

Showing 9 changed files with 157 additions and 16 deletions.
diff --git a/test/bench/bench.cpp b/test/bench/bench.cpp
@@ -58,7 +58,7 @@ std::vector<BenchmarkCase::Input> load_inputs(const StateTransitionTest& state_t
 BenchmarkCase load_benchmark(const fs::path& path, const std::string& name_prefix)
 {
     std::ifstream f{path};
-    auto state_test = evmone::test::load_state_test(f);
+    auto state_test = std::move(evmone::test::load_state_tests(f).at(0));
 
     const auto name = name_prefix + path.stem().string();
     const auto code = state_test.pre_state.get(state_test.multi_tx.to.value()).code;

diff --git a/test/integration/statetest/CMakeLists.txt b/test/integration/statetest/CMakeLists.txt
@@ -30,7 +30,7 @@ B\.
   T
 SuiteA\.
   test1
-  test2
+  test2_multi
 ]]
 )
 
@@ -57,13 +57,13 @@ B\.
   T
 SuiteA\.
   test1
-  test2
+  test2_multi
   test1
 .*test/integration/statetest/tests1/B\.
   T
 \.
   test1
-  test2
+  test2_multi
 ]]
 )
 
@@ -76,6 +76,16 @@ set_tests_properties(
     PASS_REGULAR_EXPRESSION "path: Path does not exist: invalid\\.json"
 )
 
+add_test(
+    NAME ${PREFIX}/multi_test
+    COMMAND evmone-statetest ${TESTS1}/SuiteA/test2_multi.json
+)
+set_tests_properties(
+    ${PREFIX}/multi_test PROPERTIES
+    # Make sure both tests in the file are executed (both should fail).
+    PASS_REGULAR_EXPRESSION "test_case_1.*test_case_2"
+)
+
 add_test(
     NAME ${PREFIX}/trace
     COMMAND evmone-statetest ${TESTS1}/SuiteA/test1.json --trace

diff --git a/test/integration/statetest/tests1/SuiteA/test2.json b/test/integration/statetest/tests1/SuiteA/test2.json
diff --git a/test/integration/statetest/tests1/SuiteA/test2_multi.json b/test/integration/statetest/tests1/SuiteA/test2_multi.json
@@ -0,0 +1,96 @@
+{
+  "test_case_1": {
+    "env": {
+      "currentBaseFee": "0x0a",
+      "currentCoinbase": "0x0000000000000000000000000000000000000000",
+      "currentDifficulty": "0x020000",
+      "currentGasLimit": "0xff112233445566",
+      "currentNumber": "0x01",
+      "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000",
+      "currentTimestamp": "0x03e8"
+    },
+    "post": {
+      "London": [
+        {
+          "hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+          "indexes": {
+            "data": 0,
+            "gas": 0,
+            "value": 0
+          },
+          "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"
+        }
+      ]
+    },
+    "pre": {
+      "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+        "balance": "0x0de0b6b3a7640000",
+        "code": "0x",
+        "nonce": "0x00",
+        "storage": {}
+      }
+    },
+    "transaction": {
+      "data": [
+        "0x"
+      ],
+      "gasLimit": [
+        "0x061a80"
+      ],
+      "gasPrice": "0x0a",
+      "nonce": "0x00",
+      "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+      "to": "0x0000000000000000000000000000000000000000",
+      "value": [
+        "0x1"
+      ]
+    }
+  },
+  "test_case_2": {
+    "env": {
+      "currentBaseFee": "0x0a",
+      "currentCoinbase": "0x0000000000000000000000000000000000000000",
+      "currentDifficulty": "0x020000",
+      "currentGasLimit": "0xff112233445566",
+      "currentNumber": "0x01",
+      "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000",
+      "currentTimestamp": "0x03e8"
+    },
+    "post": {
+      "London": [
+        {
+          "hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+          "indexes": {
+            "data": 0,
+            "gas": 0,
+            "value": 0
+          },
+          "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"
+        }
+      ]
+    },
+    "pre": {
+      "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+        "balance": "0x0de0b6b3a7640000",
+        "code": "0x",
+        "nonce": "0x00",
+        "storage": {}
+      }
+    },
+    "transaction": {
+      "data": [
+        "0x"
+      ],
+      "gasLimit": [
+        "0x061a80"
+      ],
+      "gasPrice": "0x0a",
+      "nonce": "0x00",
+      "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+      "to": "0x0000000000000000000000000000000000000000",
+      "value": [
+        "0x2"
+      ]
+    }
+  }
+}
diff --git a/test/statetest/statetest.cpp b/test/statetest/statetest.cpp
@@ -26,7 +26,9 @@ class StateTest : public testing::Test
     void TestBody() final
     {
         std::ifstream f{m_json_test_file};
-        evmone::test::run_state_test(evmone::test::load_state_test(f), m_vm, m_trace);
+        const auto tests = evmone::test::load_state_tests(f);
+        for (const auto& test : tests)
+            evmone::test::run_state_test(test, m_vm, m_trace);
     }
 };
 

diff --git a/test/statetest/statetest.hpp b/test/statetest/statetest.hpp
@@ -53,6 +53,7 @@ struct StateTransitionTest
         std::vector<Expectation> expectations;
     };
 
+    std::string name;
     state::State pre_state;
     state::BlockInfo block;
     TestMultiTransaction multi_tx;
@@ -93,7 +94,7 @@ state::Transaction from_json<state::Transaction>(const json::json& j);
 /// Exports the State (accounts) to JSON format (aka pre/post/alloc state).
 json::json to_json(const std::unordered_map<address, state::Account>& accounts);
 
-StateTransitionTest load_state_test(std::istream& input);
+std::vector<StateTransitionTest> load_state_tests(std::istream& input);
 
 /// Validates an Ethereum state:
 /// - checks that there are no zero-value storage entries,

diff --git a/test/statetest/statetest_loader.cpp b/test/statetest/statetest_loader.cpp
@@ -412,13 +412,8 @@ static void from_json(const json::json& j, StateTransitionTest::Case::Expectatio
     o.exception = j.contains("expectException");
 }
 
-static void from_json(const json::json& j, StateTransitionTest& o)
+static void from_json(const json::json& j_t, StateTransitionTest& o)
 {
-    if (!j.is_object() || j.empty())
-        throw std::invalid_argument{"JSON test must be an object with single key of the test name"};
-
-    const auto& j_t = *j.begin();  // Content is in a dict with the test name.
-
     o.pre_state = from_json<state::State>(j_t.at("pre"));
 
     o.multi_tx = j_t.at("transaction").get<TestMultiTransaction>();
@@ -442,9 +437,19 @@ static void from_json(const json::json& j, StateTransitionTest& o)
     }
 }
 
-StateTransitionTest load_state_test(std::istream& input)
+static void from_json(const json::json& j, std::vector<StateTransitionTest>& o)
+{
+    for (const auto& elem_it : j.items())
+    {
+        auto test = elem_it.value().get<StateTransitionTest>();
+        test.name = elem_it.key();
+        o.emplace_back(std::move(test));
+    }
+}
+
+std::vector<StateTransitionTest> load_state_tests(std::istream& input)
 {
-    return json::json::parse(input).get<StateTransitionTest>();
+    return json::json::parse(input).get<std::vector<StateTransitionTest>>();
 }
 
 void validate_state(const state::State& state, evmc_revision rev)

diff --git a/test/statetest/statetest_runner.cpp b/test/statetest/statetest_runner.cpp
@@ -11,6 +11,7 @@ namespace evmone::test
 {
 void run_state_test(const StateTransitionTest& test, evmc::VM& vm, bool trace_summary)
 {
+    SCOPED_TRACE(test.name);
     for (const auto& [rev, cases] : test.cases)
     {
         validate_state(test.pre_state, rev);

diff --git a/test/unittests/statetest_loader_test.cpp b/test/unittests/statetest_loader_test.cpp
@@ -74,7 +74,33 @@ TEST(json_loader, int64_t)
 TEST(statetest_loader, load_empty_test)
 {
     std::istringstream s{"{}"};
-    EXPECT_THROW(load_state_test(s), std::invalid_argument);
+    EXPECT_EQ(load_state_tests(s).size(), 0);
+}
+
+TEST(statetest_loader, load_multi_test)
+{
+    std::istringstream s{R"({
+      "T1": {
+        "pre": {},
+        "transaction": {"gasPrice": "","sender": "","to": "","data": null,
+          "gasLimit": "0","value": null,"nonce" : "0"},
+        "post": {},
+        "env": {"currentNumber": "0","currentTimestamp": "0",
+          "currentGasLimit": "0","currentCoinbase": ""}
+      },
+      "T2": {
+        "pre": {},
+        "transaction": {"gasPrice": "","sender": "","to": "","data": null,
+          "gasLimit": "0","value": null,"nonce" : "0"},
+        "post": {},
+        "env": {"currentNumber": "0","currentTimestamp": "0",
+          "currentGasLimit": "0","currentCoinbase": ""}
+      }
+    })"};
+    const auto tests = load_state_tests(s);
+    ASSERT_EQ(tests.size(), 2);
+    EXPECT_EQ(tests[0].name, "T1");
+    EXPECT_EQ(tests[1].name, "T2");
 }
 
 TEST(statetest_loader, load_minimal_test)
@@ -100,7 +126,7 @@ TEST(statetest_loader, load_minimal_test)
             }
         }
     })"};
-    const StateTransitionTest st = load_state_test(s);
+    const auto st = std::move(load_state_tests(s).at(0));
     // TODO: should add some comparison operator to State, BlockInfo, AccessList
     EXPECT_EQ(st.pre_state.get_accounts().size(), 0);
     EXPECT_EQ(st.block.number, 0);