statetest: Add changes required to be executed in goevmlab (#658)

- Add `--trace-summary` CLI flag to output state root. - Make `_info` JSON section optional (goevmlab is not providing this field). - Implement convention for `Host::get_block_hash()` (done in #698).
ethereum · Sep 18, 2023 · 33c66e7 · 33c66e7
2 parents c931df7 + 184b2c4
commit 33c66e7
Show file tree

Hide file tree

Showing 7 changed files with 114 additions and 21 deletions.
diff --git a/test/integration/statetest/CMakeLists.txt b/test/integration/statetest/CMakeLists.txt
@@ -8,6 +8,7 @@ set(PREFIX ${PREFIX}/statetest)
 set(TESTS1 ${CMAKE_CURRENT_SOURCE_DIR}/tests1)
 set(TESTS2 ${CMAKE_CURRENT_SOURCE_DIR}/tests2)
 set(TESTS_EOF ${CMAKE_CURRENT_SOURCE_DIR}/eof)
+set(TESTS_TX ${CMAKE_CURRENT_SOURCE_DIR}/tx)
 
 add_test(
     NAME ${PREFIX}/no_arguments
@@ -79,9 +80,23 @@ add_test(
     NAME ${PREFIX}/trace
     COMMAND evmone-statetest ${TESTS1}/SuiteA/test1.json --trace
 )
+set(EXPECTED_TRACE [[
+{"pc":0,"op":96,"gas":"0x5c878","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH1"}
+{"pc":2,"op":96,"gas":"0x5c875","gasCost":"0x3","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"PUSH1"}
+{"pc":4,"op":1,"gas":"0x5c872","gasCost":"0x3","memSize":0,"stack":["0x1","0x1"],"depth":1,"refund":0,"opName":"ADD"}
+{"pc":5,"op":96,"gas":"0x5c86f","gasCost":"0x3","memSize":0,"stack":["0x2"],"depth":1,"refund":0,"opName":"PUSH1"}
+{"pc":7,"op":85,"gas":"0x5c86c","gasCost":"0x0","memSize":0,"stack":["0x2","0x0"],"depth":1,"refund":0,"opName":"SSTORE"}
+{"pc":8,"op":0,"gas":"0x57218","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"}
+{"pass":true,"gasUsed":"0xa868","stateRoot":"0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530"}
+]])
+# Escape regex special characters.
+string(REPLACE "{" "\\{" EXPECTED_TRACE ${EXPECTED_TRACE})
+string(REPLACE "}" "\\}" EXPECTED_TRACE ${EXPECTED_TRACE})
+string(REPLACE "[" "\\[" EXPECTED_TRACE ${EXPECTED_TRACE})
+string(REPLACE "]" "\\]" EXPECTED_TRACE ${EXPECTED_TRACE})
 set_tests_properties(
     ${PREFIX}/trace PROPERTIES
-    PASS_REGULAR_EXPRESSION [=[\{"pc":4,"op":1,"gas":"0x5c872","gasCost":"0x3","memSize":0,"stack":\["0x1","0x1"\],"depth":1,"refund":0,"opName":"ADD"\}]=]
+    PASS_REGULAR_EXPRESSION ${EXPECTED_TRACE}
 )
 
 add_test(
@@ -92,3 +107,12 @@ set_tests_properties(
     ${PREFIX}/invalid_eof_in_state PROPERTIES
     PASS_REGULAR_EXPRESSION "EOF container at 0x0000000000000000000000000000000000bade0f is invalid"
 )
+
+add_test(
+    NAME ${PREFIX}/tx_invalid_nonce
+    COMMAND evmone-statetest ${TESTS_TX}/invalid_nonce.json
+)
+set_tests_properties(
+    ${PREFIX}/tx_invalid_nonce PROPERTIES
+    PASS_REGULAR_EXPRESSION "unexpected invalid transaction: nonce too high"
+)
diff --git a/test/integration/statetest/tx/invalid_nonce.json b/test/integration/statetest/tx/invalid_nonce.json
@@ -0,0 +1,42 @@
+{
+  "invalid_nonce": {
+    "env": {
+      "currentBaseFee": "0x0a",
+      "currentCoinbase": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+      "currentDifficulty": "0x020000",
+      "currentGasLimit": "0xff112233445566",
+      "currentNumber": "0x01",
+      "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000",
+      "currentTimestamp": "0x03e8"
+    },
+    "post": {
+      "Shanghai": [
+        {
+          "hash": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+          "indexes": {
+            "data": 0,
+            "gas": 0,
+            "value": 0
+          },
+          "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"
+        }
+      ]
+    },
+    "pre": {},
+    "transaction": {
+      "data": [
+        "0x"
+      ],
+      "gasLimit": [
+        "0x00"
+      ],
+      "gasPrice": "0x0a",
+      "nonce": "0x01",
+      "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+      "to": "0x00",
+      "value": [
+        "0x00"
+      ]
+    }
+  }
+}
diff --git a/test/statetest/statetest.cpp b/test/statetest/statetest.cpp
@@ -14,27 +14,28 @@ class StateTest : public testing::Test
 {
     fs::path m_json_test_file;
     evmc::VM& m_vm;
+    bool m_trace = false;
 
 public:
-    explicit StateTest(fs::path json_test_file, evmc::VM& vm) noexcept
-      : m_json_test_file{std::move(json_test_file)}, m_vm{vm}
+    explicit StateTest(fs::path json_test_file, evmc::VM& vm, bool trace) noexcept
+      : m_json_test_file{std::move(json_test_file)}, m_vm{vm}, m_trace{trace}
     {}
 
     void TestBody() final
     {
         std::ifstream f{m_json_test_file};
-        evmone::test::run_state_test(evmone::test::load_state_test(f), m_vm);
+        evmone::test::run_state_test(evmone::test::load_state_test(f), m_vm, m_trace);
     }
 };
 
-void register_test(const std::string& suite_name, const fs::path& file, evmc::VM& vm)
+void register_test(const std::string& suite_name, const fs::path& file, evmc::VM& vm, bool trace)
 {
     testing::RegisterTest(suite_name.c_str(), file.stem().string().c_str(), nullptr, nullptr,
         file.string().c_str(), 0,
-        [file, &vm]() -> testing::Test* { return new StateTest(file, vm); });
+        [file, &vm, trace]() -> testing::Test* { return new StateTest(file, vm, trace); });
 }
 
-void register_test_files(const fs::path& root, evmc::VM& vm)
+void register_test_files(const fs::path& root, evmc::VM& vm, bool trace)
 {
     if (is_directory(root))
     {
@@ -46,11 +47,11 @@ void register_test_files(const fs::path& root, evmc::VM& vm)
         std::sort(test_files.begin(), test_files.end());
 
         for (const auto& p : test_files)
-            register_test(fs::relative(p, root).parent_path().string(), p, vm);
+            register_test(fs::relative(p, root).parent_path().string(), p, vm, trace);
     }
     else  // Treat as a file.
     {
-        register_test(root.parent_path().string(), root, vm);
+        register_test(root.parent_path().string(), root, vm, trace);
     }
 }
 }  // namespace
@@ -80,18 +81,21 @@ int main(int argc, char* argv[])
             ->required()
             ->check(CLI::ExistingPath);
 
-        bool trace_flag = false;
-        app.add_flag("--trace", trace_flag, "Enable EVM tracing");
+        bool trace = false;
+        bool trace_summary = false;
+        const auto trace_opt = app.add_flag("--trace", trace, "Enable EVM tracing");
+        app.add_flag("--trace-summary", trace_summary, "Output trace summary only")
+            ->excludes(trace_opt);
 
         CLI11_PARSE(app, argc, argv);
 
         evmc::VM vm{evmc_create_evmone(), {{"O", "0"}}};
 
-        if (trace_flag)
+        if (trace)
             vm.set_option("trace", "1");
 
         for (const auto& p : paths)
-            register_test_files(p, vm);
+            register_test_files(p, vm, trace || trace_summary);
 
         return RUN_ALL_TESTS();
     }

diff --git a/test/statetest/statetest.hpp b/test/statetest/statetest.hpp
@@ -92,7 +92,10 @@ StateTransitionTest load_state_test(std::istream& input);
 /// Throws exception on any invalid EOF in state.
 void validate_deployed_code(const state::State& state, evmc_revision rev);
 
-void run_state_test(const StateTransitionTest& test, evmc::VM& vm);
+/// Execute the state @p test using the @p vm.
+///
+/// @param trace_summary  Output execution summary to the default trace stream.
+void run_state_test(const StateTransitionTest& test, evmc::VM& vm, bool trace_summary);
 
 /// Computes the hash of the RLP-encoded list of transaction logs.
 /// This method is only used in tests.

diff --git a/test/statetest/statetest_loader.cpp b/test/statetest/statetest_loader.cpp
@@ -397,10 +397,13 @@ static void from_json(const json::json& j, StateTransitionTest& o)
 
     o.block = from_json<state::BlockInfo>(j_t.at("env"));
 
-    if (const auto& info = j_t.at("_info"); info.contains("labels"))
+    if (const auto info_it = j_t.find("_info"); info_it != j_t.end())
     {
-        for (const auto& [j_id, j_label] : info.at("labels").items())
-            o.input_labels.emplace(from_json<uint64_t>(j_id), j_label);
+        if (const auto labels_it = info_it->find("labels"); labels_it != info_it->end())
+        {
+            for (const auto& [j_id, j_label] : labels_it->items())
+                o.input_labels.emplace(from_json<uint64_t>(j_id), j_label);
+        }
     }
 
     for (const auto& [rev_name, expectations] : j_t.at("post").items())

diff --git a/test/statetest/statetest_runner.cpp b/test/statetest/statetest_runner.cpp
@@ -9,7 +9,7 @@
 
 namespace evmone::test
 {
-void run_state_test(const StateTransitionTest& test, evmc::VM& vm)
+void run_state_test(const StateTransitionTest& test, evmc::VM& vm, bool trace_summary)
 {
     for (const auto& [rev, cases] : test.cases)
     {
@@ -33,12 +33,30 @@ void run_state_test(const StateTransitionTest& test, evmc::VM& vm)
             // Finalize block with reward 0.
             state::finalize(state, rev, test.block.coinbase, 0, {}, {});
 
+            const auto state_root = state::mpt_hash(state.get_accounts());
+
+            if (trace_summary)
+            {
+                std::clog << '{';
+                if (holds_alternative<state::TransactionReceipt>(res))  // if tx valid
+                {
+                    const auto& r = get<state::TransactionReceipt>(res);
+                    if (r.status == EVMC_SUCCESS)
+                        std::clog << R"("pass":true)";
+                    else
+                        std::clog << R"("pass":false,"error":")" << r.status << '"';
+                    std::clog << R"(,"gasUsed":"0x)" << std::hex << r.gas_used << R"(",)";
+                }
+                std::clog << R"("stateRoot":"0x)" << hex(state_root) << "\"}\n";
+            }
+
             if (holds_alternative<state::TransactionReceipt>(res))
                 EXPECT_EQ(logs_hash(get<state::TransactionReceipt>(res).logs), expected.logs_hash);
             else
-                EXPECT_TRUE(expected.exception);
+                EXPECT_TRUE(expected.exception)
+                    << "unexpected invalid transaction: " << get<std::error_code>(res).message();
 
-            EXPECT_EQ(state::mpt_hash(state.get_accounts()), expected.state_hash);
+            EXPECT_EQ(state_root, expected.state_hash);
         }
     }
 }

diff --git a/test/unittests/statetest_loader_test.cpp b/test/unittests/statetest_loader_test.cpp
@@ -74,7 +74,6 @@ TEST(statetest_loader, load_minimal_test)
 {
     std::istringstream s{R"({
         "test": {
-            "_info": {},
             "pre": {},
             "transaction": {
                 "gasPrice": "",