statetest: Load tests from multiple dirs/files

circle.yml

@@ -349,11 +349,7 @@ jobs:
       - run:
           name: "State tests"
           working_directory: ~/build
-          command: bin/evmone-statetest ~/tests/GeneralStateTests
-      - run:
-          name: "State tests (legacy)"
-          working_directory: ~/build
-          command: bin/evmone-statetest ~/tests/LegacyTests/Constantinople/GeneralStateTests
+          command: bin/evmone-statetest ~/tests/GeneralStateTests ~/tests/LegacyTests/Constantinople/GeneralStateTests
       - collect_coverage_gcc
       - upload_coverage:
           flags: statetests

test/integration/statetest/CMakeLists.txt

@@ -6,14 +6,15 @@
 
 set(PREFIX ${PREFIX}/statetest)
 set(TESTS1 ${CMAKE_CURRENT_SOURCE_DIR}/tests1)
+set(TESTS2 ${CMAKE_CURRENT_SOURCE_DIR}/tests2)
 
 add_test(
-    NAME ${PREFIX}/missing_arg
+    NAME ${PREFIX}/no_tests
     COMMAND evmone-statetest
 )
 set_tests_properties(
-    ${PREFIX}/missing_arg PROPERTIES
-    PASS_REGULAR_EXPRESSION "Missing argument with the path to the tests directory"
+    ${PREFIX}/no_tests PROPERTIES
+    PASS_REGULAR_EXPRESSION "Running 0 tests from 0 test suites"
 )
 
 add_test(
@@ -30,3 +31,45 @@ SuiteA\.
   test2
 ]]
 )
+
+add_test(
+    NAME ${PREFIX}/single_file_list
+    COMMAND evmone-statetest ${TESTS1}/SuiteA/test1.json --gtest_list_tests
+)
+set_tests_properties(
+    ${PREFIX}/single_file_list PROPERTIES
+    PASS_REGULAR_EXPRESSION [[
+.*test/integration/statetest/tests1/SuiteA\.
+  test1
+]]
+)
+
+add_test(
+    NAME ${PREFIX}/multiple_args_list
+    COMMAND evmone-statetest ${TESTS1} ${TESTS2} ${TESTS1}/B/T.json ${TESTS1}/SuiteA --gtest_list_tests
+)
+set_tests_properties(
+    ${PREFIX}/multiple_args_list PROPERTIES
+    PASS_REGULAR_EXPRESSION [[
+B\.
+  T
+SuiteA\.
+  test1
+  test2
+  test1
+.*test/integration/statetest/tests1/B\.
+  T
+\.
+  test1
+  test2
+]]
+)
+
+add_test(
+    NAME ${PREFIX}/invalid_path
+    COMMAND evmone-statetest invalid.json
+)
+set_tests_properties(
+    ${PREFIX}/invalid_path PROPERTIES
+    PASS_REGULAR_EXPRESSION "invalid path: invalid\\.json"
+)

test/statetest/statetest.cpp

@@ -20,33 +20,48 @@ class StateTest : public testing::Test
     void TestBody() final { evmone::test::load_state_test(m_json_test_file); }
 };
 
-void register_test_files(const fs::path& root_dir)
+void register_test(const std::string& suite_name, const fs::path& file)
 {
-    std::vector<fs::path> test_files;
-    std::copy_if(fs::recursive_directory_iterator{root_dir}, fs::recursive_directory_iterator{},
-        std::back_inserter(test_files), [](const fs::directory_entry& entry) {
-            return entry.is_regular_file() && entry.path().extension() == ".json";
-        });
-    std::sort(test_files.begin(), test_files.end());
-    for (const auto& p : test_files)
+    testing::RegisterTest(suite_name.c_str(), file.stem().string().c_str(), nullptr, nullptr,
+        file.string().c_str(), 0, [file]() -> testing::Test* { return new StateTest(file); });
+}
+
+void register_test_files(const fs::path& root)
+{
+    if (is_directory(root))
+    {
+        std::vector<fs::path> test_files;
+        std::copy_if(fs::recursive_directory_iterator{root}, fs::recursive_directory_iterator{},
+            std::back_inserter(test_files), [](const fs::directory_entry& entry) {
+                return entry.is_regular_file() && entry.path().extension() == ".json";
+            });
+        std::sort(test_files.begin(), test_files.end());
+
+        for (const auto& p : test_files)
+            register_test(fs::relative(p, root).parent_path().string(), p);
+    }
+    else if (is_regular_file(root))
     {
-        const auto d = fs::relative(p, root_dir);
-        testing::RegisterTest(d.parent_path().string().c_str(), d.stem().string().c_str(), nullptr,
-            nullptr, p.string().c_str(), 0, [p]() -> testing::Test* { return new StateTest(p); });
+        register_test(root.parent_path().string(), root);
     }
+    else
+        throw std::invalid_argument("invalid path: " + root.string());
 }
 }  // namespace
 
+
 int main(int argc, char* argv[])
 {
-    testing::InitGoogleTest(&argc, argv);  // Process GoogleTest flags.
-
-    if (argc != 2)
+    try
     {
-        std::cerr << "Missing argument with the path to the tests directory\n";
+        testing::InitGoogleTest(&argc, argv);  // Process GoogleTest flags.
+        for (int i = 1; i < argc; ++i)
+            register_test_files(argv[i]);
+        return RUN_ALL_TESTS();
+    }
+    catch (const std::exception& ex)
+    {
+        std::cerr << ex.what() << "\n";
         return -1;
     }
-
-    register_test_files(argv[1]);
-    return RUN_ALL_TESTS();
 }