From 33e493500e795f666933a5a56555b8c000502e48 Mon Sep 17 00:00:00 2001 From: zexoverz Date: Tue, 15 Sep 2026 14:14:20 +0700 Subject: [PATCH] test: Count deselected files apart from skipped ones A file whose fixtures were all filtered out by -k is the one case where run_fixture_file() returns no result. It was counted as skipped, which is the bucket for a file the tool could not run. Now it is reported as deselected, as pytest does, and neither listed nor allowed to count as verified. --- .../evmone-cli/test/CMakeLists.txt | 19 ++++++++++++++++++ test/unittests/test_driver_test.cpp | 20 +++++++++++++++++-- test/utils/test_driver.cpp | 11 +++++++++- test/utils/test_driver.hpp | 1 + 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/test/integration/evmone-cli/test/CMakeLists.txt b/test/integration/evmone-cli/test/CMakeLists.txt index 8b3bcc4b40..760411a50a 100644 --- a/test/integration/evmone-cli/test/CMakeLists.txt +++ b/test/integration/evmone-cli/test/CMakeLists.txt @@ -55,6 +55,25 @@ set_tests_properties( ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION "all_unsupported\\.json::a_engine - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*::b_engine_x - .*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in") +# A filter which selects nothing deselects every file: counted apart from skipped, not named, +# and still nothing verified. +add_test(NAME ${PREFIX}/test_dir_deselected COMMAND evmone-cli test + ${DATA}/testcmd -k no_such_name) +set_tests_properties( + ${PREFIX}/test_dir_deselected PROPERTIES + PASS_REGULAR_EXPRESSION "collected 1 file\n\nd[^\n]*\n\n=+ 0 passed, 1 deselected in" + FAIL_REGULAR_EXPRESSION "skipped|SKIPPED") +add_test(NAME ${PREFIX}/test_dir_deselected_exit_code COMMAND evmone-cli test + ${DATA}/testcmd -k no_such_name) +set_tests_properties(${PREFIX}/test_dir_deselected_exit_code PROPERTIES WILL_FAIL TRUE) + +# A file the filter empties is deselected; one which declined its fixtures is still skipped. +add_test(NAME ${PREFIX}/test_dir_deselected_beside_skipped COMMAND evmone-cli test + ${DATA}/testcmd_skipped -k no_such_name) +set_tests_properties( + ${PREFIX}/test_dir_deselected_beside_skipped PROPERTIES + PASS_REGULAR_EXPRESSION "0 passed, 4 skipped, 1 deselected in") + # The --histogram output reaches the report, and the progress row keeps out of its way: # a run without it ends every line with the percentage done. add_test(NAME ${PREFIX}/test_histogram COMMAND evmone-cli test --histogram diff --git a/test/unittests/test_driver_test.cpp b/test/unittests/test_driver_test.cpp index 8c013927e9..07d1a543ac 100644 --- a/test/unittests/test_driver_test.cpp +++ b/test/unittests/test_driver_test.cpp @@ -186,10 +186,26 @@ TEST(test_driver, a_file_is_skipped_only_when_nothing_in_it_ran) TEST(test_driver, a_file_whose_fixtures_were_all_filtered_out_verifies_nothing) { - // A filter selecting nothing leaves no result at all, which is not a pass. + // A filter selecting nothing leaves no result at all: not a pass, and not a skip either. const std::vector cases{holding("a file", {})}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, NOTHING_VERIFIED); - EXPECT_NE(output.find("0 passed, 1 skipped"), std::string::npos); + EXPECT_NE(output.find("0 passed, 1 deselected in"), std::string::npos); + EXPECT_EQ(output.find("skipped"), std::string::npos); + // Deselected files are not listed. + EXPECT_EQ(output.find("short test summary info"), std::string::npos); +} + +TEST(test_driver, deselected_is_counted_apart_from_skipped) +{ + const std::vector cases{ + holding("ran", {Outcome::passed}), + holding("declined", {Outcome::skipped}), + holding("filtered out", {}), + }; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, SUCCESS); + EXPECT_NE(output.find("1 passed, 1 skipped, 1 deselected in"), std::string::npos); } diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 713986448c..f88678a75f 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -192,6 +192,7 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio Progress row{out, cases.size()}; size_t failed = 0; size_t skipped = 0; + size_t deselected = 0; size_t passed = 0; for (const auto& test : cases) @@ -204,11 +205,14 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio // The file counts once, for the worst its fixtures reached. It is skipped only when // nothing in it ran at all, so one fixture running is enough to give it a verdict. + // No result at all means the filter emptied the file: deselected, not skipped. static constexpr auto is = [](Outcome outcome) { return [outcome](const Result& result) { return result.outcome == outcome; }; }; auto outcome = Outcome::passed; - if (std::ranges::any_of(results, is(Outcome::failed))) + if (results.empty()) + outcome = Outcome::deselected; + else if (std::ranges::any_of(results, is(Outcome::failed))) outcome = Outcome::failed; else if (std::ranges::none_of(results, is(Outcome::passed))) outcome = Outcome::skipped; @@ -224,6 +228,9 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio case Outcome::skipped: ++skipped; break; + case Outcome::deselected: + ++deselected; + break; } // Every fixture which did not pass is named, including one declined by a file which @@ -276,6 +283,8 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio summary << passed << " passed"; if (skipped != 0) summary << ", " << skipped << " skipped"; + if (deselected != 0) + summary << ", " << deselected << " deselected"; summary << " in " << std::fixed << std::setprecision(2) << elapsed.count() << "s"; out << '\n'; banner(out, std::move(summary).str()); diff --git a/test/utils/test_driver.hpp b/test/utils/test_driver.hpp index 908aff786b..133421d6df 100644 --- a/test/utils/test_driver.hpp +++ b/test/utils/test_driver.hpp @@ -28,6 +28,7 @@ enum class Outcome : char passed = '.', failed = 'F', skipped = 's', + deselected = 'd', ///< A file whose fixtures were all filtered out. }; /// What running one fixture produced.