test: Count deselected files apart from skipped ones

test/integration/evmone-cli/test/CMakeLists.txt

@@ -55,6 +55,25 @@ set_tests_properties(
     ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION
     "all_unsupported\\.json::a_engine - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*::b_engine_x - .*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in")
 
+# A filter which selects nothing deselects every file: counted apart from skipped, not named,
+# and still nothing verified.
+add_test(NAME ${PREFIX}/test_dir_deselected COMMAND evmone-cli test
+    ${DATA}/testcmd -k no_such_name)
+set_tests_properties(
+    ${PREFIX}/test_dir_deselected PROPERTIES
+    PASS_REGULAR_EXPRESSION "collected 1 file\n\nd[^\n]*\n\n=+ 0 passed, 1 deselected in"
+    FAIL_REGULAR_EXPRESSION "skipped|SKIPPED")
+add_test(NAME ${PREFIX}/test_dir_deselected_exit_code COMMAND evmone-cli test
+    ${DATA}/testcmd -k no_such_name)
+set_tests_properties(${PREFIX}/test_dir_deselected_exit_code PROPERTIES WILL_FAIL TRUE)
+
+# A file the filter empties is deselected; one which declined its fixtures is still skipped.
+add_test(NAME ${PREFIX}/test_dir_deselected_beside_skipped COMMAND evmone-cli test
+    ${DATA}/testcmd_skipped -k no_such_name)
+set_tests_properties(
+    ${PREFIX}/test_dir_deselected_beside_skipped PROPERTIES
+    PASS_REGULAR_EXPRESSION "0 passed, 4 skipped, 1 deselected in")
+
 # The --histogram output reaches the report, and the progress row keeps out of its way:
 # a run without it ends every line with the percentage done.
 add_test(NAME ${PREFIX}/test_histogram COMMAND evmone-cli test --histogram

test/unittests/test_driver_test.cpp

@@ -186,10 +186,26 @@ TEST(test_driver, a_file_is_skipped_only_when_nothing_in_it_ran)
 
 TEST(test_driver, a_file_whose_fixtures_were_all_filtered_out_verifies_nothing)
 {
-    // A filter selecting nothing leaves no result at all, which is not a pass.
+    // A filter selecting nothing leaves no result at all: not a pass, and not a skip either.
     const std::vector<TestCase> cases{holding("a file", {})};
 
     const auto [exit_code, output] = run(cases);
     EXPECT_EQ(exit_code, NOTHING_VERIFIED);
-    EXPECT_NE(output.find("0 passed, 1 skipped"), std::string::npos);
+    EXPECT_NE(output.find("0 passed, 1 deselected in"), std::string::npos);
+    EXPECT_EQ(output.find("skipped"), std::string::npos);
+    // Deselected files are not listed.
+    EXPECT_EQ(output.find("short test summary info"), std::string::npos);
+}
+
+TEST(test_driver, deselected_is_counted_apart_from_skipped)
+{
+    const std::vector<TestCase> cases{
+        holding("ran", {Outcome::passed}),
+        holding("declined", {Outcome::skipped}),
+        holding("filtered out", {}),
+    };
+
+    const auto [exit_code, output] = run(cases);
+    EXPECT_EQ(exit_code, SUCCESS);
+    EXPECT_NE(output.find("1 passed, 1 skipped, 1 deselected in"), std::string::npos);
 }

test/utils/test_driver.cpp

@@ -192,6 +192,7 @@ int run_tests(std::span<const TestCase> cases, std::ostream& out, const RunOptio
     Progress row{out, cases.size()};
     size_t failed = 0;
     size_t skipped = 0;
+    size_t deselected = 0;
     size_t passed = 0;
 
     for (const auto& test : cases)
@@ -204,11 +205,14 @@ int run_tests(std::span<const TestCase> cases, std::ostream& out, const RunOptio
 
         // The file counts once, for the worst its fixtures reached. It is skipped only when
         // nothing in it ran at all, so one fixture running is enough to give it a verdict.
+        // No result at all means the filter emptied the file: deselected, not skipped.
         static constexpr auto is = [](Outcome outcome) {
             return [outcome](const Result& result) { return result.outcome == outcome; };
         };
         auto outcome = Outcome::passed;
-        if (std::ranges::any_of(results, is(Outcome::failed)))
+        if (results.empty())
+            outcome = Outcome::deselected;
+        else if (std::ranges::any_of(results, is(Outcome::failed)))
             outcome = Outcome::failed;
         else if (std::ranges::none_of(results, is(Outcome::passed)))
             outcome = Outcome::skipped;
@@ -224,6 +228,9 @@ int run_tests(std::span<const TestCase> cases, std::ostream& out, const RunOptio
         case Outcome::skipped:
             ++skipped;
             break;
+        case Outcome::deselected:
+            ++deselected;
+            break;
         }
 
         // Every fixture which did not pass is named, including one declined by a file which
@@ -276,6 +283,8 @@ int run_tests(std::span<const TestCase> cases, std::ostream& out, const RunOptio
     summary << passed << " passed";
     if (skipped != 0)
         summary << ", " << skipped << " skipped";
+    if (deselected != 0)
+        summary << ", " << deselected << " deselected";
     summary << " in " << std::fixed << std::setprecision(2) << elapsed.count() << "s";
     out << '\n';
     banner(out, std::move(summary).str());

test/utils/test_driver.hpp

@@ -28,6 +28,7 @@ enum class Outcome : char
     passed = '.',
     failed = 'F',
     skipped = 's',
+    deselected = 'd',  ///< A file whose fixtures were all filtered out.
 };
 
 /// What running one fixture produced.