Add -minimal flag (#562)
diff --git a/examples/all_features/CMakeLists.txt b/examples/all_features/CMakeLists.txt
index 9ad62e2..0433dd1 100644
--- a/examples/all_features/CMakeLists.txt
+++ b/examples/all_features/CMakeLists.txt
@@ -31,6 +31,7 @@
namespace7.cpp
namespace8.cpp
namespace9.cpp
+ no_failures.cpp
)
# add the executable
@@ -86,12 +87,14 @@
doctest_add_test(NAME first_last ${common_args} -f=2 -l=4 -sf=*coverage*) # run a range
doctest_add_test(NAME filter_1 ${common_args} -ts=none) # should filter out all
# -order-by=name to avoid different output depending on the compiler used. See https://github.com/onqtam/doctest/issues/287
-doctest_add_test(NAME filter_2 COMMAND $<TARGET_FILE:all_features> -tse=* -nv -order-by=name) # should filter out all + print skipped
-doctest_add_test(NAME filter_3 ${common_args} -sc=from*,sc* -sce=sc2 -sf=*subcases*) # enter a specific subcase - sc1
-doctest_add_test(NAME order_1 ${common_args} -ob=suite -ns -sf=*test_cases_and_suites*)
-doctest_add_test(NAME order_2 ${common_args} -ob=name -sf=*test_cases_and_suites*)
-doctest_add_test(NAME order_3 ${common_args} -ob=rand -sfe=*) # exclude everything for no output
-doctest_add_test(NAME quiet ${common_args} -q -sf=*test_cases_and_suites*) # quiet
+doctest_add_test(NAME filter_2 COMMAND $<TARGET_FILE:all_features> -tse=* -nv -order-by=name) # should filter out all + print skipped
+doctest_add_test(NAME filter_3 ${common_args} -sc=from*,sc* -sce=sc2 -sf=*subcases*) # enter a specific subcase - sc1
+doctest_add_test(NAME order_1 ${common_args} -ob=suite -ns -sf=*test_cases_and_suites*)
+doctest_add_test(NAME order_2 ${common_args} -ob=name -sf=*test_cases_and_suites*)
+doctest_add_test(NAME order_3 ${common_args} -ob=rand -sfe=*) # exclude everything for no output
+doctest_add_test(NO_OUTPUT NAME quiet ${common_args} -q -sf=*test_cases_and_suites*) # quiet
+doctest_add_test(NAME minimal ${common_args} -m -sf=*test_cases_and_suites*) # minimal with summary
+doctest_add_test(NAME minimal_no_fail ${common_args} -m -sf=*no_failures.cpp) # minimal
################################################################################
## VARIATION OF THE BUILD WITH DOCTEST DISABLED - SHOULD STILL COMPILE
diff --git a/examples/all_features/no_failures.cpp b/examples/all_features/no_failures.cpp
new file mode 100644
index 0000000..ae74047
--- /dev/null
+++ b/examples/all_features/no_failures.cpp
@@ -0,0 +1,13 @@
+#include <doctest/doctest.h>
+
+TEST_CASE("no checks") {}
+
+TEST_CASE("simple check") {
+ CHECK(1 == 1);
+}
+
+TEST_SUITE("some suite") {
+ TEST_CASE("fails - and its allowed" * doctest::may_fail()) { FAIL(""); }
+}
+
+TEST_CASE("should fail and no output" * doctest::should_fail() * doctest::no_breaks() * doctest::no_output()) { FAIL(""); }
diff --git a/examples/all_features/test_output/filter_2.txt b/examples/all_features/test_output/filter_2.txt
index 8abc26e..e89520f 100644
--- a/examples/all_features/test_output/filter_2.txt
+++ b/examples/all_features/test_output/filter_2.txt
@@ -1,6 +1,6 @@
[doctest] run with "--help" for options
===============================================================================
-[doctest] test cases: 0 | 0 passed | 0 failed | 90 skipped
+[doctest] test cases: 0 | 0 passed | 0 failed | 94 skipped
[doctest] assertions: 0 | 0 passed | 0 failed |
[doctest] Status: SUCCESS!
Program code.
diff --git a/examples/all_features/test_output/filter_2_xml.txt b/examples/all_features/test_output/filter_2_xml.txt
index 9bec5b7..72b34e3 100644
--- a/examples/all_features/test_output/filter_2_xml.txt
+++ b/examples/all_features/test_output/filter_2_xml.txt
@@ -51,6 +51,9 @@
<TestCase name="explicit failures 2" filename="logging.cpp" line="0" skipped="true"/>
<TestCase name="expressions should be evaluated only once" filename="assertion_macros.cpp" line="0" skipped="true"/>
</TestSuite>
+ <TestSuite name="some suite">
+ <TestCase name="fails - and its allowed" filename="no_failures.cpp" line="0" may_fail="true" skipped="true"/>
+ </TestSuite>
<TestSuite name="test suite with a description">
<TestCase name="fails - and its allowed" filename="test_cases_and_suites.cpp" line="0" description="regarding failures" may_fail="true" skipped="true"/>
<TestCase name="fails 1 time as it should" filename="test_cases_and_suites.cpp" line="0" description="regarding failures" skipped="true"/>
@@ -84,6 +87,7 @@
<TestCase name="namespace 7 member vs global" filename="namespace7.cpp" line="0" skipped="true"/>
<TestCase name="namespace 8 friend vs global" filename="namespace8.cpp" line="0" skipped="true"/>
<TestCase name="namespace 9 both global" filename="namespace9.cpp" line="0" skipped="true"/>
+ <TestCase name="no checks" filename="no_failures.cpp" line="0" skipped="true"/>
<TestCase name="normal macros" filename="assertion_macros.cpp" line="0" skipped="true"/>
</TestSuite>
<TestSuite name="ts1">
@@ -97,11 +101,13 @@
<TestCase name="part of some TS" filename="test_cases_and_suites.cpp" line="0" skipped="true"/>
</TestSuite>
<TestSuite>
+ <TestCase name="should fail and no output" filename="no_failures.cpp" line="0" should_fail="true" skipped="true"/>
<TestCase name="should fail and no output" filename="test_cases_and_suites.cpp" line="0" should_fail="true" skipped="true"/>
<TestCase name="should fail because of an exception" filename="test_cases_and_suites.cpp" line="0" skipped="true"/>
<TestCase name="signed integers stuff<int>" filename="templated_test_cases.cpp" line="0" skipped="true"/>
<TestCase name="signed integers stuff<short int>" filename="templated_test_cases.cpp" line="0" skipped="true"/>
<TestCase name="signed integers stuff<signed char>" filename="templated_test_cases.cpp" line="0" skipped="true"/>
+ <TestCase name="simple check" filename="no_failures.cpp" line="0" skipped="true"/>
</TestSuite>
<TestSuite name="skipped test cases">
<TestCase name="skipped - inherited from the test suite" filename="test_cases_and_suites.cpp" line="0" skipped="true"/>
@@ -128,6 +134,6 @@
<TestCase name="will end from an unknown exception" filename="coverage_maxout.cpp" line="0" skipped="true"/>
</TestSuite>
<OverallResultsAsserts successes="0" failures="0"/>
- <OverallResultsTestCases successes="0" failures="0" skipped="90"/>
+ <OverallResultsTestCases successes="0" failures="0" skipped="94"/>
</doctest>
Program code.
diff --git a/examples/all_features/test_output/minimal.txt b/examples/all_features/test_output/minimal.txt
new file mode 100644
index 0000000..4901abb
--- /dev/null
+++ b/examples/all_features/test_output/minimal.txt
@@ -0,0 +1,100 @@
+===============================================================================
+test_cases_and_suites.cpp(0):
+TEST CASE: should fail because of an exception
+
+test_cases_and_suites.cpp(0): ERROR: test case THREW exception: 0
+
+===============================================================================
+test_cases_and_suites.cpp(0):
+TEST SUITE: scoped test suite
+TEST CASE: part of scoped
+
+test_cases_and_suites.cpp(0): FATAL ERROR:
+
+===============================================================================
+test_cases_and_suites.cpp(0):
+TEST SUITE: scoped test suite
+TEST CASE: part of scoped 2
+
+test_cases_and_suites.cpp(0): FATAL ERROR:
+
+===============================================================================
+test_cases_and_suites.cpp(0):
+TEST SUITE: some TS
+TEST CASE: part of some TS
+
+test_cases_and_suites.cpp(0): FATAL ERROR:
+
+===============================================================================
+test_cases_and_suites.cpp(0):
+TEST CASE: fixtured test - not part of a test suite
+
+test_cases_and_suites.cpp(0): ERROR: CHECK( data == 85 ) is NOT correct!
+ values: CHECK( 21 == 85 )
+
+===============================================================================
+test_cases_and_suites.cpp(0):
+TEST SUITE: ts1
+TEST CASE: normal test in a test suite from a decorator
+
+test_cases_and_suites.cpp(0): MESSAGE: failing because of the timeout decorator!
+
+Test case exceeded time limit of 0.000001!
+===============================================================================
+test_cases_and_suites.cpp(0):
+DESCRIPTION: this test has overridden its skip decorator
+TEST SUITE: skipped test cases
+TEST CASE: unskipped
+
+test_cases_and_suites.cpp(0): FATAL ERROR:
+
+===============================================================================
+test_cases_and_suites.cpp(0):
+DESCRIPTION: regarding failures
+TEST SUITE: test suite with a description
+TEST CASE: fails - and its allowed
+
+test_cases_and_suites.cpp(0): FATAL ERROR:
+
+Allowed to fail so marking it as not failed
+===============================================================================
+test_cases_and_suites.cpp(0):
+DESCRIPTION: regarding failures
+TEST SUITE: test suite with a description
+TEST CASE: fails as it should
+
+test_cases_and_suites.cpp(0): FATAL ERROR:
+
+Failed as expected so marking it as not failed
+===============================================================================
+test_cases_and_suites.cpp(0):
+DESCRIPTION: regarding failures
+TEST SUITE: test suite with a description
+TEST CASE: doesn't fail but it should have
+
+Should have failed but didn't! Marking it as failed!
+===============================================================================
+test_cases_and_suites.cpp(0):
+DESCRIPTION: regarding failures
+TEST SUITE: test suite with a description
+TEST CASE: fails 1 time as it should
+
+test_cases_and_suites.cpp(0): FATAL ERROR:
+
+Failed exactly 1 times as expected so marking it as not failed!
+===============================================================================
+test_cases_and_suites.cpp(0):
+DESCRIPTION: regarding failures
+TEST SUITE: test suite with a description
+TEST CASE: fails more times as it should
+
+test_cases_and_suites.cpp(0): ERROR:
+
+test_cases_and_suites.cpp(0): ERROR:
+
+Didn't fail exactly 1 times so marking it as failed!
+===============================================================================
+[doctest] test cases: 15 | 6 passed | 9 failed |
+[doctest] assertions: 12 | 1 passed | 11 failed |
+[doctest] Status: FAILURE!
+Program code.
diff --git a/examples/all_features/test_output/minimal_junit.txt b/examples/all_features/test_output/minimal_junit.txt
new file mode 100644
index 0000000..47e66d0
--- /dev/null
+++ b/examples/all_features/test_output/minimal_junit.txt
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+ <testsuite name="all_features" errors="1" failures="1" tests="12">
+ <testcase classname="test_cases_and_suites.cpp" name="an empty test that will succeed - not part of a test suite" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="should fail because of an exception" status="run">
+ <error message="exception">
+ 0
+ </error>
+ </testcase>
+ <testcase classname="test_cases_and_suites.cpp" name="part of scoped" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="part of scoped 2" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="part of some TS" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="fixtured test - not part of a test suite" status="run">
+ <failure message="21 == 85" type="CHECK">
+test_cases_and_suites.cpp(0):
+CHECK( data == 85 ) is NOT correct!
+ values: CHECK( 21 == 85 )
+
+ </failure>
+ </testcase>
+ <testcase classname="test_cases_and_suites.cpp" name="normal test in a test suite from a decorator" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="unskipped" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="fails - and its allowed" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="doesn't fail which is fine" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="fails as it should" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="doesn't fail but it should have" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="fails 1 time as it should" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="fails more times as it should" status="run"/>
+ <testcase classname="test_cases_and_suites.cpp" name="should fail and no output" status="run"/>
+ </testsuite>
+</testsuites>
+Program code.
diff --git a/examples/all_features/test_output/minimal_no_fail.txt b/examples/all_features/test_output/minimal_no_fail.txt
new file mode 100644
index 0000000..afcd6d9
--- /dev/null
+++ b/examples/all_features/test_output/minimal_no_fail.txt
@@ -0,0 +1,9 @@
+===============================================================================
+no_failures.cpp(0):
+TEST SUITE: some suite
+TEST CASE: fails - and its allowed
+
+no_failures.cpp(0): FATAL ERROR:
+
+Allowed to fail so marking it as not failed
+Program code.
diff --git a/examples/all_features/test_output/minimal_no_fail_junit.txt b/examples/all_features/test_output/minimal_no_fail_junit.txt
new file mode 100644
index 0000000..a71f8b5
--- /dev/null
+++ b/examples/all_features/test_output/minimal_no_fail_junit.txt
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+ <testsuite name="all_features" errors="0" failures="0" tests="3">
+ <testcase classname="no_failures.cpp" name="no checks" status="run"/>
+ <testcase classname="no_failures.cpp" name="simple check" status="run"/>
+ <testcase classname="no_failures.cpp" name="fails - and its allowed" status="run"/>
+ <testcase classname="no_failures.cpp" name="should fail and no output" status="run"/>
+ </testsuite>
+</testsuites>
+Program code.
diff --git a/examples/all_features/test_output/minimal_no_fail_xml.txt b/examples/all_features/test_output/minimal_no_fail_xml.txt
new file mode 100644
index 0000000..46601b2
--- /dev/null
+++ b/examples/all_features/test_output/minimal_no_fail_xml.txt
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<doctest binary="all_features">
+ <Options order_by="file" rand_seed="324" first="0" last="4294967295" abort_after="0" subcase_filter_levels="2147483647" case_sensitive="false" no_throw="false" no_skip="false"/>
+ <TestSuite>
+ <TestCase name="no checks" filename="no_failures.cpp" line="0">
+ <OverallResultsAsserts successes="0" failures="0"/>
+ </TestCase>
+ <TestCase name="simple check" filename="no_failures.cpp" line="0">
+ <OverallResultsAsserts successes="1" failures="0"/>
+ </TestCase>
+ </TestSuite>
+ <TestSuite name="some suite">
+ <TestCase name="fails - and its allowed" filename="no_failures.cpp" line="0" may_fail="true">
+ <Message type="FATAL ERROR" filename="no_failures.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="1"/>
+ </TestCase>
+ </TestSuite>
+ <TestSuite>
+ <TestCase name="should fail and no output" filename="no_failures.cpp" line="0" should_fail="true">
+ <Message type="FATAL ERROR" filename="no_failures.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="1"/>
+ </TestCase>
+ </TestSuite>
+ <OverallResultsAsserts successes="1" failures="2"/>
+ <OverallResultsTestCases successes="4" failures="0"/>
+</doctest>
+Program code.
diff --git a/examples/all_features/test_output/minimal_xml.txt b/examples/all_features/test_output/minimal_xml.txt
new file mode 100644
index 0000000..ece0fb6
--- /dev/null
+++ b/examples/all_features/test_output/minimal_xml.txt
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<doctest binary="all_features">
+ <Options order_by="file" rand_seed="324" first="0" last="4294967295" abort_after="0" subcase_filter_levels="2147483647" case_sensitive="false" no_throw="false" no_skip="false"/>
+ <TestSuite>
+ <TestCase name="an empty test that will succeed - not part of a test suite" filename="test_cases_and_suites.cpp" line="0">
+ <OverallResultsAsserts successes="0" failures="0"/>
+ </TestCase>
+ <TestCase name="should fail because of an exception" filename="test_cases_and_suites.cpp" line="0">
+ <Exception crash="false">
+ 0
+ </Exception>
+ <OverallResultsAsserts successes="1" failures="0"/>
+ </TestCase>
+ </TestSuite>
+ <TestSuite name="scoped test suite">
+ <TestCase name="part of scoped" filename="test_cases_and_suites.cpp" line="0">
+ <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="1"/>
+ </TestCase>
+ <TestCase name="part of scoped 2" filename="test_cases_and_suites.cpp" line="0">
+ <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="1"/>
+ </TestCase>
+ </TestSuite>
+ <TestSuite name="some TS">
+ <TestCase name="part of some TS" filename="test_cases_and_suites.cpp" line="0">
+ <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="1"/>
+ </TestCase>
+ </TestSuite>
+ <TestSuite>
+ <TestCase name="fixtured test - not part of a test suite" filename="test_cases_and_suites.cpp" line="0">
+ <Expression success="false" type="CHECK" filename="test_cases_and_suites.cpp" line="0">
+ <Original>
+ data == 85
+ </Original>
+ <Expanded>
+ 21 == 85
+ </Expanded>
+ </Expression>
+ <OverallResultsAsserts successes="0" failures="1"/>
+ </TestCase>
+ </TestSuite>
+ <TestSuite name="ts1">
+ <TestCase name="normal test in a test suite from a decorator" filename="test_cases_and_suites.cpp" line="0">
+ <Message type="WARNING" filename="test_cases_and_suites.cpp" line="0">
+ <Text>
+ failing because of the timeout decorator!
+ </Text>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="0"/>
+ </TestCase>
+ </TestSuite>
+ <TestSuite name="skipped test cases">
+ <TestCase name="unskipped" filename="test_cases_and_suites.cpp" line="0" description="this test has overridden its skip decorator">
+ <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="1"/>
+ </TestCase>
+ </TestSuite>
+ <TestSuite name="test suite with a description">
+ <TestCase name="fails - and its allowed" filename="test_cases_and_suites.cpp" line="0" description="regarding failures" may_fail="true">
+ <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="1"/>
+ </TestCase>
+ <TestCase name="doesn't fail which is fine" filename="test_cases_and_suites.cpp" line="0" description="regarding failures" may_fail="true">
+ <OverallResultsAsserts successes="0" failures="0"/>
+ </TestCase>
+ <TestCase name="fails as it should" filename="test_cases_and_suites.cpp" line="0" description="regarding failures" should_fail="true">
+ <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="1"/>
+ </TestCase>
+ <TestCase name="doesn't fail but it should have" filename="test_cases_and_suites.cpp" line="0" description="regarding failures" should_fail="true">
+ <OverallResultsAsserts successes="0" failures="0"/>
+ </TestCase>
+ <TestCase name="fails 1 time as it should" filename="test_cases_and_suites.cpp" line="0" description="regarding failures">
+ <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="1" expected_failures="1"/>
+ </TestCase>
+ <TestCase name="fails more times as it should" filename="test_cases_and_suites.cpp" line="0" description="regarding failures">
+ <Message type="ERROR" filename="test_cases_and_suites.cpp" line="0">
+ <Text/>
+ </Message>
+ <Message type="ERROR" filename="test_cases_and_suites.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="2" expected_failures="1"/>
+ </TestCase>
+ </TestSuite>
+ <TestSuite>
+ <TestCase name="should fail and no output" filename="test_cases_and_suites.cpp" line="0" should_fail="true">
+ <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0">
+ <Text/>
+ </Message>
+ <OverallResultsAsserts successes="0" failures="1"/>
+ </TestCase>
+ </TestSuite>
+ <OverallResultsAsserts successes="1" failures="11"/>
+ <OverallResultsTestCases successes="6" failures="9"/>
+</doctest>
+Program code.
diff --git a/examples/all_features/test_output/quiet.txt b/examples/all_features/test_output/quiet.txt
deleted file mode 100644
index fd6c04e..0000000
--- a/examples/all_features/test_output/quiet.txt
+++ /dev/null
@@ -1 +0,0 @@
-Program code.
diff --git a/examples/all_features/test_output/quiet_junit.txt b/examples/all_features/test_output/quiet_junit.txt
deleted file mode 100644
index fd6c04e..0000000
--- a/examples/all_features/test_output/quiet_junit.txt
+++ /dev/null
@@ -1 +0,0 @@
-Program code.
diff --git a/examples/all_features/test_output/quiet_xml.txt b/examples/all_features/test_output/quiet_xml.txt
deleted file mode 100644
index fd6c04e..0000000
--- a/examples/all_features/test_output/quiet_xml.txt
+++ /dev/null
@@ -1 +0,0 @@
-Program code.