diff --git a/sbf-cpp/test.cpp b/sbf-cpp/test.cpp index 1a33aed..ab6f239 100644 --- a/sbf-cpp/test.cpp +++ b/sbf-cpp/test.cpp @@ -24,10 +24,6 @@ using std::for_each; // using namespace Test; namespace Test { - const string fn1(const string& s, int l) { - return s.substr(0, l); - } - // Test lifecycle // suiteSetupFn(); - This is called to allocate any suite level resources. This is called once when the suite begins. // These functions may be called in parallel but execution will not proceed past this block until they have all finished. @@ -85,135 +81,6 @@ namespace Test { // } // Then some test harness either generated or explicit can call test_main_ThingDoer(...) and optionally reported there. Reporting granularity is controlled by how frequently you call collect_and_report_TestResults(...). // You can combine test results with results = results + testFn(..); and then collect_and_report_TestResults on the aggregate TestResults value. - - template - TestResults execute_suite( - string suite_label, - function function_to_test, - vector> tests, - MaybeTestCompareFunction maybe_suite_compare_function, - MaybeTestConfigureFunction maybe_suite_before_each_function, - MaybeTestConfigureFunction maybe_suite_after_each_function, - bool is_enabled) { - TestResults results; - cout << "🚀 Beginning Suite: " << suite_label << endl; - - // Step 1: Suite Setup - - if (maybe_suite_before_each_function.has_value()) { - (*maybe_suite_before_each_function)(); - } - - // Step 2: Execute Tests - for_each(tests.begin(), tests.end(), [&suite_label, &function_to_test, &results, &maybe_suite_compare_function]( - TestTuple test_data - ) { - // Step 2a: Extract our variables from the TestTuple. - const std::string& test_name = get<0>(test_data); - const std::string qualified_test_name = suite_label + "::" + test_name; - const TResult& expected_output = get<1>(test_data); - std::tuple input_params = get<2>(test_data); - MaybeTestCompareFunction maybe_compare_function = get<3>(test_data); - TestCompareFunction compare_function = maybe_compare_function.has_value() - ? *maybe_compare_function - : maybe_suite_compare_function.has_value() - ? *maybe_suite_compare_function - : [](const TResult& l, const TResult& r){return l==r;}; - MaybeTestConfigureFunction before_each = get<4>(test_data); - MaybeTestConfigureFunction after_each = get<5>(test_data); - - // Step 2b: Test Setup - cout << " Beginning Test: " << qualified_test_name << endl; - if(before_each.has_value()) { - (*before_each)(); - } - - TResult actual; - try { - // Step 2c: Execute the test method. - actual = std::apply(function_to_test, input_params); - } catch(const std::exception& ex) { - cout << " ERROR: Caught exception \"" << ex.what() << "\"" << endl; - } catch(const std::string& message) { - cout << " ERROR: Caught string \"" << message << "\"" << endl; - } catch(...) { - cout << " ERROR: Caught something that is neither an std::exception nor a std::string." << endl; - } - - // Step 2d: Pass or fail. - TestResults result; - if (compare_function(expected_output, actual)) { - result = TestResults().pass(); - cout << " PASSED" << endl; - } else { - result = TestResults().fail(); - cout << " FAILED: expected: " << expected_output << ", actual: " << actual << endl; - } - results += result; - - // Step 2e: Test Teardown - if (after_each.has_value()) { - (*after_each)(); - } - cout << " Ending Test: " << test_name << endl; - }); - - // Step 3: Suite Teardown - if (maybe_suite_after_each_function.has_value()) { - (*maybe_suite_after_each_function)(); - } - cout << "Ending Suite: " << suite_label << endl; - return results; - } - - TestResults do_the_other_thing(){ - auto p1 = "Microsoft QBasic"; - auto p2 = 5; - // auto exp = "Micro"; - string s = fn1("Microsoft QBasic", 5); - TestResults tr; - - // tr = tr + execute_suite( - // "Test 8 Function", - // (function)fn1, - // vector>({ - // // vector, MaybeTestCompareFunction>>({ - // make_tuple( - // string("should do something"), // test_name - // string("Micro"), // expectedOutput - // make_tuple((string)p1, p2),// inputParams, - // std::nullopt, // compare_function - // std::nullopt, // before_each - // std::nullopt, // after_each - // true - // ), - // make_test( - // "should do something else", - // "Micro", - // make_tuple((string)p1, p2) - // ) - // })); - - auto test_data8 = vector>({ - make_test( - "Test 8 equals", "Micro", make_tuple((string)p1, p2), - [](const string& l, const string& r){ return l==r;}), - make_test( - "Test 8 not equals", "Micro", make_tuple((string)p1, p2), - [](const string& l, const string& r){ return l!=r;} - ), - make_test("Test 8 default compare", "Micro", make_tuple((string)p1, p2)), - make_test("Test 8 default compare", "Micro", make_tuple((string)p1, p2)), - make_test("Test 8 default compare", "Micro", make_tuple((string)p1, p2)) - }); - tr = tr + execute_suite( - "Test 8 Function with extra data", - (function)fn1, - test_data8 - ); - - return tr; - } // _Step_9 - if T2 is a single value then make_tuple(T2) and call longer version // auto testFunction = [](int id){return id==0?"":"";}; @@ -226,29 +93,60 @@ namespace Test { // Also allow make_tuple(T2) if the last param is not a tuple. TestResults::TestResults() - : failed_(0) + : errors_(0) + , failed_(0) , passed_(0) , skipped_(0) , total_(0) {} TestResults::TestResults(const TestResults& other) - : failed_(other.failed_) + : error_messages_(other.error_messages_) + , errors_(other.errors_) + , failed_(other.failed_) + , failure_messages_(other.failure_messages_) , passed_(other.passed_) + , skip_messages_(other.skip_messages_) , skipped_(other.skipped_) , total_(other.total_) {} - TestResults::TestResults(uint32_t failed, uint32_t passed, uint32_t skipped, uint32_t total) - : failed_(failed) + TestResults::TestResults(uint32_t errors, uint32_t failed, uint32_t passed, uint32_t skipped, uint32_t total, vector error_messages, vector failure_messages, vector skip_messages) + : error_messages_(error_messages) + , errors_(errors) + , failed_(failed) + , failure_messages_(failure_messages) , passed_(passed) + , skip_messages_(skip_messages) , skipped_(skipped) , total_(total) {} + TestResults& TestResults::error() { + errors_++; + return *this; + } + + TestResults& TestResults::error(string message) { + errors_++; + error_messages_.push_back(message); + return *this; + } + TestResults& TestResults::fail() { total_++; failed_++; return *this; } + TestResults& TestResults::fail(const string& message) { + total_++; + failed_++; + failure_messages_.push_back(message); + return *this; + } + + vector TestResults::failure_messages() { + return failure_messages_; + } + TestResults& TestResults::pass() { total_++; passed_++; @@ -261,6 +159,25 @@ namespace Test { return *this; } + TestResults& TestResults::skip(const string& message) { + total_++; + skipped_++; + skip_messages_.push_back(message); + return *this; + } + + vector TestResults::skip_messages() { + return skip_messages_; + } + + vector TestResults::error_messages() { + return error_messages_; + } + + uint32_t TestResults::errors() { + return errors_; + } + uint32_t TestResults::failed() { return failed_; } @@ -278,16 +195,34 @@ namespace Test { } TestResults TestResults::operator+(const TestResults& other) const { + vector error_messages; + error_messages.insert(error_messages.end(), error_messages_.begin(), error_messages_.end()); + error_messages.insert(error_messages.end(), other.error_messages_.begin(), other.error_messages_.end()); + vector failure_messages; + failure_messages.insert(failure_messages.end(), failure_messages_.begin(), failure_messages_.end()); + failure_messages.insert(failure_messages.end(), other.failure_messages_.begin(), other.failure_messages_.end()); + vector skip_messages; + skip_messages.insert(skip_messages.end(), skip_messages_.begin(), skip_messages_.end()); + skip_messages.insert(skip_messages.end(), other.skip_messages_.begin(), other.skip_messages_.end()); + return TestResults( + errors_ + other.errors_, failed_ + other.failed_, passed_ + other.passed_, skipped_ + other.skipped_, - total_ + other.total_); + total_ + other.total_, + error_messages, + failure_messages, + skip_messages); } TestResults& TestResults::operator+=(const TestResults& other) { + error_messages_.insert(error_messages_.end(), other.error_messages_.begin(), other.error_messages_.end()); + errors_ += other.errors_; failed_ += other.failed_; + failure_messages_.insert(failure_messages_.end(), other.failure_messages_.begin(), other.failure_messages_.end()); passed_ += other.passed_; + skip_messages_.insert(skip_messages_.end(), other.skip_messages_.begin(), other.skip_messages_.end()); skipped_ += other.skipped_; total_ += other.total_; return *this; diff --git a/sbf-cpp/test.h b/sbf-cpp/test.h index dfcee32..1af22b9 100644 --- a/sbf-cpp/test.h +++ b/sbf-cpp/test.h @@ -1,9 +1,11 @@ -#ifndef TEST_H__ +#ifndef TEST_H__ #define TEST_H__ #include #include #include #include +#include +#include // Test lifecycle // suite_setup_function(); - This is called to allocate any suite level resources. This is called once when the suite begins. @@ -19,6 +21,26 @@ // Collect reports - Ths step is not visible to the user at this point, but data returned by all of the test functions is collected here. This is where you will eventually be able to format/log data for reports. // suite_teardown_function(); - This is called after all test calls have completed, all test_teardown_function calls have completed, and all test reports/logs have been written. You should free any resources allocated in suite_setup_function. +// Tuple printer from: https://stackoverflow.com/questions/6245735/pretty-print-stdtuple/31116392#58417285 +template +auto& operator<<(std::basic_ostream& os, std::tuple const& t) { + std::apply([&os](auto&&... args) {((os << args << " "), ...);}, t); + return os; +} + +template +auto& operator<<(std::basic_ostream& os, std::vector v) { + os << "[ "; + for (auto it = v.begin(); it != v.end(); it++) { + if (it != v.begin()) { + os << ", "; + } + os << *it; + } + os << " ]"; + return os; +} + namespace Test { using std::tuple; using std::pair; @@ -36,15 +58,33 @@ namespace Test { TestResults(const TestResults& other); /// @brief Creates a new TestResults instance with specific counts. + /// @param errors The number of errors while running the tests. /// @param failed The number of failed tests. /// @param passed The number of passed tests. /// @param skipped The number of skipped tests. /// @param total The total number of tests run. This should equal the sum of failed, passed, and skipped tests. - TestResults(uint32_t failed, uint32_t passed, uint32_t skipped, uint32_t total); + /// @param error_messages The list of error messages. + /// @param failure_messages The list of failure messages. + /// @param skip_messages The list of skip messages. + TestResults(uint32_t errors, uint32_t failed, uint32_t passed, uint32_t skipped, uint32_t total, std::vector error_messages, std::vector failure_messages, std::vector skip_messages); + /// @brief Adds an error. This increments errors. + /// @return A reference to this instance. Used for chaining. + TestResults& error(); + + /// @brief Adds an error with a message. This increments errors as well as saving the error message. + /// @param message The error message. + /// @return A reference to this instance. Used for chaining. + TestResults& error(std::string message); + /// @brief Adds a failed test. This increments total and failed. /// @return A reference to this instance. Used for chaining. TestResults& fail(); + + /// @brief Adds a failed test with a message. This increments total and failed as well as saving the failure message. + /// @param message The reason the test failed. + /// @return A reference to this instance. Used for chaining. + TestResults& fail(const std::string& message); /// @brief Adds a passed test. This increments total and passed. /// @return A reference to this instance. Used for chaining. @@ -54,9 +94,26 @@ namespace Test { /// @return A reference to this instance. Used for chaining. TestResults& skip(); + /// @brief Adds a skipped test with a message. This increments total and skipped as well as saving the skip message. + /// @param message The reason the test was skipped. + /// @return A reference to this instance. Used for chaining. + TestResults& skip(const std::string& message); + + /// @brief Getter for the list of error messages. + /// @return + vector error_messages(); + + /// @brief Getter for the count of errors. + /// @return + uint32_t errors(); + /// @brief Getter for the count of failed tests. /// @return The count of failed tests. uint32_t failed(); + + /// @brief Getter for the list of failure messages. + /// @return The list of failure messages. + vector failure_messages(); /// @brief Getter for the count of passed tests. /// @return The count of passed tests. @@ -65,6 +122,10 @@ namespace Test { /// @brief Getter for the count of skipped tests. /// @return The count of skipped tests. uint32_t skipped(); + + /// @brief Getter for the list of skip messages. + /// @return The list of skip messages. + vector skip_messages(); /// @brief Getter for the count of total tests. /// @return The count of total tests run. @@ -81,8 +142,12 @@ namespace Test { TestResults& operator+=(const TestResults& other); private: + std::vector error_messages_; + uint32_t errors_; uint32_t failed_; + std::vector failure_messages_; uint32_t passed_; + std::vector skip_messages_; uint32_t skipped_; uint32_t total_; }; @@ -100,13 +165,14 @@ namespace Test { using TestConfigureFunction = std::function; using MaybeTestConfigureFunction = std::optional; + // TODO: For some reason all hell breaks loose if test_name or expected output are const&. Figure out why. /// @brief /// @tparam TResult /// @tparam ...TInputParams template using TestTuple = std::tuple< - const std::string& /* test_name */, - const TResult& /* expected_output */, + std::string /* test_name */, + TResult /* expected_output */, std::tuple /* input_params - The input parameters for this test. These will be used when calling std::apply with function_to_test to execute the test. */, MaybeTestCompareFunction /* test_compare_function - If this is not nullprt then this function will be called instead of suite_compare_function to determine if the test passes. Use this to check for side effects of the test. Return true if the test passes and false otherwise. */, MaybeTestConfigureFunction /* test_setup_function - If this is not nullptr this function is called before each test to setup the environment. It is called with std::apply and input_params so you can use them to mock records with specific IDs or calculate an expected result. */, @@ -184,7 +250,93 @@ namespace Test { MaybeTestConfigureFunction before_all = std::nullopt, MaybeTestConfigureFunction after_all = std::nullopt, bool is_enabled = true - ); + ) { + TestResults results; + std::cout << "🚀Beginning Suite: " << suite_label << std::endl; + + // Step 1: Suite Setup + + if (before_all.has_value()) { + (*before_all)(); + } + + // Step 2: Execute Tests + for_each(tests.begin(), tests.end(), [&suite_label, &function_to_test, &results, &suite_compare]( + TestTuple test_data + ) { + + // Step 2a: Extract our variables from the TestTuple. + const std::string& test_name = std::get<0>(test_data); + const std::string qualified_test_name = suite_label + "::" + test_name; + const TResult& expected_output = std::get<1>(test_data); + std::tuple input_params = std::get<2>(test_data); + MaybeTestCompareFunction maybe_compare_function = std::get<3>(test_data); + TestCompareFunction compare_function = maybe_compare_function.has_value() + ? *maybe_compare_function + : suite_compare.has_value() + ? *suite_compare + : [](const TResult& l, const TResult& r){return l==r;}; + MaybeTestConfigureFunction before_each = std::get<4>(test_data); + MaybeTestConfigureFunction after_each = std::get<5>(test_data); + bool is_enabled = std::get<6>(test_data); + + if (!is_enabled) { + std::cout << " 🚧Skipping Test: " << test_name << std::endl; + results.skip("🚧Skipping Test: " + qualified_test_name); + return; + } + + // Step 2b: Test Setup + std::cout << " Beginning Test: " << test_name << std::endl; + if(before_each.has_value()) { + (*before_each)(); + } + + TResult actual; + try { + // Step 2c: Execute the test method. + actual = std::apply(function_to_test, input_params); + } catch(const std::exception& ex) { + std::ostringstream os; + os << "Caught exception \"" << ex.what() << "\""; + results.error("🔥ERROR: " + qualified_test_name + " " + os.str()); + std::cout << " 🔥ERROR: " << os.str() << std::endl; + } catch(const std::string& message) { + std::ostringstream os; + os << "Caught string \"" << message << "\""; + results.error("🔥ERROR: " + qualified_test_name + " " + os.str()); + std::cout << " 🔥ERROR: " << os.str() << std::endl; + } catch(...) { + string message = "Caught something that is neither an std::exception nor a std::string."; + results.error("🔥ERROR: " + qualified_test_name + " " + message); + std::cout << " 🔥ERROR: " << message << std::endl; + } + + // Step 2d: Pass or fail. + if (compare_function(expected_output, actual)) { + results.pass(); + std::cout << " ✅PASSED" << std::endl; + } else { + std::ostringstream os; + os << "expected: " << expected_output << ", actual: " << actual; + results.fail("❌FAILED: " + qualified_test_name + " " + os.str()); + std::cout << " ❌FAILED: " << os.str() << std::endl; + } + + // Step 2e: Test Teardown + if (after_each.has_value()) { + (*after_each)(); + } + std::cout << " Ending Test: " << test_name << std::endl; + }); + + // Step 3: Suite Teardown + if (after_all.has_value()) { + (*after_all)(); + } + std::cout << "Ending Suite: " << suite_label << std::endl; + return results; + } /// @brief /// @tparam TResult The result type of the test. @@ -198,7 +350,8 @@ namespace Test { /// @param is_enabled If false this test run is not executed and considered skipped for reporting purposes. /// @return A TestTuple suitable for use as a test run when calling test_fn. template - TestTuple make_test( + TestTuple + make_test( const string& test_name, const TResult& expected, tuple input_params, @@ -206,7 +359,14 @@ namespace Test { MaybeTestConfigureFunction before_each = std::nullopt, MaybeTestConfigureFunction after_each = std::nullopt, bool is_enabled = true) { - return make_tuple(test_name, expected, input_params, test_compare_fn, before_each, after_each, is_enabled); + return make_tuple( + test_name, + expected, + input_params, + test_compare_fn, + before_each, + after_each, + is_enabled); } /// @brief @@ -235,7 +395,12 @@ namespace Test { template TestResults execute_suite(const TestSuite& test_suite) { - return std::apply(execute_suite, test_suite); + return execute_suite( + std::get<0>(test_suite), + std::get<1>(test_suite), + std::get<2>(test_suite) + // TODO: make this work for the optional parts of the tuple too. + ); } /// @brief