maj/StorytellersBestFriend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TinyTest improvements.

Browse Source 
Adds ostream formatting for tuples and vectors.
Moves execute_suite implementation into test.h.
Adds a better test execution sequence and better test reporting.
Adds message support for test errors/failures/skips.
Removes more dead code from test.cpp.
This commit is contained in:
Tom Hicks
2023-04-15 16:22:40 -07:00
parent 7f95c4dabb
commit 3751d78dfe
2 changed files with 246 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
sbf-cpp/test.cpp
View File

@@ -24,10 +24,6 @@ using std::for_each;
// using namespace Test;
namespace Test {
const string fn1(const string& s, int l) {
return s.substr(0, l);
}
// Test lifecycle
// suiteSetupFn(); - This is called to allocate any suite level resources. This is called once when the suite begins.
// These functions may be called in parallel but execution will not proceed past this block until they have all finished.
@@ -86,135 +82,6 @@ namespace Test {
// Then some test harness either generated or explicit can call test_main_ThingDoer(...) and optionally reported there. Reporting granularity is controlled by how frequently you call collect_and_report_TestResults(...).
// You can combine test results with results = results + testFn(..); and then collect_and_report_TestResults on the aggregate TestResults value.
template <typename TResult, typename... TInputParams>
TestResults execute_suite(
string suite_label,
function<TResult(TInputParams...)> function_to_test,
vector<TestTuple<TResult, TInputParams...>> tests,
MaybeTestCompareFunction<TResult> maybe_suite_compare_function,
MaybeTestConfigureFunction maybe_suite_before_each_function,
MaybeTestConfigureFunction maybe_suite_after_each_function,
bool is_enabled) {
TestResults results;
cout << "🚀 Beginning Suite: " << suite_label << endl;
// Step 1: Suite Setup
if (maybe_suite_before_each_function.has_value()) {
(*maybe_suite_before_each_function)();
}
// Step 2: Execute Tests
for_each(tests.begin(), tests.end(), [&suite_label, &function_to_test, &results, &maybe_suite_compare_function](
TestTuple<TResult, TInputParams...> test_data
) {
// Step 2a: Extract our variables from the TestTuple.
const std::string& test_name = get<0>(test_data);
const std::string qualified_test_name = suite_label + "::" + test_name;
const TResult& expected_output = get<1>(test_data);
std::tuple<TInputParams...> input_params = get<2>(test_data);
MaybeTestCompareFunction<TResult> maybe_compare_function = get<3>(test_data);
TestCompareFunction<TResult> compare_function = maybe_compare_function.has_value()
? *maybe_compare_function
: maybe_suite_compare_function.has_value()
? *maybe_suite_compare_function
: [](const TResult& l, const TResult& r){return l==r;};
MaybeTestConfigureFunction before_each = get<4>(test_data);
MaybeTestConfigureFunction after_each = get<5>(test_data);
// Step 2b: Test Setup
cout << " Beginning Test: " << qualified_test_name << endl;
if(before_each.has_value()) {
(*before_each)();
}
TResult actual;
try {
// Step 2c: Execute the test method.
actual = std::apply(function_to_test, input_params);
} catch(const std::exception& ex) {
cout << " ERROR: Caught exception \"" << ex.what() << "\"" << endl;
} catch(const std::string& message) {
cout << " ERROR: Caught string \"" << message << "\"" << endl;
} catch(...) {
cout << " ERROR: Caught something that is neither an std::exception nor a std::string." << endl;
}
// Step 2d: Pass or fail.
TestResults result;
if (compare_function(expected_output, actual)) {
result = TestResults().pass();
cout << " PASSED" << endl;
} else {
result = TestResults().fail();
cout << " FAILED: expected: " << expected_output << ", actual: " << actual << endl;
}
results += result;
// Step 2e: Test Teardown
if (after_each.has_value()) {
(*after_each)();
}
cout << " Ending Test: " << test_name << endl;
});
// Step 3: Suite Teardown
if (maybe_suite_after_each_function.has_value()) {
(*maybe_suite_after_each_function)();
}
cout << "Ending Suite: " << suite_label << endl;
return results;
}
TestResults do_the_other_thing(){
auto p1 = "Microsoft QBasic";
auto p2 = 5;
// auto exp = "Micro";
string s = fn1("Microsoft QBasic", 5);
TestResults tr;
// tr = tr + execute_suite<string, const string&, int>(
// "Test 8 Function",
// (function<string(const string&, int)>)fn1,
// vector<TestTuple<string, const string&, int>>({
// // vector<tuple<string, string, tuple<const string&, int>, MaybeTestCompareFunction<string>>>({
// make_tuple(
// string("should do something"), // test_name
// string("Micro"), // expectedOutput
// make_tuple((string)p1, p2),// inputParams,
// std::nullopt, // compare_function
// std::nullopt, // before_each
// std::nullopt, // after_each
// true
// ),
// make_test<string, string, int>(
// "should do something else",
// "Micro",
// make_tuple((string)p1, p2)
// )
// }));
auto test_data8 = vector<TestTuple<string, const string&, int>>({
make_test<string, string, int>(
"Test 8 equals", "Micro", make_tuple((string)p1, p2),
[](const string& l, const string& r){ return l==r;}),
make_test<string, string, int>(
"Test 8 not equals", "Micro", make_tuple((string)p1, p2),
[](const string& l, const string& r){ return l!=r;}
),
make_test<string, string, int>("Test 8 default compare", "Micro", make_tuple((string)p1, p2)),
make_test<string, string, int>("Test 8 default compare", "Micro", make_tuple((string)p1, p2)),
make_test<string, string, int>("Test 8 default compare", "Micro", make_tuple((string)p1, p2))
});
tr = tr + execute_suite<string, const string&, int>(
"Test 8 Function with extra data",
(function<string(const string&, int)>)fn1,
test_data8
);
return tr;
}
// _Step_9 - if T2 is a single value then make_tuple<T2>(T2) and call longer version
// auto testFunction = [](int id){return id==0?"":"";};
// auto compareFunction = [](const string a, const string b){return a==b;};
@@ -226,29 +93,60 @@ namespace Test {
// Also allow make_tuple(T2) if the last param is not a tuple.
TestResults::TestResults()
: failed_(0)
: errors_(0)
, failed_(0)
, passed_(0)
, skipped_(0)
, total_(0) {}
TestResults::TestResults(const TestResults& other)
: failed_(other.failed_)
: error_messages_(other.error_messages_)
, errors_(other.errors_)
, failed_(other.failed_)
, failure_messages_(other.failure_messages_)
, passed_(other.passed_)
, skip_messages_(other.skip_messages_)
, skipped_(other.skipped_)
, total_(other.total_) {}
TestResults::TestResults(uint32_t failed, uint32_t passed, uint32_t skipped, uint32_t total)
: failed_(failed)
TestResults::TestResults(uint32_t errors, uint32_t failed, uint32_t passed, uint32_t skipped, uint32_t total, vector<string> error_messages, vector<string> failure_messages, vector<string> skip_messages)
: error_messages_(error_messages)
, errors_(errors)
, failed_(failed)
, failure_messages_(failure_messages)
, passed_(passed)
, skip_messages_(skip_messages)
, skipped_(skipped)
, total_(total) {}
TestResults& TestResults::error() {
errors_++;
return *this;
}
TestResults& TestResults::error(string message) {
errors_++;
error_messages_.push_back(message);
return *this;
}
TestResults& TestResults::fail() {
total_++;
failed_++;
return *this;
}
TestResults& TestResults::fail(const string& message) {
total_++;
failed_++;
failure_messages_.push_back(message);
return *this;
}
vector<string> TestResults::failure_messages() {
return failure_messages_;
}
TestResults& TestResults::pass() {
total_++;
passed_++;
@@ -261,6 +159,25 @@ namespace Test {
return *this;
}
TestResults& TestResults::skip(const string& message) {
total_++;
skipped_++;
skip_messages_.push_back(message);
return *this;
}
vector<string> TestResults::skip_messages() {
return skip_messages_;
}
vector<string> TestResults::error_messages() {
return error_messages_;
}
uint32_t TestResults::errors() {
return errors_;
}
uint32_t TestResults::failed() {
return failed_;
}
@@ -278,16 +195,34 @@ namespace Test {
}
TestResults TestResults::operator+(const TestResults& other) const {
vector<string> error_messages;
error_messages.insert(error_messages.end(), error_messages_.begin(), error_messages_.end());
error_messages.insert(error_messages.end(), other.error_messages_.begin(), other.error_messages_.end());
vector<string> failure_messages;
failure_messages.insert(failure_messages.end(), failure_messages_.begin(), failure_messages_.end());
failure_messages.insert(failure_messages.end(), other.failure_messages_.begin(), other.failure_messages_.end());
vector<string> skip_messages;
skip_messages.insert(skip_messages.end(), skip_messages_.begin(), skip_messages_.end());
skip_messages.insert(skip_messages.end(), other.skip_messages_.begin(), other.skip_messages_.end());
return TestResults(
errors_ + other.errors_,
failed_ + other.failed_,
passed_ + other.passed_,
skipped_ + other.skipped_,
total_ + other.total_);
total_ + other.total_,
error_messages,
failure_messages,
skip_messages);
}
TestResults& TestResults::operator+=(const TestResults& other) {
error_messages_.insert(error_messages_.end(), other.error_messages_.begin(), other.error_messages_.end());
errors_ += other.errors_;
failed_ += other.failed_;
failure_messages_.insert(failure_messages_.end(), other.failure_messages_.begin(), other.failure_messages_.end());
passed_ += other.passed_;
skip_messages_.insert(skip_messages_.end(), other.skip_messages_.begin(), other.skip_messages_.end());
skipped_ += other.skipped_;
total_ += other.total_;
return *this;

181
sbf-cpp/test.h
View File

@@ -1,9 +1,11 @@
#ifndef TEST_H__
#ifndef TEST_H__
#define TEST_H__
#include <cstdint>
#include <tuple>
#include <utility>
#include <string>
#include <iostream>
#include <sstream>
// Test lifecycle
// suite_setup_function(); - This is called to allocate any suite level resources. This is called once when the suite begins.
@@ -19,6 +21,26 @@
// Collect reports - Ths step is not visible to the user at this point, but data returned by all of the test functions is collected here. This is where you will eventually be able to format/log data for reports.
// suite_teardown_function(); - This is called after all test calls have completed, all test_teardown_function calls have completed, and all test reports/logs have been written. You should free any resources allocated in suite_setup_function.
// Tuple printer from: https://stackoverflow.com/questions/6245735/pretty-print-stdtuple/31116392#58417285
template<typename TChar, typename TTraits, typename... TArgs>
auto& operator<<(std::basic_ostream<TChar, TTraits>& os, std::tuple<TArgs...> const& t) {
std::apply([&os](auto&&... args) {((os << args << " "), ...);}, t);
return os;
}
template<typename TChar, typename TTraits, typename TItem>
auto& operator<<(std::basic_ostream<TChar, TTraits>& os, std::vector<TItem> v) {
os << "[ ";
for (auto it = v.begin(); it != v.end(); it++) {
if (it != v.begin()) {
os << ", ";
}
os << *it;
}
os << " ]";
return os;
}
namespace Test {
using std::tuple;
using std::pair;
@@ -36,16 +58,34 @@ namespace Test {
TestResults(const TestResults& other);
/// @brief Creates a new TestResults instance with specific counts.
/// @param errors The number of errors while running the tests.
/// @param failed The number of failed tests.
/// @param passed The number of passed tests.
/// @param skipped The number of skipped tests.
/// @param total The total number of tests run. This should equal the sum of failed, passed, and skipped tests.
TestResults(uint32_t failed, uint32_t passed, uint32_t skipped, uint32_t total);
/// @param error_messages The list of error messages.
/// @param failure_messages The list of failure messages.
/// @param skip_messages The list of skip messages.
TestResults(uint32_t errors, uint32_t failed, uint32_t passed, uint32_t skipped, uint32_t total, std::vector<std::string> error_messages, std::vector<std::string> failure_messages, std::vector<std::string> skip_messages);
/// @brief Adds an error. This increments errors.
/// @return A reference to this instance. Used for chaining.
TestResults& error();
/// @brief Adds an error with a message. This increments errors as well as saving the error message.
/// @param message The error message.
/// @return A reference to this instance. Used for chaining.
TestResults& error(std::string message);
/// @brief Adds a failed test. This increments total and failed.
/// @return A reference to this instance. Used for chaining.
TestResults& fail();
/// @brief Adds a failed test with a message. This increments total and failed as well as saving the failure message.
/// @param message The reason the test failed.
/// @return A reference to this instance. Used for chaining.
TestResults& fail(const std::string& message);
/// @brief Adds a passed test. This increments total and passed.
/// @return A reference to this instance. Used for chaining.
TestResults& pass();
@@ -54,10 +94,27 @@ namespace Test {
/// @return A reference to this instance. Used for chaining.
TestResults& skip();
/// @brief Adds a skipped test with a message. This increments total and skipped as well as saving the skip message.
/// @param message The reason the test was skipped.
/// @return A reference to this instance. Used for chaining.
TestResults& skip(const std::string& message);
/// @brief Getter for the list of error messages.
/// @return
vector<string> error_messages();
/// @brief Getter for the count of errors.
/// @return
uint32_t errors();
/// @brief Getter for the count of failed tests.
/// @return The count of failed tests.
uint32_t failed();
/// @brief Getter for the list of failure messages.
/// @return The list of failure messages.
vector<string> failure_messages();
/// @brief Getter for the count of passed tests.
/// @return The count of passed tests.
uint32_t passed();
@@ -66,6 +123,10 @@ namespace Test {
/// @return The count of skipped tests.
uint32_t skipped();
/// @brief Getter for the list of skip messages.
/// @return The list of skip messages.
vector<string> skip_messages();
/// @brief Getter for the count of total tests.
/// @return The count of total tests run.
uint32_t total();
@@ -81,8 +142,12 @@ namespace Test {
TestResults& operator+=(const TestResults& other);
private:
std::vector<std::string> error_messages_;
uint32_t errors_;
uint32_t failed_;
std::vector<std::string> failure_messages_;
uint32_t passed_;
std::vector<std::string> skip_messages_;
uint32_t skipped_;
uint32_t total_;
};
@@ -100,13 +165,14 @@ namespace Test {
using TestConfigureFunction = std::function<void()>;
using MaybeTestConfigureFunction = std::optional<TestConfigureFunction>;
// TODO: For some reason all hell breaks loose if test_name or expected output are const&. Figure out why.
/// @brief
/// @tparam TResult
/// @tparam ...TInputParams
template<typename TResult, typename... TInputParams>
using TestTuple = std::tuple<
const std::string& /* test_name */,
const TResult& /* expected_output */,
std::string /* test_name */,
TResult /* expected_output */,
std::tuple<TInputParams...> /* input_params - The input parameters for this test. These will be used when calling std::apply with function_to_test to execute the test. */,
MaybeTestCompareFunction<TResult> /* test_compare_function - If this is not nullprt then this function will be called instead of suite_compare_function to determine if the test passes. Use this to check for side effects of the test. Return true if the test passes and false otherwise. */,
MaybeTestConfigureFunction /* test_setup_function - If this is not nullptr this function is called before each test to setup the environment. It is called with std::apply and input_params so you can use them to mock records with specific IDs or calculate an expected result. */,
@@ -184,7 +250,93 @@ namespace Test {
MaybeTestConfigureFunction before_all = std::nullopt,
MaybeTestConfigureFunction after_all = std::nullopt,
bool is_enabled = true
);
) {
TestResults results;
std::cout << "🚀Beginning Suite: " << suite_label << std::endl;
// Step 1: Suite Setup
if (before_all.has_value()) {
(*before_all)();
}
// Step 2: Execute Tests
for_each(tests.begin(), tests.end(), [&suite_label, &function_to_test, &results, &suite_compare](
TestTuple<TResult, TInputParams...> test_data
) {
// Step 2a: Extract our variables from the TestTuple.
const std::string& test_name = std::get<0>(test_data);
const std::string qualified_test_name = suite_label + "::" + test_name;
const TResult& expected_output = std::get<1>(test_data);
std::tuple<TInputParams...> input_params = std::get<2>(test_data);
MaybeTestCompareFunction<TResult> maybe_compare_function = std::get<3>(test_data);
TestCompareFunction<TResult> compare_function = maybe_compare_function.has_value()
? *maybe_compare_function
: suite_compare.has_value()
? *suite_compare
: [](const TResult& l, const TResult& r){return l==r;};
MaybeTestConfigureFunction before_each = std::get<4>(test_data);
MaybeTestConfigureFunction after_each = std::get<5>(test_data);
bool is_enabled = std::get<6>(test_data);
if (!is_enabled) {
std::cout << " 🚧Skipping Test: " << test_name << std::endl;
results.skip("🚧Skipping Test: " + qualified_test_name);
return;
}
// Step 2b: Test Setup
std::cout << " Beginning Test: " << test_name << std::endl;
if(before_each.has_value()) {
(*before_each)();
}
TResult actual;
try {
// Step 2c: Execute the test method.
actual = std::apply(function_to_test, input_params);
} catch(const std::exception& ex) {
std::ostringstream os;
os << "Caught exception \"" << ex.what() << "\"";
results.error("🔥ERROR: " + qualified_test_name + " " + os.str());
std::cout << " 🔥ERROR: " << os.str() << std::endl;
} catch(const std::string& message) {
std::ostringstream os;
os << "Caught string \"" << message << "\"";
results.error("🔥ERROR: " + qualified_test_name + " " + os.str());
std::cout << " 🔥ERROR: " << os.str() << std::endl;
} catch(...) {
string message = "Caught something that is neither an std::exception nor a std::string.";
results.error("🔥ERROR: " + qualified_test_name + " " + message);
std::cout << " 🔥ERROR: " << message << std::endl;
}
// Step 2d: Pass or fail.
if (compare_function(expected_output, actual)) {
results.pass();
std::cout << " ✅PASSED" << std::endl;
} else {
std::ostringstream os;
os << "expected: " << expected_output << ", actual: " << actual;
results.fail("❌FAILED: " + qualified_test_name + " " + os.str());
std::cout << " ❌FAILED: " << os.str() << std::endl;
}
// Step 2e: Test Teardown
if (after_each.has_value()) {
(*after_each)();
}
std::cout << " Ending Test: " << test_name << std::endl;
});
// Step 3: Suite Teardown
if (after_all.has_value()) {
(*after_all)();
}
std::cout << "Ending Suite: " << suite_label << std::endl;
return results;
}
/// @brief
/// @tparam TResult The result type of the test.
@@ -198,7 +350,8 @@ namespace Test {
/// @param is_enabled If false this test run is not executed and considered skipped for reporting purposes.
/// @return A TestTuple suitable for use as a test run when calling test_fn.
template<typename TResult, typename... TInputParams>
TestTuple<TResult, TInputParams...> make_test(
TestTuple<TResult, TInputParams...>
make_test(
const string& test_name,
const TResult& expected,
tuple<TInputParams...> input_params,
@@ -206,7 +359,14 @@ namespace Test {
MaybeTestConfigureFunction before_each = std::nullopt,
MaybeTestConfigureFunction after_each = std::nullopt,
bool is_enabled = true) {
return make_tuple(test_name, expected, input_params, test_compare_fn, before_each, after_each, is_enabled);
return make_tuple(
test_name,
expected,
input_params,
test_compare_fn,
before_each,
after_each,
is_enabled);
}
/// @brief
@@ -235,7 +395,12 @@ namespace Test {
template <typename TResult, typename... TInputParams>
TestResults execute_suite(const TestSuite<TResult, TInputParams...>& test_suite) {
return std::apply(execute_suite<TResult, TInputParams...>, test_suite);
return execute_suite<TResult, TInputParams...>(
std::get<0>(test_suite),
std::get<1>(test_suite),
std::get<2>(test_suite)
// TODO: make this work for the optional parts of the tuple too.
);
}
/// @brief