libwreport  3.6
utils/tests.h
1 #ifndef WREPORT_TESTS_H
2 #define WREPORT_TESTS_H
3 
12 #include <string>
13 #include <sstream>
14 #include <exception>
15 #include <functional>
16 #include <vector>
17 
18 namespace wreport {
19 namespace tests {
20 struct LocationInfo;
21 }
22 }
23 
24 /*
25  * These global arguments will be shadowed by local variables in functions that
26  * implement tests.
27  *
28  * They are here to act as default root nodes to fulfill method signatures when
29  * tests are called from outside other tests.
30  */
31 extern const wreport::tests::LocationInfo wreport_test_location_info;
32 
33 namespace wreport {
34 namespace tests {
35 
53 struct LocationInfo : public std::stringstream
54 {
55  LocationInfo() {}
56 
61  std::ostream& operator()();
62 };
63 
66 {
67  const char* file;
68  int line;
69  const char* call;
70  std::string local_info;
71 
72  TestStackFrame(const char* file, int line, const char* call)
73  : file(file), line(line), call(call)
74  {
75  }
76 
77  TestStackFrame(const char* file, int line, const char* call, const LocationInfo& local_info)
78  : file(file), line(line), call(call), local_info(local_info.str())
79  {
80  }
81 
82  std::string format() const;
83 
84  void format(std::ostream& out) const;
85 };
86 
87 struct TestStack : public std::vector<TestStackFrame>
88 {
89  using vector::vector;
90 
92  std::string backtrace() const;
93 
95  void backtrace(std::ostream& out) const;
96 };
97 
102 struct TestFailed : public std::exception
103 {
104  std::string message;
105  TestStack stack;
106 
107  TestFailed(const std::exception& e);
108 
109  template<typename ...Args>
110  TestFailed(const std::exception& e, Args&&... args)
111  : TestFailed(e)
112  {
113  add_stack_info(std::forward<Args>(args)...);
114  }
115 
116  TestFailed(const std::string& message) : message(message) {}
117 
118  template<typename ...Args>
119  TestFailed(const std::string& message, Args&&... args)
120  : TestFailed(message)
121  {
122  add_stack_info(std::forward<Args>(args)...);
123  }
124 
125  const char* what() const noexcept override { return message.c_str(); }
126 
127  template<typename ...Args>
128  void add_stack_info(Args&&... args) { stack.emplace_back(std::forward<Args>(args)...); }
129 };
130 
134 struct TestSkipped : public std::exception
135 {
136 };
137 
142 #define WREPORT_TEST_INFO(name) \
143  wreport::tests::LocationInfo wreport_test_location_info; \
144  wreport::tests::LocationInfo& name = wreport_test_location_info
145 
146 
148 template<typename A>
149 void assert_true(const A& actual)
150 {
151  if (actual) return;
152  std::stringstream ss;
153  ss << "actual value " << actual << " is not true";
154  throw TestFailed(ss.str());
155 };
156 
157 void assert_true(std::nullptr_t actual);
158 
160 template<typename A>
161 void assert_false(const A& actual)
162 {
163  if (!actual) return;
164  std::stringstream ss;
165  ss << "actual value " << actual << " is not false";
166  throw TestFailed(ss.str());
167 };
168 
169 void assert_false(std::nullptr_t actual);
170 
175 template<typename A, typename E>
176 void assert_equal(const A& actual, const E& expected)
177 {
178  if (actual == expected) return;
179  std::stringstream ss;
180  ss << "value '" << actual << "' is different than the expected '" << expected << "'";
181  throw TestFailed(ss.str());
182 }
183 
188 template<typename A, typename E>
189 void assert_not_equal(const A& actual, const E& expected)
190 {
191  if (actual != expected) return;
192  std::stringstream ss;
193  ss << "value '" << actual << "' is not different than the expected '" << expected << "'";
194  throw TestFailed(ss.str());
195 }
196 
198 template<typename A, typename E>
199 void assert_less(const A& actual, const E& expected)
200 {
201  if (actual < expected) return;
202  std::stringstream ss;
203  ss << "value '" << actual << "' is not less than the expected '" << expected << "'";
204  throw TestFailed(ss.str());
205 }
206 
208 template<typename A, typename E>
209 void assert_less_equal(const A& actual, const E& expected)
210 {
211  if (actual <= expected) return;
212  std::stringstream ss;
213  ss << "value '" << actual << "' is not less than or equals to the expected '" << expected << "'";
214  throw TestFailed(ss.str());
215 }
216 
218 template<typename A, typename E>
219 void assert_greater(const A& actual, const E& expected)
220 {
221  if (actual > expected) return;
222  std::stringstream ss;
223  ss << "value '" << actual << "' is not greater than the expected '" << expected << "'";
224  throw TestFailed(ss.str());
225 }
226 
228 template<typename A, typename E>
229 void assert_greater_equal(const A& actual, const E& expected)
230 {
231  if (actual >= expected) return;
232  std::stringstream ss;
233  ss << "value '" << actual << "' is not greater than or equals to the expected '" << expected << "'";
234  throw TestFailed(ss.str());
235 }
236 
238 void assert_startswith(const std::string& actual, const std::string& expected);
239 
241 void assert_endswith(const std::string& actual, const std::string& expected);
242 
244 void assert_contains(const std::string& actual, const std::string& expected);
245 
247 void assert_not_contains(const std::string& actual, const std::string& expected);
248 
255 void assert_re_matches(const std::string& actual, const std::string& expected);
256 
263 void assert_not_re_matches(const std::string& actual, const std::string& expected);
264 
265 
266 template<class A>
267 struct Actual
268 {
269  A _actual;
270  Actual(const A& actual) : _actual(actual) {}
271  ~Actual() {}
272 
273  void istrue() const { assert_true(_actual); }
274  void isfalse() const { assert_false(_actual); }
275  template<typename E> void operator==(const E& expected) const { assert_equal(_actual, expected); }
276  template<typename E> void operator!=(const E& expected) const { assert_not_equal(_actual, expected); }
277  template<typename E> void operator<(const E& expected) const { return assert_less(_actual, expected); }
278  template<typename E> void operator<=(const E& expected) const { return assert_less_equal(_actual, expected); }
279  template<typename E> void operator>(const E& expected) const { return assert_greater(_actual, expected); }
280  template<typename E> void operator>=(const E& expected) const { return assert_greater_equal(_actual, expected); }
281 };
282 
284 {
285  const char* _actual;
286  ActualCString(const char* s) : _actual(s) {}
287 
288  void istrue() const { return assert_true(_actual); }
289  void isfalse() const { return assert_false(_actual); }
290  void operator==(const char* expected) const;
291  void operator==(const std::string& expected) const;
292  void operator!=(const char* expected) const;
293  void operator!=(const std::string& expected) const;
294  void operator<(const std::string& expected) const;
295  void operator<=(const std::string& expected) const;
296  void operator>(const std::string& expected) const;
297  void operator>=(const std::string& expected) const;
298  void startswith(const std::string& expected) const;
299  void endswith(const std::string& expected) const;
300  void contains(const std::string& expected) const;
301  void not_contains(const std::string& expected) const;
302  void matches(const std::string& re) const;
303  void not_matches(const std::string& re) const;
304 };
305 
306 struct ActualStdString : public Actual<std::string>
307 {
308  ActualStdString(const std::string& s) : Actual<std::string>(s) {}
309 
310  void startswith(const std::string& expected) const;
311  void endswith(const std::string& expected) const;
312  void contains(const std::string& expected) const;
313  void not_contains(const std::string& expected) const;
314  void matches(const std::string& re) const;
315  void not_matches(const std::string& re) const;
316 };
317 
318 struct ActualDouble : public Actual<double>
319 {
320  using Actual::Actual;
321 
322  void almost_equal(double expected, unsigned places) const;
323  void not_almost_equal(double expected, unsigned places) const;
324 };
325 
326 template<typename A>
327 inline Actual<A> actual(const A& actual) { return Actual<A>(actual); }
328 inline ActualCString actual(const char* actual) { return ActualCString(actual); }
329 inline ActualCString actual(char* actual) { return ActualCString(actual); }
330 inline ActualStdString actual(const std::string& actual) { return ActualStdString(actual); }
331 inline ActualDouble actual(double actual) { return ActualDouble(actual); }
332 
333 struct ActualFunction : public Actual<std::function<void()>>
334 {
335  using Actual::Actual;
336 
337  void throws(const std::string& what_match) const;
338 };
339 
340 inline ActualFunction actual_function(std::function<void()> actual) { return ActualFunction(actual); }
341 
342 struct ActualFile : public Actual<std::string>
343 {
344  using Actual::Actual;
345 
346  void exists() const;
347  void not_exists() const;
348 };
349 
350 inline ActualFile actual_file(const std::string& pathname) { return ActualFile(pathname); }
351 
359 #define wassert(...) \
360  do { try { \
361  __VA_ARGS__ ; \
362  } catch (wreport::tests::TestFailed& e) { \
363  e.add_stack_info(__FILE__, __LINE__, #__VA_ARGS__, wreport_test_location_info); \
364  throw; \
365  } catch (std::exception& e) { \
366  throw wreport::tests::TestFailed(e, __FILE__, __LINE__, #__VA_ARGS__, wreport_test_location_info); \
367  } } while(0)
368 
370 #define wassert_true(...) wassert(actual(__VA_ARGS__).istrue())
371 
373 #define wassert_false(...) wassert(actual(__VA_ARGS__).isfalse())
374 
382 #define wcallchecked(func) \
383  [&]() { try { \
384  return func; \
385  } catch (wreport::tests::TestFailed& e) { \
386  e.add_stack_info(__FILE__, __LINE__, #func, wreport_test_location_info); \
387  throw; \
388  } catch (std::exception& e) { \
389  throw wreport::tests::TestFailed(e, __FILE__, __LINE__, #func, wreport_test_location_info); \
390  } }()
391 
392 
393 struct TestCase;
394 
399 {
401  std::string test_case;
402 
404  std::string test_method;
405 
407  std::string error_message;
408 
411 
413  std::string exception_typeid;
414 
416  bool skipped = false;
417 
418 
419  TestMethodResult(const std::string& test_case, const std::string& test_method)
420  : test_case(test_case), test_method(test_method) {}
421 
422  void set_failed(TestFailed& e)
423  {
424  error_message = e.what();
425  error_stack = e.stack;
426  if (error_message.empty())
427  error_message = "test failed with an empty error message";
428  }
429 
430  void set_exception(std::exception& e)
431  {
432  error_message = e.what();
433  if (error_message.empty())
434  error_message = "test threw an exception with an empty error message";
435  exception_typeid = typeid(e).name();
436  }
437 
438  void set_unknown_exception()
439  {
440  error_message = "unknown exception caught";
441  }
442 
443  void set_setup_exception(std::exception& e)
444  {
445  error_message = "[setup failed: ";
446  error_message += e.what();
447  error_message += "]";
448  }
449 
450  void set_teardown_exception(std::exception& e)
451  {
452  error_message = "[teardown failed: ";
453  error_message += e.what();
454  error_message += "]";
455  }
456 
457  bool is_success() const
458  {
459  return error_message.empty();
460  }
461 };
462 
467 {
469  std::string test_case;
471  std::vector<TestMethodResult> methods;
473  std::string fail_setup;
476  std::string fail_teardown;
478  bool skipped = false;
479 
480  TestCaseResult(const std::string& test_case) : test_case(test_case) {}
481 
482  void set_setup_failed()
483  {
484  fail_setup = "test case setup method threw an unknown exception";
485  }
486 
487  void set_setup_failed(std::exception& e)
488  {
489  fail_setup = "test case setup method threw an exception: ";
490  fail_setup += e.what();
491  }
492 
493  void set_teardown_failed()
494  {
495  fail_teardown = "test case teardown method threw an unknown exception";
496  }
497 
498  void set_teardown_failed(std::exception& e)
499  {
500  fail_teardown = "test case teardown method threw an exception: ";
501  fail_teardown += e.what();
502  }
503 
504  void add_test_method(TestMethodResult&& e)
505  {
506  methods.emplace_back(std::move(e));
507  }
508 
509  bool is_success() const
510  {
511  if (!fail_setup.empty() || !fail_teardown.empty()) return false;
512  for (const auto& m: methods)
513  if (!m.is_success())
514  return false;
515  return true;
516  }
517 };
518 
519 struct TestCase;
520 struct TestCaseResult;
521 struct TestMethod;
522 struct TestMethodResult;
523 
531 {
532  virtual ~TestController() {}
533 
539  virtual bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) { return true; }
540 
544  virtual void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) {}
545 
551  virtual bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) { return true; }
552 
556  virtual void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) {}
557 };
558 
566 {
568  std::string whitelist;
569 
571  std::string blacklist;
572 
573  bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) override;
574  void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) override;
575  bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
576  void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
577 
578  bool test_method_should_run(const std::string& fullname) const;
579 };
580 
581 
589 {
591  std::vector<TestCase*> entries;
592 
599  void register_test_case(TestCase& test_case);
600 
604  std::vector<TestCaseResult> run_tests(TestController& controller);
605 
607  static TestRegistry& get();
608 };
609 
614 {
616  std::string name;
617 
619  std::function<void()> test_function;
620 
621  TestMethod(const std::string& name, std::function<void()> test_function)
622  : name(name), test_function(test_function) {}
623 };
624 
625 
630 struct TestCase
631 {
633  std::string name;
634 
636  std::vector<TestMethod> methods;
637 
638  TestCase(const std::string& name)
639  : name(name)
640  {
642  }
643  virtual ~TestCase() {}
644 
652  virtual void register_tests() = 0;
653 
657  virtual void setup() {}
658 
662  virtual void teardown() {}
663 
667  virtual void method_setup(TestMethodResult&) {}
668 
673 
681  virtual TestCaseResult run_tests(TestController& controller);
682 
695  virtual TestMethodResult run_test(TestController& controller, TestMethod& method);
696 
700  template<typename ...Args>
701  void add_method(const std::string& name, std::function<void()> test_function)
702  {
703  methods.emplace_back(name, test_function);
704  }
705 
709  template<typename ...Args>
710  void add_method(const std::string& name, std::function<void()> test_function, Args&&... args)
711  {
712  methods.emplace_back(name, test_function, std::forward<Args>(args)...);
713  }
714 
720  template<typename FUNC, typename ...Args>
721  void add_method(const std::string& name, FUNC test_function, Args&&... args)
722  {
723  auto f = std::bind(test_function, args...);
724  methods.emplace_back(name, f);
725  }
726 };
727 
728 
739 struct Fixture
740 {
741  // Called before each test
742  void test_setup() {}
743 
744  // Called after each test
745  void test_teardown() {}
746 };
747 
748 template<typename Fixture, typename... Args>
749 static inline Fixture* fixture_factory(Args... args)
750 {
751  return new Fixture(args...);
752 }
753 
757 template<typename FIXTURE>
758 class FixtureTestCase : public TestCase
759 {
760 public:
761  typedef FIXTURE Fixture;
762 
763  Fixture* fixture = nullptr;
764  std::function<Fixture*()> make_fixture;
765 
766  template<typename... Args>
767  FixtureTestCase(const std::string& name, Args... args)
768  : TestCase(name)
769  {
770  make_fixture = std::bind(fixture_factory<FIXTURE, Args...>, args...);
771  }
772 
773  void setup() override
774  {
775  TestCase::setup();
776  fixture = make_fixture();
777  }
778 
779  void teardown() override
780  {
781  delete fixture;
782  fixture = 0;
784  }
785 
786  void method_setup(TestMethodResult& mr) override
787  {
789  if (fixture) fixture->test_setup();
790  }
791 
793  {
794  if (fixture) fixture->test_teardown();
796  }
797 
804  template<typename FUNC>
805  void add_method(const std::string& name, FUNC test_function)
806  {
807  methods.emplace_back(name, [=]() {
808  test_function(*fixture);
809  });
810  }
811 };
812 
813 #if 0
814  struct Test
815  {
816  std::string name;
817  std::function<void()> test_func;
818  };
819 
821  virtual void add_tests() {}
822 #endif
823 
824 
825 }
826 }
827 
828 #endif
Test registry.
Definition: utils/tests.h:588
virtual void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result)
Called after running a test method.
Definition: utils/tests.h:556
std::string exception_typeid
If non-empty, the test threw an exception and this is its type ID.
Definition: utils/tests.h:413
Result of running a whole test case.
Definition: utils/tests.h:466
Test case collecting several test methods, and self-registering with the singleton instance of TestRe...
Definition: utils/tests.h:630
std::string name
Name of the test case.
Definition: utils/tests.h:633
Information about one stack frame in the test execution stack.
Definition: utils/tests.h:65
Add information to the test backtrace for the tests run in the current scope.
Definition: utils/tests.h:53
std::string test_method
Name of the test method.
Definition: utils/tests.h:404
Exception thrown when a test or a test case needs to be skipped.
Definition: utils/tests.h:134
virtual bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result)
Called before running a test method.
Definition: utils/tests.h:551
void add_method(const std::string &name, std::function< void()> test_function, Args &&...args)
Register a new test method.
Definition: utils/tests.h:710
virtual void method_setup(TestMethodResult &)
Set up before the test method is run.
Definition: utils/tests.h:667
std::vector< TestMethodResult > methods
Outcome of all the methods that have been run.
Definition: utils/tests.h:471
Simple default implementation of TestController.
Definition: utils/tests.h:565
TestStack error_stack
Stack frame of where the error happened.
Definition: utils/tests.h:410
Abstract interface for the objects that supervise test execution.
Definition: utils/tests.h:530
std::string fail_teardown
Set to a non-empty string if the teardown method of the test case failed.
Definition: utils/tests.h:476
Definition: utils/tests.h:267
void teardown() override
Clean up after the test case is run.
Definition: utils/tests.h:779
Exception thrown when a test assertion fails, normally by Location::fail_test.
Definition: utils/tests.h:102
void method_teardown(TestMethodResult &mr) override
Clean up after the test method is run.
Definition: utils/tests.h:792
void register_test_case(TestCase &test_case)
Register a new test case.
std::vector< TestMethod > methods
All registered test methods.
Definition: utils/tests.h:636
Definition: utils/tests.h:87
static TestRegistry & get()
Get the singleton instance of TestRegistry.
void add_method(const std::string &name, FUNC test_function, Args &&...args)
Register a new test metheod, with arguments.
Definition: utils/tests.h:721
virtual void setup()
Set up the test case before it is run.
Definition: utils/tests.h:657
std::vector< TestCase * > entries
All known test cases.
Definition: utils/tests.h:591
Definition: utils/tests.h:283
virtual void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result)
Called after running a test case.
Definition: utils/tests.h:544
std::string test_case
Name of the test case.
Definition: utils/tests.h:469
Definition: utils/tests.h:333
Definition: utils/tests.h:318
std::string blacklist
Any method matching this glob expression will not be run.
Definition: utils/tests.h:571
Result of running a test method.
Definition: utils/tests.h:398
virtual bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result)
Called before running a test case.
Definition: utils/tests.h:539
Definition: utils/tests.h:342
void setup() override
Set up the test case before it is run.
Definition: utils/tests.h:773
std::function< void()> test_function
Main body of the test method.
Definition: utils/tests.h:619
void method_setup(TestMethodResult &mr) override
Set up before the test method is run.
Definition: utils/tests.h:786
Definition: utils/tests.h:306
void add_method(const std::string &name, FUNC test_function)
Add a method that takes a reference to the fixture as argument.
Definition: utils/tests.h:805
std::string whitelist
Any method not matching this glob expression will not be run.
Definition: utils/tests.h:568
std::string error_message
If non-empty, the test failed with this error.
Definition: utils/tests.h:407
String functions.
Definition: benchmark.h:13
Test case that includes a fixture.
Definition: utils/tests.h:758
void add_method(const std::string &name, std::function< void()> test_function)
Register a new test method.
Definition: utils/tests.h:701
Test method information.
Definition: utils/tests.h:613
virtual void method_teardown(TestMethodResult &)
Clean up after the test method is run.
Definition: utils/tests.h:672
virtual void teardown()
Clean up after the test case is run.
Definition: utils/tests.h:662
std::string name
Name of the test method.
Definition: utils/tests.h:616
Base class for test fixtures.
Definition: utils/tests.h:739
std::string fail_setup
Set to a non-empty string if the setup method of the test case failed.
Definition: utils/tests.h:473
std::string test_case
Name of the test case.
Definition: utils/tests.h:401