utils/google-benchmark/test/skip_with_error_test.cc

   1
   2 #undef NDEBUG
   3 #include <cassert>
   4 #include <vector>
   5
   6 #include "../src/check.h"  // NOTE: check.h is for internal use only!
   7 #include "benchmark/benchmark.h"
   8
   9 namespace {
  10
  11 class TestReporter : public benchmark::ConsoleReporter {
  12  public:
  13   virtual bool ReportContext(const Context& context) {
  14     return ConsoleReporter::ReportContext(context);
  15   };
  16
  17   virtual void ReportRuns(const std::vector<Run>& report) {
  18     all_runs_.insert(all_runs_.end(), begin(report), end(report));
  19     ConsoleReporter::ReportRuns(report);
  20   }
  21
  22   TestReporter() {}
  23   virtual ~TestReporter() {}
  24
  25   mutable std::vector<Run> all_runs_;
  26 };
  27
  28 struct TestCase {
  29   std::string name;
  30   bool error_occurred;
  31   std::string error_message;
  32
  33   typedef benchmark::BenchmarkReporter::Run Run;
  34
  35   void CheckRun(Run const& run) const {
  36     CHECK(name == run.benchmark_name) << "expected " << name << " got "
  37                                       << run.benchmark_name;
  38     CHECK(error_occurred == run.error_occurred);
  39     CHECK(error_message == run.error_message);
  40     if (error_occurred) {
  41       // CHECK(run.iterations == 0);
  42     } else {
  43       CHECK(run.iterations != 0);
  44     }
  45   }
  46 };
  47
  48 std::vector<TestCase> ExpectedResults;
  49
  50 int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) {
  51   for (auto TC : v) {
  52     TC.name = base_name + TC.name;
  53     ExpectedResults.push_back(std::move(TC));
  54   }
  55   return 0;
  56 }
  57
  58 #define CONCAT(x, y) CONCAT2(x, y)
  59 #define CONCAT2(x, y) x##y
  60 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
  61
  62 }  // end namespace
  63
  64 void BM_error_before_running(benchmark::State& state) {
  65   state.SkipWithError("error message");
  66   while (state.KeepRunning()) {
  67     assert(false);
  68   }
  69 }
  70 BENCHMARK(BM_error_before_running);
  71 ADD_CASES("BM_error_before_running", {{"", true, "error message"}});
  72
  73 void BM_error_during_running(benchmark::State& state) {
  74   int first_iter = true;
  75   while (state.KeepRunning()) {
  76     if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
  77       assert(first_iter);
  78       first_iter = false;
  79       state.SkipWithError("error message");
  80     } else {
  81       state.PauseTiming();
  82       state.ResumeTiming();
  83     }
  84   }
  85 }
  86 BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);
  87 ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},
  88                                       {"/1/threads:2", true, "error message"},
  89                                       {"/1/threads:4", true, "error message"},
  90                                       {"/1/threads:8", true, "error message"},
  91                                       {"/2/threads:1", false, ""},
  92                                       {"/2/threads:2", false, ""},
  93                                       {"/2/threads:4", false, ""},
  94                                       {"/2/threads:8", false, ""}});
  95
  96 void BM_error_after_running(benchmark::State& state) {
  97   while (state.KeepRunning()) {
  98     benchmark::DoNotOptimize(state.iterations());
  99   }
 100   if (state.thread_index <= (state.threads / 2))
 101     state.SkipWithError("error message");
 102 }
 103 BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);
 104 ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},
 105                                      {"/threads:2", true, "error message"},
 106                                      {"/threads:4", true, "error message"},
 107                                      {"/threads:8", true, "error message"}});
 108
 109 void BM_error_while_paused(benchmark::State& state) {
 110   bool first_iter = true;
 111   while (state.KeepRunning()) {
 112     if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
 113       assert(first_iter);
 114       first_iter = false;
 115       state.PauseTiming();
 116       state.SkipWithError("error message");
 117     } else {
 118       state.PauseTiming();
 119       state.ResumeTiming();
 120     }
 121   }
 122 }
 123 BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);
 124 ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"},
 125                                     {"/1/threads:2", true, "error message"},
 126                                     {"/1/threads:4", true, "error message"},
 127                                     {"/1/threads:8", true, "error message"},
 128                                     {"/2/threads:1", false, ""},
 129                                     {"/2/threads:2", false, ""},
 130                                     {"/2/threads:4", false, ""},
 131                                     {"/2/threads:8", false, ""}});
 132
 133 int main(int argc, char* argv[]) {
 134   benchmark::Initialize(&argc, argv);
 135
 136   TestReporter test_reporter;
 137   benchmark::RunSpecifiedBenchmarks(&test_reporter);
 138
 139   typedef benchmark::BenchmarkReporter::Run Run;
 140   auto EB = ExpectedResults.begin();
 141
 142   for (Run const& run : test_reporter.all_runs_) {
 143     assert(EB != ExpectedResults.end());
 144     EB->CheckRun(run);
 145     ++EB;
 146   }
 147   assert(EB == ExpectedResults.end());
 148
 149   return 0;
 150 }