risorse | good unit tests

Good Unit Tests /8

Questa parte (l'ottava; qui la prima, qui la seconda, qui la terza, qui la quarta, qui la quinta, qui la sesta e qui la settima) introduce la possibilità di emettere della messaggistica in console.

Notifiche emesse dal codice cliente

Sono disponibili quattro tipi di notifiche:

EVAL

visualizza il valore di un'espressione se il test fallisce;

INFO

visualizza un messaggio testuale libero se il test fallisce;

WARN

visualizza un messaggio testuale libero indipendentemente dall'esito del test;

FAIL

visualizza un messaggio testuale libero indipendentemente dall'esito del test e causa il fallimento dello stesso.

EVAL

L'istruzione EVAL cattura l'espressione specificata notificandone il valore al report. A tal fine si rende necessario introdurre un nuovo tipo di informativa, Eval, nonché estendere l'interfaccia Report:

/*
 * NOTE: `Failure` struct has been renamed in `Notice`!
 */
...

struct Eval : public Notice {
  template <typename T>
  Eval(const char* expression, const T& value, const char* file, int line) : Notice("eval", file, line) {
    description << expression << " evaluates to " << value;
  }
};
...

class Report {
  ...
  static void failure(const Notice& failure) {
    ...
  }
  static void eval(const Notice& eval) {
    if (report_)
      report_->onEval(eval);
  }  ...

  virtual void onFailure(const Notice& /*failure*/) { }
  virtual void onEval(const Notice& /*eval*/) { }
  ...

};
...

#define EVAL(expr_) \
  do { \
    gut::Report::eval(gut::Eval(#expr_, expr_, __FILE__, __LINE__)); \
  } while (0)

Il programma di test diventa:

...

class LastFailureTestReport : public gut::Report {
  std::string& what_;
  std::string& failure_;
  std::string& eval_;
  // disabled!
  LastFailure& operator=(const LastFailure&);
  TestReport& operator=(const TestReport&);
public:
  LastFailure(std::string& what) : what_(what) {
  TestReport(std::string& failure, std::string& eval) : failure_(failure), eval_(eval) {
  }
protected:
  virtual void onFailure(const gut::Notice& failure) {
    whatfailure_ = failure.what();
  }
  virtual void onEval(const Notice& eval) {
    eval_ = eval.what();
  }
};

...
int main() {

  std::string lastFailure;
  gut::Report::set(std::make_shared<LastFailure>(lastFailure));
  std::string lastEval;
  gut::Report::set(std::make_shared<TestReport>(lastFailure, lastEval));
  ...

  // test EVAL
  assert(lastEval == "");
  EVAL(i1);
  assert(lastEval == "[eval] i1 evaluates to 1");
  EVAL((i1 + 3 * i2));
  assert(lastEval == "[eval] (i1 + 3 * i2) evaluates to 7");

  return 0;
}

Affinché l'esito del test continui ad apparire accanto al nome dello stesso, il DefaultReport emette tutte le notifiche eval assieme, al termine del test, se questo fallisce:

class DefaultReport : public Report {
  ...
  std::vector<std::string> evals_;
public:
  ...
protected:
  ...
  virtual void onEndTest() {
    if (testFailures_ == 0)
      std::cout << "OK" << std::endl;
    else
      ++failedTests_;
      testFailed();
    clear();
  }
  ...
  virtual void onFailure(const Notice& failure) {
    if (testFailures_ == 0)
      std::cout << "FAILED" << std::endl;
    std::cout << " " << failure.location.file << "(" << failure.location.line << ") : " << failure.what() << std::endl;
    std::cout << toString(failure) << std::endl;
    ++testFailures_;
    ++totalFailures_;
  }
  virtual void onEval(const Notice& eval) {
    evals_.push_back(toString(eval));
  }
  virtual void onQuit() {
      ++failedTests_;
      testFailed();
  }
  void testFailed() {
    ++failedTests_;
    flushEvals();
  }
  void flushEvals() {
    if (!evals_.empty()) {
      for (auto eval : evals_)
        std::cout << eval << std::endl;
    }
  }
  void clear() {
    evals_.clear();
  }
  std::string toString(const Notice& notice) {
    std::stringstream ss;
    ss << " " << notice.location.file << "(" << notice.location.line << ") : " << notice.what();
    return ss.str();
  }
};

Introducendo un'istruzione EVAL nel codice d'esempio l'effetto è nullo:

...
TEST("Initial list is empty") {
  RecentlyUsedList anEmptyList;

  EVAL(anEmptyList.size());

  CHECK(anEmptyList.empty());
  CHECK(anEmptyList.size() == 0);
}

/* output:
 *
 * Test suite started...
 * Initial list is empty: OK
 * Insertion to empty list is retained: OK
 * Distinct insertions are retained in stack order: OK
 * Duplicate insertions are moved to the front but not inserted: OK
 * Out of range indexing throws exception: OK
 * Ran 5 test(s) in 0s.
 * OK - all tests passed.
 */

Tuttavia, se il test fallisce, compare anche la valutazione dell'espressione richiesta:

...
TEST("Initial list is empty") {
  RecentlyUsedList anEmptyList;

  EVAL(anEmptyList.size());
  CHECK(false);

  CHECK(anEmptyList.empty());
  CHECK(anEmptyList.size() == 0);
}

/* output:
 *
 * Test suite started...
 * Initial list is empty: FAILED
 *  example.cpp(32) : [error] false evaluates to false
 *  example.cpp(31) : [eval] anEmptyList.size() evaluates to 0
 * Insertion to empty list is retained: OK
 * Distinct insertions are retained in stack order: OK
 * Duplicate insertions are moved to the front but not inserted: OK
 * Out of range indexing throws exception: OK
 * Ran 5 test(s) in 0s.
 * FAILED - 1 failure(s) in 1 test(s).
 */

Da notare che la notifica [eval] segue quella [error], sebbene nel codice la preceda.

INFO

In modo del tutto analogo a quanto fatto per EVAL, si procede per INFO:

...
struct Info : public Notice {
  Info(const char* message, const char* file, int line) : Notice("info", file, line) {
    description << message;
  }
};
...

class Report {
...
public:
  ...
  static void info(const Notice& info) {
    if (report_)
      report_->onInfo(info);
  }
  ...
protected:
  ...
  virtual void onInfo(const Notice& /*info*/) { }
  virtual void onQuit() { }
};

#define INFO(message_) \
  do { \
    gut::Report::info(gut::Info(message_, __FILE__, __LINE__)); \
  } while (0)

Il codice di test diventa:

...
class TestReport : public gut::Report {
  std::string& failure_;
  std::string& eval_;
  std::string& info_;
  // disabled!
  TestReport& operator=(const Report&);
public:
  TestReport(
    std::string& failure,
    std::string& eval,
    std::string& info)
  : failure_(failure), eval_(eval), info_(info) {
  }
protected:
  virtual void onFailure(const gut::Notice& failure) {
    failure_ = failure.what();
  }
  virtual void onEval(const gut::Notice& eval) {
    eval_ = eval.what();
  }
  virtual void onInfo(const gut::Notice& info) {
    info_ = info.what();
  }
};
...

int main() {

  std::string lastFailure;
  std::string lastEval;
  std::string lastInfo;
  gut::Report::set(
    std::make_shared<TestReport>(
      lastFailure, lastEval, lastInfo));
  ...

  // test INFO
  assert(lastInfo == "");
  INFO("message #1");
  assert(lastInfo == "[info] message #1");
  INFO("message #2");
  assert(lastInfo == "[info] message #2");

  return 0;
}

Un esempio di utilizzo, dopo aver riadattato il DefaultReport per emettere le notifiche INFO solo se il test fallisce, intercalandole con le notifiche EVAL secondo l'ordine di ricevimento:

class DefaultReport : public Report {
  ...
  std::vector<std::string> evalslog_;
  ...
protected:
  ...
  virtual void onEval(const Notice& eval) {
    evalslog_.push_back(toString(eval));
  }
  virtual void onInfo(const Notice& info) {
    log_.push_back(toString(info));
  }
  ...
  void testFailed() {
    ++failedTests_;
    flushEvals();
    flushLog();
  }
  void flushEvals() {
    if (!evals_.empty()) {
      for (auto eval : evals_)
        std::cout << eval << std::endl;
    }
  }
  void flushLog() {
    if (!log_.empty()) {
      for (auto message : log_)
        std::cout << message << std::endl;
    }
  }
  void clear() {
    evalslog_.clear();
  }

  ...
};
...

TEST("Initial list is empty") {
  RecentlyUsedList anEmptyList;

  INFO("after anEmptyList initialization");

  CHECK(anEmptyList.empty());
  CHECK(anEmptyList.size() == 0);
}
...

/* output:
 *
 * Test suite started...
 * Initial list is empty: OK
 * Insertion to empty list is retained: OK
 * Distinct insertions are retained in stack order: OK
 * Duplicate insertions are moved to the front but not inserted: OK
 * Out of range indexing throws exception: OK
 * Ran 5 test(s) in 0.001s.
 * OK - all tests passed.
 */

Se il test ha successo, l'effetto dell'istruzione INFO è nullo; il messaggio viene infatti emesso solo se il test fallisce, sempre dopo la segnalazione dell'errore:

TEST("Initial list is empty") {
  RecentlyUsedList anEmptyList;

  INFO("after anEmptyList initialization");
  CHECK(false);

  CHECK(anEmptyList.empty());
  CHECK(anEmptyList.size() == 0);
}
...

/* output:
 *
 * Test suite started...
 * Initial list is empty: FAILED
 *  example.cpp(33) : [error] false evaluates to false
 *  example.cpp(32) : [info] after anEmptyList initialization
 * Insertion to empty list is retained: OK
 * Distinct insertions are retained in stack order: OK
 * Duplicate insertions are moved to the front but not inserted: OK
 * Out of range indexing throws exception: OK
 * Ran 5 test(s) in 0.001s.
 * FAILED - 1 failure(s) in 1 test(s).
 */

L'ordine relativo delle istruzioni EVAL/INFO è mantenuto:

TEST("Initial list is empty") {
  RecentlyUsedList anEmptyList;

  EVAL(anEmptyList.size());
  INFO("after anEmptyList initialization");
  CHECK (false);

  CHECK(anEmptyList.empty());
  CHECK(anEmptyList.size() == 0);
}

/* output:
 *
 * Test suite started...
 * Initial list is empty: FAILED
 *  example.cpp(33) : [error] false evaluates to false
 *  example.cpp(31) : [eval] anEmptyList.size() evaluates to 0
 *  example.cpp(32) : [info] after anEmptyList initialization
 * Insertion to empty list is retained: OK
 * Distinct insertions are retained in stack order: OK
 * Duplicate insertions are moved to the front but not inserted: OK
 * Out of range indexing throws exception: OK
 * Ran 5 test(s) in 0.002s.
 * FAILED - 1 failure(s) in 1 test(s).
 */

Avendo introdotto il concetto di log in DefaultReport, con una semplice modifica si può fare in modo che tutte le notifiche vengano emesse nell'esatto ordine di ricezione:

class DefaultReport : public Report {
...
protected:
  ...
  virtual void onFailure(const Notice& failure) {
    if (testFailures_ == 0)
      std::cout << "FAILED" << std::endl;
    std::cout << toString(failure) << std::endl;
    log_.push_back(toString(failure));
    ++testFailures_;
    ++totalFailures_;
  }
  ...
};

/* output:
 *
 * Test suite started...
 * Initial list is empty: FAILED
 *  example.cpp(31) : [eval] anEmptyList.size() evaluates to 0
 *  example.cpp(32) : [info] after anEmptyList initialization
 *  example.cpp(33) : [error] false evaluates to false
 * Insertion to empty list is retained: OK
 * Distinct insertions are retained in stack order: OK
 * Duplicate insertions are moved to the front but not inserted: OK
 * Out of range indexing throws exception: OK
 * Ran 5 test(s) in 0.001s.
 * FAILED - 1 failure(s) in 1 test(s).
 */

WARN

Le notifiche WARN sono a tutti gli effetti delle INFO con la differenza che queste vanno visualizzate anche quanto il test ha successo. Necessitando del tipo di notifica in fase di visualizzazione, conviene rifattorizzare la classe Notice:

#define PICK_NAME(id_) e_ ## id_,
#define PICK_LABEL(id_) #id_,

#define LEVELS(lambda_) \
  lambda_(info) \
  lambda_(warning) \
  lambda_(error) \
  lambda_(fatal) \

enum Level { LEVELS(PICK_NAME) };
static std::string level_name[] = { LEVELS(PICK_LABEL) };

class Notice {
  Level level_;
  Location location_;
  std::ostringstream content_;
public:
  Notice(Level level, const char* file, int line) : level_(level), location_(file, line) {
    content_ << "[" << level_name[level] << "] ";
  }
  Notice(const Notice& notice) : level_(notice.level_), location_(notice.location_) {
    content_ << notice.content_.str();
  }
  Level level() const {
    return level_;
  }
  std::string what() const {
    return content_.str();
  }
  std::string toString() const {
    std::ostringstream ss;
    ss << location_.file << "(" << location_.line << ") : " << what();
    return ss.str();
  }
protected:
  std::ostream& content() {
    return content_;
  }
};

struct Error : public Notice {
  Error(const char* file, int line) : Notice("error"e_error, file, line) { }
};

struct Fatal : public Notice {
  Fatal(const char* file, int line) : Notice("fatal"e_fatal, file, line) { }
};

struct CheckFailure : public Error {
  CheckFailure(const char* expression, const std::string& expansion, const char* file, int line) : Error(file, line) {
    descriptioncontent() << expression << " evaluates to " << expansion;
  }
};

struct RequireFailure : public Fatal {
  RequireFailure(const char* expression, const std::string& expansion, const char* file, int line) : Fatal(file, line) {
    descriptioncontent() << expression << " evaluates to " << expansion;
  }
};

struct NoThrowFailure : public Error {
  NoThrowFailure(const char* expression, const char* file, int line) : Error(file, line) {
    descriptioncontent() << expression << " did not throw";
  }
};

struct WrongTypedExceptionFailure : public Error {
  WrongTypedExceptionFailure(const char* expression, const std::exception& exception, const char* file, int line) : Error(file, line) {
    descriptioncontent() << expression << " threw an unexpected exception \"" << exception.what() << "\"";
  }
};

struct WrongExceptionFailure : public Error {
  WrongExceptionFailure(const char* expression, const char* file, int line) : Error(file, line) {
    descriptioncontent() << expression << " threw an unknown exception";
  }
};

struct UnexpectedExceptionFailure : public Fatal {
  UnexpectedExceptionFailure(const std::exception& exception, const char* file, int line) : Fatal(file, line) {
    descriptioncontent() << "unexpected exception \"" << exception.what() << "\" caught";
  }
};

struct UnknownExceptionFailure : public Fatal {
  UnknownExceptionFailure(const char* file, int line) : Fatal(file, line) {
    descriptioncontent() << "unknown exception caught";
  }
};

struct Eval : public Notice {
  template <typename T>
  Eval(const char* expression, const T& value, const char* file, int line) : Notice("eval"e_info, file, line) {
    descriptioncontent() << expression << " evaluates to " << value;
  }
};

struct Info : public Notice {
  Info(const char* message, const char* file, int line) : Notice("info"e_info, file, line) {
    descriptioncontent() << message;
  }
};

struct Warn : public Notice {
  Warn(const char* message, const char* file, int line) : Notice(e_warning, file, line) {
    content() << message;
  }
};
...

class Report {
...
public:
  ...
  static void warn(const Notice& warn) {
    if (report_)
      report_->onWarn(warn);
  }
  ...
protected:
  ...
  virtual void onWarn(const Notice& /*warn*/) { }
  virtual void onQuit() { }
};
...

class DefaultReport : public Report {
  ...
protected:
  ...
  std::string toString(const Notice& notice) {
    std::stringstream ss;
    ss << " " << notice.location.file << "(" << notice.location.line << ") : " << notice.what();
    ss << " " << notice.toString();
    return ss.str();
  }
};
...

#define WARN(message_) \
  do { \
    gut::Report::warn(gut::Warn(message_, __FILE__, __LINE__)); \
  } while (0)

La codifica delle tipologie di notifica è stata realizzata tramite x-macro. Il programma di test diventa:

class TestReport : public gut::Report {
  ...
  std::string& info_;
  std::string& warn_;
  ...
public:
  TestReport(
    std::string& failure,
    std::string& eval,
    std::string& info,
    std::string& warn)
  : failure_(failure), eval_(eval), info_(info), warn_(warn) {
  }
protected:
  ...
  virtual void onWarn(const gut::Notice& warn) {
    warn_ = warn.what();
  }
};
...

int main() {

  std::string lastFailure;
  std::string lastEval;
  std::string lastInfo;
  std::string lastWarn;
  gut::Report::set(
    std::make_shared<TestReport>(
      lastFailure, lastEval, lastInfo, lastWarn));
  ...

  // test EVAL
  assert(lastEval == "");
  EVAL(i1);
  assert(lastEval == "[evalinfo] i1 evaluates to 1");
  EVAL((i1 + 3 * i2));
  assert(lastEval == "[evalinfo] (i1 + 3 * i2) evaluates to 7");
  ...

  // test WARN
  assert(lastWarn == "");
  WARN("message #1");
  assert(lastWarn == "[warning] message #1");
  WARN("message #2");
  assert(lastWarn == "[warning] message #2");

  return 0;
}

L'esempio, una volta riadattato l'oggetto DefaultReport, diventa:

class DefaultReport : public Report {
  ...
  size_t failedTests_;
  std::vector<std::stringNotice> log_;
public:
  ...
  virtual void onEndTest() {
    if (testFailures_ == 0) {
      std::cout << "OK" << std::endl;
      flushLog(e_warning);
    }
    else
      testFailed();
    clear();
  }
  virtual void onFailure(const Notice& failure) {
    if (testFailures_ == 0)
      std::cout << "FAILED" << std::endl;
    log_.push_back(toString(failure));
    ++testFailures_;
    ++totalFailures_;
  }
  virtual void onEval(const Notice& eval) {
    log_.push_back(toString(eval));
  }
  virtual void onInfo(const Notice& info) {
    log_.push_back(toString(info));
  }
  virtual void onWarn(const Notice& warn) {
    log_.push_back(warn);
  }
  virtual void onQuit() {
    testFailed();
  }
  void testFailed() {
    ++failedTests_;
    flushLog(e_info);
  }
  void flushLog(Level minLevel) {
    if (!log_.empty()) {
      for (auto message : log_)
        std::cout << message << std::endl;
    for (auto notice : log_) {
      if (notice.level() >= minLevel)
        std::cout << " " << notice.toString() << std::endl;
    }
  }
  void clear() {
    log_.clear();
  }
  std::string toString(const Notice& notice) {
    std::stringstream ss;
    ss << " " << notice.toString();
    return ss.str();
  }
};
...

TEST("Initial list is empty") {
  RecentlyUsedList anEmptyList;

  WARN("verify that...");

  CHECK(anEmptyList.empty());
  CHECK(anEmptyList.size() == 0);
}

/* output:
 *
 * Test suite started...
 * Initial list is empty: OK
 *  example.cpp(31) : [warning] verify that...
 * Insertion to empty list is retained: OK
 * Distinct insertions are retained in stack order: OK
 * Duplicate insertions are moved to the front but not inserted: OK
 * Out of range indexing throws exception: OK
 * Ran 5 test(s) in 0.001s.
 * OK - all tests passed.
 */

Come si può notare, l'avviso è riportato anche se il test si è concluso con successo.

FAIL

La richiesta FAIL causa l'immediato fallimento del test, alla stregua di una REQUEST:

namespace gut {
  ...

struct UserFailure : public Fatal {
  UserFailure(const char* message, const char* file, int line) : Fatal(file, line) {
    content() << message;
  }
};
...

} // namespace gut
...

#define FAIL(message_) \
  do { \
    gut::Report::failure(gut::UserFailure(message_, __FILE__, __LINE__)); \
  } while (0)

Il programma di test diventa:

int main() {
  ...

  // test FAIL
  FAIL("user failure");
  assert(lastFailure == "[fatal] user failure");


  return 0;
}

Riprendendo l'esempio della RecentlyUsedList, un tipico utilizzo di FAIL è il seguente:

...

TEST("Clearing the list makes it empty") {
  FAIL("TODO!");
}

/* output:
 * Test suite started...
 * Initial list is empty: OK
 * Insertion to empty list is retained: OK
 * Distinct insertions are retained in stack order: OK
 * Duplicate insertions are moved to the front but not inserted: OK
 * Out of range indexing throws exception: OK
 * Clearing the list makes it empty: FAILED
 *  example.cpp(79) : [fatal] TODO!
 * Ran 6 test(s) in 0s.
 * FAILED - 1 failure(s) in 1 test(s).
 */

Varie

Colori in console

Gran parte dei framework di test ricorrono al testo colorato per evidenziare l'esito dei test; sfruttando del codice a suo tempo già descritto, si può ottenere qualcosa di analogo:

...
#ifdef GUT_HAS_CHRONO
#include <chrono>
#else
#include <time.h>
#endif

#include "colors.h"

#define INT_BASE Dec
#define CHAR_BASE Hex
...

class DefaultReport : public Report {
  ...
  virtual void onEnd() {
    std::cout << "Ran " << tests_ << " test(s) in " << clock_.elapsedTime() << "s." << std::endl;
    if (failedTests_ == 0)
      std::cout << color::lime << "OK - all tests passed." << color::reset << std::endl;
    else
      std::cout << color::red << "FAILED - " << totalFailures_ << " failure(s) in " << failedTests_ << " test(s)." << color::reset << std::endl;
  }
  ...
};

L'effetto che si ottiene per un test con esito positivo è il seguente:

In caso d'errore invece:

Revisione dell'attivazione dell'opzione fail-fast

È stata modificata la modalità di attivazione dell'opzione fail-fast, introducendo la macro GUT_ENABLE_FAILFAST che va specificata dopo l'inclusione del file gut.h:

#ifdef GUT_FAILFAST
FailFast failFast_;
#endif
#define GUT_ENABLE_FAILFAST gut::FailFast failFast_;

Il programma d'esempio diventa:

#define GUT_FAILFAST

#include "gut.h"

GUT_ENABLE_FAILFAST

TEST("fail-fast /1") {
  CHECK(1 == 2);
  CHECK(2 == 3);
}

TEST("fail-fast /2") {
  CHECK(1 == 2);
  CHECK(2 == 3);
}

/* output:
 *
 * Test suite started...
 * fail-fast /1: FAILED
 *  failfast.cpp(6) : [error] 1 == 2 evaluates to 1 == 2
 * Ran 1 test(s) in 0s.
 * FAILED - 1 failure(s) in 1 test(s).
 */

Codice sorgente

• colors.h
• gut.h
• test-gut.cpp
• failfast.cpp
• example.cpp

Pagina modificata il 29/05/2013