# Mu - subx/003trace.cc

      1 //: The goal of layers is to make programs more easy to understand and more
      2 //: malleable, easy to rewrite in radical ways without accidentally breaking
      3 //: some corner case. Tests further both goals. They help understandability by
      4 //: letting one make small changes and get feedback. What if I wrote this line
      5 //: like so? What if I removed this function call, is it really necessary?
      6 //: Just try it, see if the tests pass. Want to explore rewriting this bit in
      7 //: this way? Tests put many refactorings on a firmer footing.
      8 //:
      9 //: But the usual way we write tests seems incomplete. Refactorings tend to
     10 //: work in the small, but don't help with changes to function boundaries. If
     11 //: you want to extract a new function you have to manually test-drive it to
     12 //: create tests for it. If you want to inline a function its tests are no
     13 //: longer valid. In both cases you end up having to reorganize code as well as
     14 //: tests, an error-prone activity.
     15 //:
     16 //: In response, this layer introduces the notion of *domain-driven* testing.
     17 //: We focus on the domain of inputs the whole program needs to handle rather
     18 //: than the correctness of individual functions. All tests invoke the program
     19 //: in a single way: by calling run() with some input. As the program operates
     20 //: on the input, it traces out a list of _facts_ deduced about the domain:
     21 //:   trace("label") << "fact 1: " << val;
     22 //:
     23 //: Tests can now check these facts:
     24 //:   :(scenario foo)
     25 //:   34  # call run() with this input
     26 //:   +label: fact 1: 34  # 'run' should have deduced this fact
     27 //:   -label: fact 1: 35  # the trace should not contain such a fact
     28 //:
     29 //: Since we never call anything but the run() function directly, we never have
     30 //: to rewrite the tests when we reorganize the internals of the program. We
     31 //: just have to make sure our rewrite deduces the same facts about the domain,
     32 //: and that's something we're going to have to do anyway.
     33 //:
     34 //: To avoid the combinatorial explosion of integration tests, each layer
     35 //: mainly logs facts to the trace with a common *label*. All tests in a layer
     36 //: tend to check facts with this label. Validating the facts logged with a
     37 //: specific label is like calling functions of that layer directly.
     38 //:
     39 //: To build robust tests, trace facts about your domain rather than details of
     40 //: how you computed them.
     41 //:
     42 //: More details: http://akkartik.name/blog/tracing-tests
     43 //:
     44 //: ---
     45 //:
     46 //: Between layers and domain-driven testing, programming starts to look like a
     47 //: fundamentally different activity. Instead of a) superficial, b) local rules
     48 //: on c) code [like say http://blog.bbv.ch/2013/06/05/clean-code-cheat-sheet],
     49 //: we allow programmers to engage with the a) deep, b) global structure of the
     50 //: c) domain. If you can systematically track discontinuities in the domain,
     51 //: you don't care if the code used gotos as long as it passed the tests. If
     52 //: tests become more robust to run it becomes easier to try out radically
     53 //: different implementations for the same program. If code is super-easy to
     54 //: rewrite, it becomes less important what indentation style it uses, or that
     55 //: the objects are appropriately encapsulated, or that the functions are
     56 //: referentially transparent.
     57 //:
     58 //: Instead of plumbing, programming becomes building and gradually refining a
     59 //: map of the environment the program must operate under. Whether a program is
     60 //: 'correct' at a given point in time is a red herring; what matters is
     61 //: avoiding regression by monotonically nailing down the more 'eventful' parts
     62 //: of the terrain. It helps readers new and old, and rewards curiosity, to
     63 //: organize large programs in self-similar hierarchies of example scenarios
     64 //: colocated with the code that makes them work.
     65 //:
     66 //:   "Programming properly should be regarded as an activity by which
     67 //:   programmers form a mental model, rather than as production of a program."
     68 //:   -- Peter Naur (http://alistair.cockburn.us/ASD+book+extract%3A+%22Naur,+Ehn,+Musashi%22)
     69
     70 :(before "End Types")
     71 struct trace_line {
     72   int depth;  // optional field just to help browse traces later
     73   string label;
     74   string contents;
     75   trace_line(string l, string c) :depth(0), label(l), contents(c) {}
     76   trace_line(int d, string l, string c) :depth(d), label(l), contents(c) {}
     77 };
     78
     79 //: Support for tracing an entire run.
     80 //: Traces can have a lot of overhead, so only turn them on when asked.
     81 :(before "End Commandline Options(*arg)")
     82 else if (is_equal(*arg, "--trace")) {
     83   Save_trace = true;
     84 }
     85 :(before "End Commandline Parsing")
     86 if (Save_trace) {
     87   cerr << "initializing trace\n";
     88   Trace_stream = new trace_stream;
     89 }
     90 :(code)
     91 void cleanup_main() {
     92   if (!Trace_stream) return;
     93   if (Save_trace)
     94     Trace_stream->save();
     95   delete Trace_stream;
     96   Trace_stream = NULL;
     97 }
     98 :(before "End One-time Setup")
     99 atexit(cleanup_main);
    100
    101 :(before "End Types")
    102 // Pre-define some global constants that trace_stream needs to know about.
    103 // Since they're in the Types section, they'll be included in any cleaved
    104 // compilation units. So no extern linkage.
    105 const int Max_depth = 9999;
    106 const int Error_depth = 0;  // definitely always print errors
    107 const int Warn_depth = 1;
    108
    109 struct trace_stream {
    110   vector<trace_line> past_lines;
    111   // accumulator for current line
    112   ostringstream* curr_stream;
    113   string curr_label;
    114   int curr_depth;
    115   int collect_depth;
    116   ofstream null_stream;  // never opens a file, so writes silently fail
    117   trace_stream() :curr_stream(NULL), curr_depth(Max_depth), collect_depth(Max_depth) {}
    118   ~trace_stream() { if (curr_stream) delete curr_stream; }
    119
    120   ostream& stream(string label) {
    121     return stream(Max_depth, label);
    122   }
    123
    124   ostream& stream(int depth, string label) {
    125     if (depth > collect_depth) return null_stream;
    126     curr_stream = new ostringstream;
    127     curr_label = label;
    128     curr_depth = depth;
    129     (*curr_stream) << std::hex;
    130     return *curr_stream;
    131   }
    132
    133   void save() {
    134     cerr << "saving trace to 'last_run'\n";
    135     ofstream fout("last_run");
    136     fout << readable_contents("");
    137     fout.close();
    138   }
    139
    140   // be sure to call this before messing with curr_stream or curr_label
    141   void newline();
    142   // useful for debugging
    143   string readable_contents(string label);  // empty label = show everything
    144 };
    145
    146 :(code)
    147 void trace_stream::newline() {
    148   if (!curr_stream) return;
    149   string curr_contents = curr_stream->str();
    150   if (!curr_contents.empty()) {
    151     past_lines.push_back(trace_line(curr_depth, trim(curr_label), curr_contents));  // preserve indent in contents
    152     if ((!Hide_errors && curr_depth == Error_depth)
    153         || (!Hide_warnings && !Hide_errors && curr_depth == Warn_depth)
    154         || Dump_trace
    155         || (!Dump_label.empty() && curr_label == Dump_label))
    156       cerr << curr_label << ": " << curr_contents << '\n';
    157   }
    158   delete curr_stream;
    159   curr_stream = NULL;
    160   curr_label.clear();
    161   curr_depth = Max_depth;
    162 }
    163
    164 string trace_stream::readable_contents(string label) {
    165   ostringstream output;
    166   label = trim(label);
    167   for (vector<trace_line>::iterator p = past_lines.begin();  p != past_lines.end();  ++p)
    168     if (label.empty() || label == p->label)
    169       output << std::setw(4) << p->depth << ' ' << p->label << ": " << p->contents << '\n';
    170   return output.str();
    171 }
    172
    173 :(before "End Globals")
    174 trace_stream* Trace_stream = NULL;
    175 int Trace_errors = 0;  // used only when Trace_stream is NULL
    176
    177 //: option to print trace to the screen
    178 :(before "End Globals")
    179 bool Flag_dump_trace = false;
    180 :(before "End Commandline Options(*arg)")
    181 else if (is_equal(*arg, "--dump")) {
    182   Flag_dump_trace = true;
    183 }
    184
    185 :(before "End Globals")
    186 bool Hide_errors = false;  // if set, don't print even error trace lines to screen
    187 bool Hide_warnings = false;  // if set, don't print warnings to screen
    188 bool Dump_trace = false;  // if set, print trace lines to screen
    189 string Dump_label = "";  // if set, print trace lines matching a single label to screen
    190 :(before "End Reset")
    191 Hide_errors = false;
    192 Hide_warnings = false;
    193 Dump_trace = Flag_dump_trace;
    194 Dump_label = "";
    195 //: Never dump warnings in scenarios
    196 :(before "End Test Setup")
    197 Hide_warnings = true;
    198
    199 :(before "End Includes")
    200 #define CLEAR_TRACE  delete Trace_stream, Trace_stream = new trace_stream;
    201
    202 // Top-level helper. IMPORTANT: can't nest
    203 #define trace(...)  !Trace_stream ? cerr /*print nothing*/ : Trace_stream->stream(__VA_ARGS__)
    204
    205 // Just for debugging; 'git log' should never show any calls to 'dbg'.
    206 #define dbg trace(0, "a")
    207 #define DUMP(label)  if (Trace_stream) cerr << Trace_stream->readable_contents(label);
    208
    209 // Errors and warnings are special layers.
    210 #define raise  (!Trace_stream ? (++Trace_errors,cerr) /*do print*/ : Trace_stream->stream(Error_depth, "error"))
    211 #define warn (!Trace_stream ? (++Trace_errors,cerr) /*do print*/ : Trace_stream->stream(Warn_depth, "warn"))
    212 // If we aren't yet sure how to deal with some corner case, use assert_for_now
    213 // to indicate that it isn't an inviolable invariant.
    214 #define assert_for_now assert
    215 #define raise_for_now raise
    216
    217 // Inside tests, fail any tests that displayed (unexpected) errors.
    218 // Expected errors in tests should always be hidden and silently checked for.
    219 :(before "End Test Teardown")
    220 if (Passed && !Hide_errors && trace_contains_errors()) {
    221   Passed = false;
    222 }
    223 :(code)
    224 bool trace_contains_errors() {
    225   return Trace_errors > 0 || trace_count("error") > 0;
    226 }
    227
    228 :(before "End Types")
    229 struct end {};
    230 :(code)
    231 ostream& operator<<(ostream& os, end /*unused*/) {
    232   if (Trace_stream) Trace_stream->newline();
    233   return os;
    234 }
    235
    236 :(before "End Globals")
    237 bool Save_trace = false;  // if set, write out trace to disk
    238
    239 // Trace_stream is a resource, lease_tracer uses RAII to manage it.
    240 :(before "End Types")
    241 struct lease_tracer {
    242   lease_tracer();
    243   ~lease_tracer();
    244 };
    245 :(code)
    246 lease_tracer::lease_tracer() { Trace_stream = new trace_stream; }
    247 lease_tracer::~lease_tracer() {
    248   if (Save_trace) Trace_stream->save();
    249   delete Trace_stream, Trace_stream = NULL;
    250 }
    251 :(before "End Includes")
    252 #define START_TRACING_UNTIL_END_OF_SCOPE  lease_tracer leased_tracer;
    253 :(before "End Test Setup")
    254 START_TRACING_UNTIL_END_OF_SCOPE
    255
    256 :(before "End Includes")
    257 #define CHECK_TRACE_CONTENTS(...)  check_trace_contents(__FUNCTION__, __FILE__, __LINE__, __VA_ARGS__)
    258
    259 #define CHECK_TRACE_CONTAINS_ERRORS()  CHECK(trace_contains_errors())
    260 #define CHECK_TRACE_DOESNT_CONTAIN_ERRORS() \
    261   if (Passed && trace_contains_errors()) { \
    262     cerr << "\nF - " << __FUNCTION__ << "(" << __FILE__ << ":" << __LINE__ << "): unexpected errors\n"; \
    263     DUMP("error"); \
    264     Passed = false; \
    265     return; \
    266   }
    267
    268 #define CHECK_TRACE_COUNT(label, count) \
    269   if (Passed && trace_count(label) != (count)) { \
    270     cerr << "\nF - " << __FUNCTION__ << "(" << __FILE__ << ":" << __LINE__ << "): trace_count of " << label << " should be " << count << '\n'; \
    271     cerr << "  got " << trace_count(label) << '\n';  /* multiple eval */ \
    272     DUMP(label); \
    273     Passed = false; \
    274     return;  /* Currently we stop at the very first failure. */ \
    275   }
    276
    277 #define CHECK_TRACE_DOESNT_CONTAIN(...)  CHECK(trace_doesnt_contain(__VA_ARGS__))
    278
    279 :(code)
    280 bool check_trace_contents(string FUNCTION, string FILE, int LINE, string expected) {
    281   if (!Passed) return false;
    282   if (!Trace_stream) return false;
    283   vector<string> expected_lines = split(expected, "^D");
    284   int curr_expected_line = 0;
    285   while (curr_expected_line < SIZE(expected_lines) && expected_lines.at(curr_expected_line).empty())
    286     ++curr_expected_line;
    287   if (curr_expected_line == SIZE(expected_lines)) return true;
    288   string label, contents;
    289   split_label_contents(expected_lines.at(curr_expected_line), &label, &contents);
    290   for (vector<trace_line>::iterator p = Trace_stream->past_lines.begin();  p != Trace_stream->past_lines.end();  ++p) {
    291     if (label != p->label) continue;
    292     if (contents != trim(p->contents)) continue;
    293     ++curr_expected_line;
    294     while (curr_expected_line < SIZE(expected_lines) && expected_lines.at(curr_expected_line).empty())
    295       ++curr_expected_line;
    296     if (curr_expected_line == SIZE(expected_lines)) return true;
    297     split_label_contents(expected_lines.at(curr_expected_line), &label, &contents);
    298   }
    299
    300   if (line_exists_anywhere(label, contents)) {
    301     cerr << "\nF - " << FUNCTION << "(" << FILE << ":" << LINE << "): line [" << label << ": " << contents << "] out of order in trace:\n";
    302     DUMP("");
    303   }
    304   else {
    305     cerr << "\nF - " << FUNCTION << "(" << FILE << ":" << LINE << "): missing [" << contents << "] in trace:\n";
    306     DUMP(label);
    307   }
    308   Passed = false;
    309   return false;
    310 }
    311
    312 void split_label_contents(const string& s, string* label, string* contents) {
    313   static const string delim(": ");
    314   size_t pos = s.find(delim);
    315   if (pos == string::npos) {
    316     *label = "";
    317     *contents = trim(s);
    318   }
    319   else {
    320     *label = trim(s.substr(0, pos));
    321     *contents = trim(s.substr(pos+SIZE(delim)));
    322   }
    323 }
    324
    325 bool line_exists_anywhere(const string& label, const string& contents) {
    326   for (vector<trace_line>::iterator p = Trace_stream->past_lines.begin();  p != Trace_stream->past_lines.end();  ++p) {
    327     if (label != p->label) continue;
    328     if (contents == trim(p->contents)) return true;
    329   }
    330   return false;
    331 }
    332
    333 int trace_count(string label) {
    334   return trace_count(label, "");
    335 }
    336
    337 int trace_count(string label, string line) {
    338   if (!Trace_stream) return 0;
    339   long result = 0;
    340   for (vector<trace_line>::iterator p = Trace_stream->past_lines.begin();  p != Trace_stream->past_lines.end();  ++p) {
    341     if (label == p->label) {
    342       if (line == "" || trim(line) == trim(p->contents))
    343         ++result;
    344     }
    345   }
    346   return result;
    347 }
    348
    349 int trace_count_prefix(string label, string prefix) {
    350   if (!Trace_stream) return 0;
    351   long result = 0;
    352   for (vector<trace_line>::iterator p = Trace_stream->past_lines.begin();  p != Trace_stream->past_lines.end();  ++p) {
    353     if (label == p->label) {
    354       if (starts_with(trim(p->contents), trim(prefix)))
    355         ++result;
    356     }
    357   }
    358   return result;
    359 }
    360
    361 bool trace_doesnt_contain(string label, string line) {
    362   return trace_count(label, line) == 0;
    363 }
    364
    365 bool trace_doesnt_contain(string expected) {
    366   vector<string> tmp = split_first(expected, ": ");
    367   if (SIZE(tmp) == 1) {
    368     raise << expected << ": missing label or contents in trace line\n" << end();
    369     assert(false);
    370   }
    371   return trace_doesnt_contain(tmp.at(0), tmp.at(1));
    372 }
    373
    374 vector<string> split(string s, string delim) {
    375   vector<string> result;
    376   size_t begin=0, end=s.find(delim);
    377   while (true) {
    378     if (end == string::npos) {
    379       result.push_back(string(s, begin, string::npos));
    380       break;
    381     }
    382     result.push_back(string(s, begin, end-begin));
    383     begin = end+SIZE(delim);
    384     end = s.find(delim, begin);
    385   }
    386   return result;
    387 }
    388
    389 vector<string> split_first(string s, string delim) {
    390   vector<string> result;
    391   size_t end=s.find(delim);
    392   result.push_back(string(s, 0, end));
    393   if (end != string::npos)
    394     result.push_back(string(s, end+SIZE(delim), string::npos));
    395   return result;
    396 }
    397
    398 string trim(const string& s) {
    399   string::const_iterator first = s.begin();
    400   while (first != s.end() && isspace(*first))
    401     ++first;
    402   if (first == s.end()) return "";
    403
    404   string::const_iterator last = --s.end();
    405   while (last != s.begin() && isspace(*last))
    406     --last;
    407   ++last;
    408   return string(first, last);
    409 }
    410
    411 :(before "End Includes")
    412 #include <vector>
    413 using std::vector;
    414 #include <list>
    415 using std::list;
    416 #include <set>
    417 using std::set;
    418
    419 #include <sstream>
    420 using std::istringstream;
    421 using std::ostringstream;
    422
    423 #include <fstream>
    424 using std::ifstream;
    425 using std::ofstream;