Refactor pytest backend

Instead of writing events for each phase (setup, call, teardown) of a
test (and eventually displaying them in the datatree widget) we now
accumulate results over the individual test phases and only then write
the final result of the entire test upon test completion using the
pytest_runtest_logfinish hook.

This ensures the following:
1) Capturing of output from the teardown phase even if it passes
   without an error. This fixes spyder-ide#127.
2) Differentiating between the outcome of the test itself (the call
   phase) and the entire test as a whole. An error during teardown,
   for instance, used to overwrite the (potentially passed) test result
   of the call phase. Now we get a passed test with an error during
   teardown. So in the datatree widget, the test status (first column)
   is no longer 1:1 linked with the color of this line (see also 3)).
3) Better handling of xfailed and xpassed tests which now show the
   correct status `xfailed` or `xpassed` and are colored green and
   red respectively (instead of status `skipped` and gray color for
   both of them). This fixes spyder-ide#47.
4) Better messages: the first message, which is usually the most
   important one, will be placed in the message column of the datatree
   widget. Further messages will be _appended_ to the extra_text field
   (instead of overwriting messages from previous test phases). If the
   first message spans over multiple lines then only its first line
   will be displayed in the message column and the complete message
   will be repeated in the extra_text field for better readability.
   This improves the visual impression of the datatree widget so that
   each (collapsed) row is exactly one line high.

spyder_unittest/backend/pytestrunner.py

@@ -133,30 +133,23 @@ def logreport_starttest_to_str(report):
 
 def logreport_to_testresult(report, config):
     """Convert a logreport sent by test process to a TestResult."""
-    if report['outcome'] == 'passed':
-        cat = Category.OK
-        status = 'ok'
-    elif report['outcome'] == 'failed':
+    status = report['outcome']
+    if report['outcome'] in ('failed', 'xpassed') or report['witherror']:
         cat = Category.FAIL
-        status = 'failure'
+    elif report['outcome'] in ('passed', 'xfailed'):
+        cat = Category.OK
     else:
         cat = Category.SKIP
-        status = report['outcome']
     testname = convert_nodeid_to_testname(report['nodeid'])
-    duration = report['duration']
-    message = report['message'] if 'message' in report else ''
-    if 'longrepr' not in report:
-        extra_text = ''
-    elif isinstance(report['longrepr'], tuple):
-        extra_text = report['longrepr'][2]
-    else:
-        extra_text = report['longrepr']
+    message = report.get('message', '')
+    extra_text = report.get('longrepr', '')
     if 'sections' in report:
+        if extra_text:
+            extra_text +=  '\n'
         for (heading, text) in report['sections']:
-            extra_text += '----- {} -----\n'.format(heading)
-            extra_text += text
+            extra_text += '----- {} -----\n{}'.format(heading, text)
     filename = osp.join(config.wdir, report['filename'])
     result = TestResult(cat, status, testname, message=message,
-                        time=duration, extra_text=extra_text,
+                        time=report['duration'], extra_text=extra_text,
                         filename=filename, lineno=report['lineno'])
     return result

spyder_unittest/backend/pytestworker.py

@@ -44,6 +44,16 @@ def __init__(self, writer):
         """Constructor."""
         self.writer = writer
 
+    def initialize_logreport(self):
+        """Reset accumulator variables."""
+        self.status = '---'
+        self.duration = 0
+        self.longrepr = []
+        self.sections = []
+        self.had_error = False
+        self.was_skipped = False
+        self.was_xfail = False
+
     def pytest_collectreport(self, report):
         """Called by pytest after collecting tests from a file."""
         if report.outcome == 'failed':
@@ -66,31 +76,57 @@ def pytest_runtest_logstart(self, nodeid, location):
             'event': 'starttest',
             'nodeid': nodeid
         })
+        self.initialize_logreport()
 
     def pytest_runtest_logreport(self, report):
-        """Called by pytest when a (phase of a) test is completed."""
-        if report.when in ['setup', 'teardown'] and report.outcome == 'passed':
-            return
-        data = {'event': 'logreport',
-                'when': report.when,
-                'outcome': report.outcome,
-                'nodeid': report.nodeid,
-                'sections': report.sections,
-                'duration': report.duration,
-                'filename': report.location[0],
-                'lineno': report.location[1]}
+        """Called by pytest when a phase of a test is completed."""
+        if report.when == 'call':
+            self.status = report.outcome
+            self.duration = report.duration
+        else:
+            if report.outcome == 'failed':
+                self.had_error = True
+            elif report.outcome == 'skipped':
+                self.was_skipped = True
+        if hasattr(report, 'wasxfail'):
+            self.was_xfail = True
+            self.longrepr.append(
+                report.wasxfail if report.wasxfail else 'wasxfail')
+        self.sections = report.sections  # already accumulated over phases
         if report.longrepr:
+            if hasattr(report.longrepr, 'reprcrash'):
+                self.longrepr.append(report.longrepr.reprcrash.message)
             if isinstance(report.longrepr, tuple):
-                data['longrepr'] = report.longrepr
+                self.longrepr.append(report.longrepr[2])
+            elif isinstance(report.longrepr, str):
+                self.longrepr.append(report.longrepr)
             else:
-                data['longrepr'] = str(report.longrepr)
-        if hasattr(report, 'wasxfail'):
-            data['wasxfail'] = report.wasxfail
-        if hasattr(report.longrepr, 'reprcrash'):
-            data['message'] = report.longrepr.reprcrash.message
+                self.longrepr.append(str(report.longrepr))
+
+    def pytest_runtest_logfinish(self, nodeid, location):
+        """Called by pytest when the entire test is completed."""
+        if self.was_xfail:
+            if self.status == 'passed':
+                self.status = 'xpassed'
+            else: # 'skipped'
+                self.status = 'xfailed'
+        elif self.was_skipped:
+            self.status = 'skipped'
+        data = {'event': 'logreport',
+                'outcome': self.status,
+                'witherror': self.had_error,
+                'sections': self.sections,
+                'duration': self.duration,
+                'nodeid': nodeid,
+                'filename': location[0],
+                'lineno': location[1]}
+        if self.longrepr:
+            msg_lines = self.longrepr[0].rstrip().splitlines()
+            data['message'] = msg_lines[0]
+            start_item = 1 if len(msg_lines) == 1 else 0
+            data['longrepr'] = '\n'.join(self.longrepr[start_item:])
         self.writer.write(data)
 
-
 def main(args):
     """Run pytest with the Spyder plugin."""
     if args[1] == 'file':