Fix pytest backend

spyder_unittest/backend/pytestrunner.py

@@ -133,30 +133,23 @@ def logreport_starttest_to_str(report):
 
 def logreport_to_testresult(report, config):
     """Convert a logreport sent by test process to a TestResult."""
-    if report['outcome'] == 'passed':
-        cat = Category.OK
-        status = 'ok'
-    elif report['outcome'] == 'failed':
+    status = report['outcome']
+    if report['outcome'] in ('failed', 'xpassed') or report['witherror']:
         cat = Category.FAIL
-        status = 'failure'
+    elif report['outcome'] in ('passed', 'xfailed'):
+        cat = Category.OK
     else:
         cat = Category.SKIP
-        status = report['outcome']
     testname = convert_nodeid_to_testname(report['nodeid'])
-    duration = report['duration']
-    message = report['message'] if 'message' in report else ''
-    if 'longrepr' not in report:
-        extra_text = ''
-    elif isinstance(report['longrepr'], tuple):
-        extra_text = report['longrepr'][2]
-    else:
-        extra_text = report['longrepr']
+    message = report.get('message', '')
+    extra_text = report.get('longrepr', '')
     if 'sections' in report:
+        if extra_text:
+            extra_text +=  '\n'
         for (heading, text) in report['sections']:
-            extra_text += '----- {} -----\n'.format(heading)
-            extra_text += text
+            extra_text += '----- {} -----\n{}'.format(heading, text)
     filename = osp.join(config.wdir, report['filename'])
     result = TestResult(cat, status, testname, message=message,
-                        time=duration, extra_text=extra_text,
+                        time=report['duration'], extra_text=extra_text,
                         filename=filename, lineno=report['lineno'])
     return result

spyder_unittest/backend/pytestworker.py

@@ -18,8 +18,15 @@
 import pytest
 
 # Local imports
+from spyder.config.base import get_translation
 from spyder_unittest.backend.zmqstream import ZmqStreamWriter
 
+try:
+    _ = get_translation("unittest", dirname="spyder_unittest")
+except KeyError:  # pragma: no cover
+    import gettext
+    _ = gettext.gettext
+
 
 class FileStub():
     """Stub for ZmqStreamWriter which instead writes to a file."""
@@ -44,6 +51,16 @@ def __init__(self, writer):
         """Constructor."""
         self.writer = writer
 
+    def initialize_logreport(self):
+        """Reset accumulator variables."""
+        self.status = '---'
+        self.duration = 0
+        self.longrepr = []
+        self.sections = []
+        self.had_error = False
+        self.was_skipped = False
+        self.was_xfail = False
+
     def pytest_collectreport(self, report):
         """Called by pytest after collecting tests from a file."""
         if report.outcome == 'failed':
@@ -66,31 +83,62 @@ def pytest_runtest_logstart(self, nodeid, location):
             'event': 'starttest',
             'nodeid': nodeid
         })
+        self.initialize_logreport()
 
     def pytest_runtest_logreport(self, report):
-        """Called by pytest when a (phase of a) test is completed."""
-        if report.when in ['setup', 'teardown'] and report.outcome == 'passed':
-            return
-        data = {'event': 'logreport',
-                'when': report.when,
-                'outcome': report.outcome,
-                'nodeid': report.nodeid,
-                'sections': report.sections,
-                'duration': report.duration,
-                'filename': report.location[0],
-                'lineno': report.location[1]}
+        """Called by pytest when a phase of a test is completed."""
+        if report.when == 'call':
+            self.status = report.outcome
+            self.duration = report.duration
+        else:
+            if report.outcome == 'failed':
+                self.had_error = True
+            elif report.outcome == 'skipped':
+                self.was_skipped = True
+        if hasattr(report, 'wasxfail'):
+            self.was_xfail = True
+            self.longrepr.append(report.wasxfail if report.wasxfail else _(
+                'WAS EXPECTED TO FAIL'))
+        self.sections = report.sections  # already accumulated over phases
         if report.longrepr:
+            first_msg_idx = len(self.longrepr)
+            if hasattr(report.longrepr, 'reprcrash'):
+                self.longrepr.append(report.longrepr.reprcrash.message)
             if isinstance(report.longrepr, tuple):
-                data['longrepr'] = report.longrepr
+                self.longrepr.append(report.longrepr[2])
+            elif isinstance(report.longrepr, str):
+                self.longrepr.append(report.longrepr)
             else:
-                data['longrepr'] = str(report.longrepr)
-        if hasattr(report, 'wasxfail'):
-            data['wasxfail'] = report.wasxfail
-        if hasattr(report.longrepr, 'reprcrash'):
-            data['message'] = report.longrepr.reprcrash.message
+                self.longrepr.append(str(report.longrepr))
+            if report.outcome == 'failed' and report.when in (
+                    'setup', 'teardown'):
+                self.longrepr[first_msg_idx] = '{} {}: {}'.format(
+                    _('ERROR at'), report.when, self.longrepr[first_msg_idx])
+
+    def pytest_runtest_logfinish(self, nodeid, location):
+        """Called by pytest when the entire test is completed."""
+        if self.was_xfail:
+            if self.status == 'passed':
+                self.status = 'xpassed'
+            else: # 'skipped'
+                self.status = 'xfailed'
+        elif self.was_skipped:
+            self.status = 'skipped'
+        data = {'event': 'logreport',
+                'outcome': self.status,
+                'witherror': self.had_error,
+                'sections': self.sections,
+                'duration': self.duration,
+                'nodeid': nodeid,
+                'filename': location[0],
+                'lineno': location[1]}
+        if self.longrepr:
+            msg_lines = self.longrepr[0].rstrip().splitlines()
+            data['message'] = msg_lines[0]
+            start_item = 1 if len(msg_lines) == 1 else 0
+            data['longrepr'] = '\n'.join(self.longrepr[start_item:])
         self.writer.write(data)
 
-
 def main(args):
     """Run pytest with the Spyder plugin."""
     if args[1] == 'file':

spyder_unittest/backend/tests/test_pytestrunner.py

@@ -10,6 +10,7 @@
 import os.path as osp
 
 # Third party imports
+import pytest
 from qtpy.QtCore import QByteArray
 from spyder.utils.misc import get_python_executable
 
@@ -106,8 +107,8 @@ def test_pytestrunner_process_output_with_starttest(qtbot):
 def standard_logreport_output():
     return {
         'event': 'logreport',
-        'when': 'call',
         'outcome': 'passed',
+        'witherror': False,
         'nodeid': 'foo.py::bar',
         'filename': 'foo.py',
         'lineno': 24,
@@ -120,61 +121,67 @@ def test_pytestrunner_process_output_with_logreport_passed(qtbot):
     output = [standard_logreport_output()]
     with qtbot.waitSignal(runner.sig_testresult) as blocker:
         runner.process_output(output)
-    expected = [TestResult(Category.OK, 'ok', 'foo.bar', time=42,
+    expected = [TestResult(Category.OK, 'passed', 'foo.bar', time=42,
                            filename=osp.join('ham', 'foo.py'), lineno=24)]
     assert blocker.args == [expected]
 
-def test_logreport_to_testresult_passed():
-    report = standard_logreport_output()
-    expected = TestResult(Category.OK, 'ok', 'foo.bar', time=42,
-                          filename=osp.join('ham', 'foo.py'), lineno=24)
-    assert logreport_to_testresult(report, Config(wdir='ham')) == expected
 
-def test_logreport_to_testresult_failed():
+@pytest.mark.parametrize('outcome,witherror,category', [
+    ('passed', True, Category.FAIL),
+    ('passed', False, Category.OK),
+    ('failed', True, Category.FAIL),
+    ('failed', False, Category.FAIL),
+    # ('skipped', True, this is not possible)
+    ('skipped', False, Category.SKIP),
+    ('xfailed', True, Category.FAIL),
+    ('xfailed', False, Category.OK),
+    ('xpassed', True, Category.FAIL),
+    ('xpassed', False, Category.FAIL),
+    ('---', True, Category.FAIL)
+    # ('---', False, this is not possible)
+])
+def test_logreport_to_testresult_with_outcome_and_possible_error(outcome,
+                                                                 witherror,
+                                                                 category):
     report = standard_logreport_output()
-    report['outcome'] = 'failed'
-    report['message'] = 'msg'
-    report['longrepr'] = 'exception text'
-    expected = TestResult(Category.FAIL, 'failure', 'foo.bar',
-                          message='msg', time=42, extra_text='exception text',
+    report['outcome'] = outcome
+    report['witherror'] = witherror
+    expected = TestResult(category, outcome, 'foo.bar', time=42,
                           filename=osp.join('ham', 'foo.py'), lineno=24)
     assert logreport_to_testresult(report, Config(wdir='ham')) == expected
 
-def test_logreport_to_testresult_skipped():
-    report = standard_logreport_output()
-    report['when'] = 'setup'
-    report['outcome'] = 'skipped'
-    report['longrepr'] = ('file', 24, 'skipmsg')
-    expected = TestResult(Category.SKIP, 'skipped', 'foo.bar',
-                          time=42, extra_text='skipmsg',
-                          filename=osp.join('ham', 'foo.py'), lineno=24)
-    assert logreport_to_testresult(report, Config(wdir='ham')) == expected
 
-def test_logreport_to_testresult_xfail():
+def test_logreport_to_testresult_with_message():
     report = standard_logreport_output()
-    report['outcome'] = 'skipped'
     report['message'] = 'msg'
-    report['longrepr'] = 'exception text'
-    report['wasxfail'] = ''
-    expected = TestResult(Category.SKIP, 'skipped', 'foo.bar',
-                          message='msg', time=42, extra_text='exception text',
-                          filename=osp.join('ham', 'foo.py'), lineno=24)
+    expected = TestResult(Category.OK, 'passed', 'foo.bar', message='msg',
+                          time=42, filename=osp.join('ham', 'foo.py'),
+                          lineno=24)
     assert logreport_to_testresult(report, Config(wdir='ham')) == expected
 
-def test_logreport_to_testresult_xpass():
+
+def test_logreport_to_testresult_with_extratext():
     report = standard_logreport_output()
-    report['wasxfail'] = ''
-    expected = TestResult(Category.OK, 'ok', 'foo.bar', time=42,
+    report['longrepr'] = 'long msg'
+    expected = TestResult(Category.OK, 'passed', 'foo.bar', time=42,
+                          extra_text='long msg',
                           filename=osp.join('ham', 'foo.py'), lineno=24)
     assert logreport_to_testresult(report, Config(wdir='ham')) == expected
 
-def test_logreport_to_testresult_with_output():
+
+@pytest.mark.parametrize('longrepr,prefix', [
+    ('', ''),
+    ('msg', '\n')
+])
+def test_logreport_to_testresult_with_output(longrepr, prefix):
     report = standard_logreport_output()
+    report['longrepr'] = longrepr
     report['sections'] = [['Captured stdout call', 'ham\n'],
                           ['Captured stderr call', 'spam\n']]
-    txt = ('----- Captured stdout call -----\nham\n'
+    txt = (longrepr + prefix +
+           '----- Captured stdout call -----\nham\n'
            '----- Captured stderr call -----\nspam\n')
-    expected = TestResult(Category.OK, 'ok', 'foo.bar', time=42,
+    expected = TestResult(Category.OK, 'passed', 'foo.bar', time=42,
                           extra_text=txt, filename=osp.join('ham', 'foo.py'),
                           lineno=24)
     assert logreport_to_testresult(report, Config(wdir='ham')) == expected