Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes #839: Adding disabled tests to catkin_test_results summary. #840

Closed
wants to merge 1 commit into from
Closed

Fixes #839: Adding disabled tests to catkin_test_results summary. #840

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion bin/catkin_test_results
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@ def main():
try:
results = test_results(
test_results_dir, show_verbose=args.verbose, show_all=args.all)
_, sum_errors, sum_failures = aggregate_results(results)
_, sum_errors, sum_failures, _ = aggregate_results(results)
print_summary(results, show_stable=args.all)
# Skipped tests alone should not count as a failure
if sum_errors or sum_failures:
sys.exit(1)
except Exception as e:
Expand Down
36 changes: 20 additions & 16 deletions python/catkin/test_results.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def ensure_junit_result_exist(filename):
print("Invalid XML in result file '%s' (even after trying to tidy it): %s " % (filename, str(e)), file=sys.stderr)
return False
if tree:
_, num_errors, num_failures = read_junit(filename)
_, num_errors, num_failures, _ = read_junit(filename)
if num_errors or num_failures:
return False
else:
Expand Down Expand Up @@ -110,7 +110,8 @@ def read_junit(filename):
num_tests = int(root.attrib['tests'])
num_errors = int(root.attrib['errors'])
num_failures = int(root.attrib['failures'])
return (num_tests, num_errors, num_failures)
num_skipped = int(root.get('skip', '0')) + int(root.get('disabled', '0'))
return (num_tests, num_errors, num_failures, num_skipped)


def test_results(test_results_dir, show_verbose=False, show_all=False):
Expand All @@ -120,7 +121,7 @@ def test_results(test_results_dir, show_verbose=False, show_all=False):

:param test_results_dir: str foldername
:param show_verbose: bool show output for tests which had errors or failed
:returns: dict {rel_path, (num_tests, num_errors, num_failures)}
:returns: dict {rel_path, (num_tests, num_errors, num_failures, num_skipped)}
'''
results = {}
for dirpath, dirnames, filenames in os.walk(test_results_dir):
Expand All @@ -130,12 +131,12 @@ def test_results(test_results_dir, show_verbose=False, show_all=False):
filename_abs = os.path.join(dirpath, filename)
name = filename_abs[len(test_results_dir) + 1:]
try:
num_tests, num_errors, num_failures = read_junit(filename_abs)
num_tests, num_errors, num_failures, num_skipped = read_junit(filename_abs)
except Exception as e:
if show_all:
print('Skipping "%s": %s' % (name, str(e)))
continue
results[name] = (num_tests, num_errors, num_failures)
results[name] = (num_tests, num_errors, num_failures, num_skipped)
if show_verbose and (num_errors + num_failures > 0):
print("Full test results for '%s'" % (name))
print('-------------------------------------------------')
Expand All @@ -150,17 +151,19 @@ def aggregate_results(results, callback_per_result=None):
Aggregate results

:param results: dict as from test_results()
:returns: tuple (num_tests, num_errors, num_failures)
:returns: tuple (num_tests, num_errors, num_failures, num_skipped)
"""
sum_tests = sum_errors = sum_failures = 0
sum_tests = sum_errors = sum_failures = sum_skipped = 0
for name in sorted(results.keys()):
(num_tests, num_errors, num_failures) = results[name]
(num_tests, num_errors, num_failures, num_skipped) = results[name]
sum_tests += num_tests
sum_errors += num_errors
sum_failures += num_failures
sum_skipped += num_skipped
if callback_per_result:
callback_per_result(name, num_tests, num_errors, num_failures)
return sum_tests, sum_errors, sum_failures
callback_per_result(name, num_tests, num_errors, num_failures,
num_skipped)
return sum_tests, sum_errors, sum_failures, sum_skipped


def print_summary(results, show_stable=False, show_unstable=True):
Expand All @@ -171,10 +174,11 @@ def print_summary(results, show_stable=False, show_unstable=True):
:param show_stable: print tests without failures extra
:param show_stable: print tests with failures extra
"""
def callback(name, num_tests, num_errors, num_failures):
if show_stable and not num_errors and not num_failures:
def callback(name, num_tests, num_errors, num_failures, num_skipped):
if show_stable and not num_errors and not num_failures and not num_skipped:
print('%s: %d tests' % (name, num_tests))
if show_unstable and (num_errors or num_failures):
print('%s: %d tests, %d errors, %d failures' % (name, num_tests, num_errors, num_failures))
sum_tests, sum_errors, sum_failures = aggregate_results(results, callback)
print('Summary: %d tests, %d errors, %d failures' % (sum_tests, sum_errors, sum_failures))
if show_unstable and (num_errors or num_failures or num_skipped):
print('%s: %d tests, %d errors, %d failures, %d skipped'
% (name, num_tests, num_errors, num_failures, num_skipped))
sum_tests, sum_errors, sum_failures, sum_skipped = aggregate_results(results, callback)
print('Summary: %d tests, %d errors, %d failures, %d skipped' % (sum_tests, sum_errors, sum_failures, sum_skipped))
34 changes: 23 additions & 11 deletions test/unit_tests/test_test_results.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,21 @@ def test_read_junit(self):

result_file = os.path.join(rootdir, 'test1.xml')
with open(result_file, 'w') as fhand:
fhand.write('<testsuites tests="5" failures="3" errors="1" time="35" name="AllTests"></testsuites>')
(num_tests, num_errors, num_failures) = catkin_test_results.read_junit(result_file)
self.assertEqual((5, 1, 3), (num_tests, num_errors, num_failures))
fhand.write('<testsuites tests="5" failures="3" errors="1" disabled="1" time="35" name="AllTests"></testsuites>')
(num_tests, num_errors, num_failures, num_skipped) = catkin_test_results.read_junit(result_file)
self.assertEqual((5, 1, 3, 1), (num_tests, num_errors, num_failures, num_skipped))
finally:
shutil.rmtree(rootdir)

def test_read_junit_skip(self):
try:
rootdir = tempfile.mkdtemp()

result_file = os.path.join(rootdir, 'test1.xml')
with open(result_file, 'w') as fhand:
fhand.write('<testsuites tests="5" failures="3" errors="1" skip="1" time="35" name="AllTests"></testsuites>')
(num_tests, num_errors, num_failures, num_skipped) = catkin_test_results.read_junit(result_file)
self.assertEqual((5, 1, 3, 1), (num_tests, num_errors, num_failures, num_skipped))
finally:
shutil.rmtree(rootdir)

Expand All @@ -39,9 +51,9 @@ def test_test_results(self):
for filename in ['test1.xml', 'test2.xml', 'foo.bar']:
result_file = os.path.join(rootdir, filename)
with open(result_file, 'w') as fhand:
fhand.write('<testsuites tests="5" failures="3" errors="1" time="35" name="AllTests"></testsuites>')
fhand.write('<testsuites tests="5" failures="3" errors="1" disabled="1" time="35" name="AllTests"></testsuites>')
results = catkin_test_results.test_results(rootdir)
self.assertEqual({'test1.xml': (5, 1, 3), 'test2.xml': (5, 1, 3)}, results)
self.assertEqual({'test1.xml': (5, 1, 3, 1), 'test2.xml': (5, 1, 3, 1)}, results)
finally:
shutil.rmtree(rootdir)

Expand All @@ -56,7 +68,7 @@ def test_test_results_detail(self):
with open(result_file, 'w') as fhand:
fhand.write(test_suites)
results = catkin_test_results.test_results(rootdir, show_verbose=True)
self.assertEqual({test_xml: (5, 1, 3)}, results)
self.assertEqual({test_xml: (5, 1, 3, 0)}, results)
summary = sys.stdout.getvalue()
self.assertTrue(test_xml in summary, summary)
self.assertTrue(test_suites in summary, summary)
Expand All @@ -75,7 +87,7 @@ def test_test_results_detail_with_non_ascii(self):
with open(result_file, 'w') as fhand:
fhand.write(test_suites)
results = catkin_test_results.test_results(rootdir, show_verbose=True)
self.assertEqual({test_xml: (5, 1, 3)}, results)
self.assertEqual({test_xml: (5, 1, 3, 0)}, results)
summary = sys.stdout.getvalue()
self.assertTrue(test_xml in summary, summary)
self.assertTrue(test_suites in summary, summary)
Expand All @@ -85,15 +97,15 @@ def test_test_results_detail_with_non_ascii(self):
print(summary)

def test_print_summary(self):
results = {'test1.xml': (5, 1, 3), 'test2.xml': (7, 2, 4)}
results = {'test1.xml': (5, 1, 3, 1), 'test2.xml': (7, 2, 4, 1)}
try:
oldstdout = sys.stdout
sys.stdout = StringIO()
catkin_test_results.print_summary(results)
summary = sys.stdout.getvalue()
self.assertTrue('5 tests, 1 errors, 3 failures' in summary, summary)
self.assertTrue('7 tests, 2 errors, 4 failures' in summary, summary)
self.assertTrue('12 tests, 3 errors, 7 failures' in summary, summary)
self.assertTrue('5 tests, 1 errors, 3 failures, 1 skipped' in summary, summary)
self.assertTrue('7 tests, 2 errors, 4 failures, 1 skipped' in summary, summary)
self.assertTrue('12 tests, 3 errors, 7 failures, 2 skipped' in summary, summary)

finally:
sys.stdout = oldstdout