From ce92ac21e7d31a5373fa309f111869bf56af77f8 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 5 Feb 2016 11:34:08 +0000 Subject: [PATCH 01/29] Add KiVi transport protocol support - draft --- mbed_greentea/mbed_test_api.py | 281 +++++++-------------------------- 1 file changed, 54 insertions(+), 227 deletions(-) diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index 32766ec2..4cb44836 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -19,10 +19,8 @@ import re import sys -from time import time, sleep -from Queue import Queue, Empty -from threading import Thread -from subprocess import call, Popen, PIPE +from time import time +from subprocess import call, Popen, PIPE, STDOUT from mbed_greentea.mbed_greentea_log import gt_logger @@ -66,20 +64,18 @@ "build_failed" : TEST_RESULT_BUILD_FAILED } -RE_DETECT_TESTCASE_RESULT = re.compile("\\{(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\\}") def get_test_result(output): """! Parse test 'output' data @details If test result not found returns by default TEST_RESULT_TIMEOUT value @return Returns found test result """ - result = TEST_RESULT_TIMEOUT + RE_DETECT_TESTCASE_RESULT = re.compile("\\{(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\\}") for line in "".join(output).splitlines(): search_result = RE_DETECT_TESTCASE_RESULT.search(line) if search_result and search_result.groups(): - result = TEST_RESULT_MAPPING[search_result.groups(0)[0]] - break - return result + return TEST_RESULT_MAPPING[search_result.groups(0)[0]] + return TEST_RESULT_TIMEOUT def run_host_test(image_path, @@ -118,119 +114,15 @@ def run_host_test(image_path, file name was given as switch option) """ - class StdInObserver(Thread): - """ Process used to read stdin only as console input from MUT - """ - def __init__(self): - Thread.__init__(self) - self.queue = Queue() - self.daemon = True - self.active = True - self.start() - - def run(self): - while self.active: - c = sys.stdin.read(1) - self.queue.put(c) - - def stop(self): - self.active = False - - class FileObserver(Thread): - """ process used to read file content as console input from MUT - """ - def __init__(self, filename): - Thread.__init__(self) - self.filename = filename - self.queue = Queue() - self.daemon = True - self.active = True - self.start() - - def run(self): - with open(self.filename) as f: - while self.active: - c = f.read(1) - self.queue.put(c) - - def stop(self): - self.active = False - - class ProcessObserver(Thread): - """ Default process used to observe stdout of another process as console input from MUT - """ - def __init__(self, proc): - Thread.__init__(self) - self.proc = proc - self.queue = Queue() - self.daemon = True - self.active = True - self.start() - - def run(self): - while self.active: - c = self.proc.stdout.read(1) - self.queue.put(c) - - def stop(self): - self.active = False - - # Try stopping mbed-host-test - try: - self.proc.stdin.close() - finally: - pass - - # Give 5 sec for mbedhtrun to exit - ret_code = None - for i in range(5): - ret_code = self.proc.poll() - # A None value indicates that the process hasn't terminated yet. - if ret_code is not None: - break - sleep(1) - - if ret_code is None: # Kill it - print 'Terminating mbed-host-test(mbedhtrun) process (PID %s)' % self.proc.pid - try: - self.proc.terminate() - except Exception as e: - print "ProcessObserver.stop(): %s" % str(e) - - def get_char_from_queue(obs): - """ Get character from queue safe way - """ + def run_command(cmd): + """! Runs command and prints proc stdout on screen """ try: - c = obs.queue.get(block=True, timeout=0.5) - # signals to queue job is done - obs.queue.task_done() - except Empty: - c = None - except: - raise - return c - - def filter_queue_char(c): - """ Filters out non ASCII characters from serial port - """ - if ord(c) not in range(128): - c = ' ' - return c - - def get_auto_property_value(property_name, line): - """! Scans auto detection line from MUT and returns scanned parameter 'property_name' - @details Host test case has to print additional properties for test to be set up - @return Returns string or None if property search failed - """ - result = None - if re.search("HOST: Property '%s'"% property_name, line) is not None: - property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line) - if property is not None and len(property.groups()) == 1: - result = property.groups()[0] - return result - - # Detect from where input should be taken, if no --digest switch is specified - # normal test execution can be performed + p = Popen(cmd, + stdout=PIPE, + stderr=STDOUT) + except OSError as e: + print "mbedgt: run_command(%s) ret= %d failed: "% (cmd, int(ret)), str(e), e.child_traceback + return iter(p.stdout.readline, b'') if verbose: gt_logger.gt_log("selecting test case observer...") @@ -240,124 +132,57 @@ def get_auto_property_value(property_name, line): # Select who will digest test case serial port data if digest_source == 'stdin': # When we want to scan stdin for test results - obs = StdInObserver() + raise NotImplementedError elif digest_source is not None: # When we want to open file to scan for test results - obs = FileObserver(digest_source) - else: - # Command executing CLI for host test supervisor (in detect-mode) - cmd = ["mbedhtrun", - '-d', disk, - '-p', port, - '-f', '"%s"'% image_path, - ] + raise NotImplementedError + + # Command executing CLI for host test supervisor (in detect-mode) + cmd = ["mbedhtrun", + '-d', disk, + '-p', port, + '-f', '"%s"'% image_path, + ] + + # Add extra parameters to host_test + if program_cycle_s is not None: + cmd += ["-C", str(program_cycle_s)] + if copy_method is not None: + cmd += ["-c", copy_method] + if micro is not None: + cmd += ["-m", micro] + if reset is not None: + cmd += ["-r", reset] + if reset_tout is not None: + cmd += ["-R", str(reset_tout)] + if json_test_cfg is not None: + cmd += ["--test-cfg", '"%s"' % str(json_test_cfg)] + if run_app is not None: + cmd += ["--run"] # -f stores binary name! + if enum_host_tests_path: + cmd += ["-e", '"%s"'% enum_host_tests_path] - # Add extra parameters to host_test - if program_cycle_s is not None: - cmd += ["-C", str(program_cycle_s)] - if copy_method is not None: - cmd += ["-c", copy_method] - if micro is not None: - cmd += ["-m", micro] - if reset is not None: - cmd += ["-r", reset] - if reset_tout is not None: - cmd += ["-R", str(reset_tout)] - if json_test_cfg is not None: - cmd += ["--test-cfg", '"%s"' % str(json_test_cfg)] - if run_app is not None: - cmd += ["--run"] # -f stores binary name! - if enum_host_tests_path: - cmd += ["-e", '"%s"'% enum_host_tests_path] - - if verbose: - gt_logger.gt_log_tab("calling mbedhtrun: %s"% " ".join(cmd)) - gt_logger.gt_log("mbed-host-test-runner: started") - proc = Popen(cmd, stdin=PIPE, stdout=PIPE) - obs = ProcessObserver(proc) + if verbose: + gt_logger.gt_log_tab("calling mbedhtrun: %s"% " ".join(cmd)) + gt_logger.gt_log("mbed-host-test-runner: started") - result = None - update_once_flag = {} # Stores flags checking if some auto-parameter was already set - unknown_property_count = 0 - total_duration = 20 # This for flashing, reset and other serial port operations - line = '' - output = [] + htrun_output = '' start_time = time() - while (time() - start_time) < (total_duration): - try: - c = get_char_from_queue(obs) - except Exception as e: - output.append('get_char_from_queue(obs): %s'% str(e)) - break - if c: - if verbose: - sys.stdout.write(c) - c = filter_queue_char(c) - output.append(c) - # Give the mbed under test a way to communicate the end of the test - if c in ['\n', '\r']: - - # Check for unknown property prints - # If there are too many we will stop test execution and assume test is not ported - if "HOST: Unknown property" in line: - unknown_property_count += 1 - if unknown_property_count >= max_failed_properties: - output.append('{{error}}') - break - - # Checking for auto-detection information from the test about MUT reset moment - if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line: - # We will update this marker only once to prevent multiple time resets - update_once_flag['reset_target'] = True - start_time = time() - total_duration = duration # After reset we gonna use real test case duration - - # Checking for auto-detection information from the test about timeout - auto_timeout_val = get_auto_property_value('timeout', line) - if 'timeout' not in update_once_flag and auto_timeout_val is not None: - # We will update this marker only once to prevent multiple time resets - update_once_flag['timeout'] = True - total_duration = int(auto_timeout_val) - # Detect mbed assert: - if 'mbed assertation failed: ' in line: - output.append('{{mbed_assert}}') - break - - # Check for test end. Only a '{{end}}' in the start of line indicates a test end. - # A sub string '{{end}}' may also appear in an error message. - if re.search('^\{\{end\}\}', line, re.I): - break - line = '' - else: - line += c - else: - result = TEST_RESULT_TIMEOUT - - if not '{end}' in line: - output.append('{{end}}') - - c = get_char_from_queue(obs) - - if c: - if verbose: - sys.stdout.write(c) - c = filter_queue_char(c) - output.append(c) - - # Stop test process - obs.stop() + for line in run_command(cmd): + htrun_output += line + sys.stdout.write(line) + sys.stdout.flush() end_time = time() testcase_duration = end_time - start_time # Test case duration from reset to {end} + result = get_test_result(htrun_output) + if verbose: gt_logger.gt_log("mbed-host-test-runner: stopped") - if not result: - result = get_test_result(output) - if verbose: gt_logger.gt_log("mbed-host-test-runner: returned '%s'"% result) - return (result, "".join(output), testcase_duration, duration) + return (result, "".join(htrun_output), testcase_duration, duration) def run_cli_command(cmd, shell=True, verbose=False): """! Runs command from command line @@ -373,11 +198,13 @@ def run_cli_command(cmd, shell=True, verbose=False): result = False if verbose: print "mbedgt: [ret=%d] Command: %s"% (int(ret), cmd) - except Exception as e: + except OSError as e: result = False if verbose: print "mbedgt: [ret=%d] Command: %s"% (int(ret), cmd) print str(e) + print "mbedgt: traceback..." + print e.child_traceback return (result, ret) def run_cli_process(cmd): From 63b5bcdb1452753402ce10952bbba5323184d5f7 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 5 Feb 2016 14:59:29 +0000 Subject: [PATCH 02/29] Polishing --- mbed_greentea/mbed_test_api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index 4cb44836..dc644a3d 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -70,9 +70,9 @@ def get_test_result(output): @details If test result not found returns by default TEST_RESULT_TIMEOUT value @return Returns found test result """ - RE_DETECT_TESTCASE_RESULT = re.compile("\\{(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\\}") + re_detect = re.compile("\\{(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\\}") for line in "".join(output).splitlines(): - search_result = RE_DETECT_TESTCASE_RESULT.search(line) + search_result = re_detect.search(line) if search_result and search_result.groups(): return TEST_RESULT_MAPPING[search_result.groups(0)[0]] return TEST_RESULT_TIMEOUT @@ -121,7 +121,8 @@ def run_command(cmd): stdout=PIPE, stderr=STDOUT) except OSError as e: - print "mbedgt: run_command(%s) ret= %d failed: "% (cmd, int(ret)), str(e), e.child_traceback + print "mbedgt: run_command(%s) ret= %d failed: %s"% (str(cmd), + str(e), e.child_traceback) return iter(p.stdout.readline, b'') if verbose: From 0b1cc56b0aec348f8285e8705151a1a2adc677fa Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 5 Feb 2016 15:17:55 +0000 Subject: [PATCH 03/29] Removed parallel tests Parallel tests are incompatible with new htrun --- test/mbed_gt_test_parallel.py | 238 ---------------------------------- 1 file changed, 238 deletions(-) delete mode 100644 test/mbed_gt_test_parallel.py diff --git a/test/mbed_gt_test_parallel.py b/test/mbed_gt_test_parallel.py deleted file mode 100644 index 4551bd75..00000000 --- a/test/mbed_gt_test_parallel.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright 2015 ARM Limited, All rights reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -import time - -from random import uniform -from mock import patch -from mbed_greentea import mbed_greentea_cli - -class TestmbedGt(unittest.TestCase): - """! Mbed greentea parallelisation tests - """ - def setUp(self): - """ - Called before test function - :return: - """ - pass - - def tearDown(self): - """ - Called after test function - :return: - """ - pass - - @patch('mbed_greentea.mbed_greentea_cli.optparse.OptionParser') - @patch('mbed_greentea.mbed_greentea_cli.load_ctest_testsuite') - @patch('mbed_greentea.mbed_greentea_cli.mbed_lstools.create') - @patch('mbed_greentea.mbed_test_api.Popen') - def test_basic_parallel_execution(self, popen_mock, mbedLstools_mock, loadCtestTestsuite_mock, optionParser_mock): - #runHostTest_mock.side_effect = run_host_test_mock - popen_mock.side_effect = PopenMock - mbedLstools_mock.side_effect = MbedsMock - loadCtestTestsuite_mock.return_value = load_ctest_testsuite_mock() - - my_gt_opts = GtOptions(list_of_targets="frdm-k64f-gcc", - parallel_test_exec=3, - test_by_names="mbed-drivers-test-stdio", - use_target_ids="02400203A0811E505D7DE3D9", - report_junit_file_name="junitTestReport") - OptionParserMock.static_options = my_gt_opts - optionParser_mock.side_effect = OptionParserMock - - mbed_greentea_cli.main() - -class PopenMock: - def __init__(self, *args, **kwargs): - self.stdout = StdOutMock() - self.stdin = StdOutMock() - - def communicate(self): - return "_stdout", "_stderr" - - def returncode(self): - return 0 - - def terminate(self): - pass - - def poll(self): - return 0 - - -class StdOutMock: - def __init__(self): - self.str = """MBED: Instrumentation: 'COM11' and disk: 'E:'\nHOST: Copy image onto target... -\t1 file(s) copied. -HOST: Initialize serial port... -...port ready! -HOST: Reset target... -HOST: Detecting test case properties... -HOST: Property 'timeout' = '20' -HOST: Property 'host_test_name' = 'echo' -HOST: Property 'description' = 'serial interrupt test' -HOST: Property 'test_id' = 'MBED_14' -HOST: Start test... -...port ready! -HOST: Starting the ECHO test -.................................................. -{{success}} -{{end}}""" - self.offset = 0 - - def read(self, size): - if self.offset < len(self.str): - ret = [self.str[i] for i in range(self.offset, self.offset + size)] - self.offset += size - return ''.join(ret) - else: - time.sleep(uniform(0.1, 2)) - self.offset = 0 - return None - - def close(self): - pass - - -def run_host_test_mock(*args, **kwargs): - random_testduration = uniform(0.1, 2) - time.sleep(random_testduration) - return ('OK', 'single_test_output', random_testduration, 10) - - -def load_ctest_testsuite_mock(): - return {'mbed-drivers-test-echo': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-echo.bin', - 'mbed-drivers-test-time_us': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-time_us.bin', - 'mbed-drivers-test-serial_interrupt': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-serial_interrupt.bin', - 'mbed-drivers-test-blinky': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-blinky.bin', - 'mbed-drivers-test-functionpointer': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-functionpointer.bin', - 'mbed-drivers-test-stdio': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-stdio.bin', - 'mbed-drivers-test-eventhandler': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-eventhandler.bin', - 'mbed-drivers-test-stl': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-stl.bin', - 'mbed-drivers-test-div': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-div.bin', - 'mbed-drivers-test-rtc': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-rtc.bin', - 'mbed-drivers-test-cstring': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-cstring.bin', - 'mbed-drivers-test-cpp': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-cpp.bin', - 'mbed-drivers-test-timeout': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-timeout.bin', - 'mbed-drivers-test-ticker_3': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-ticker_3.bin', - 'mbed-drivers-test-ticker_2': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-ticker_2.bin', - 'mbed-drivers-test-heap_and_stack': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-heap_and_stack.bin', - 'mbed-drivers-test-hello': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-hello.bin', - 'mbed-drivers-test-ticker': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-ticker.bin', - 'mbed-drivers-test-dev_null': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-dev_null.bin', - 'mbed-drivers-test-basic': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-basic.bin', - 'mbed-drivers-test-asynch_spi': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-asynch_spi.bin', - 'mbed-drivers-test-sleep_timeout': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-sleep_timeout.bin', - 'mbed-drivers-test-detect': '.\\build\\frdm-k64f-gcc\\test\\mbed-drivers-test-detect.bin'} - - -class GtOptions: - def __init__(self, - list_of_targets, - test_by_names=None, - skip_test=None, - only_build_tests=False, - skip_yotta_build=True, - copy_method=None, - parallel_test_exec=0, - verbose_test_configuration_only=False, - build_to_release=False, - build_to_debug=False, - list_binaries=False, - map_platform_to_yt_target={}, - use_target_ids=False, - lock_by_target=False, - digest_source=None, - json_test_configuration=None, - run_app=None, - report_junit_file_name=None, - report_text_file_name=None, - report_json=False, - report_fails=False, - verbose_test_result_only=False, - enum_host_tests=None, - yotta_search_for_mbed_target=False, - plain=False, - shuffle_test_order=False, - shuffle_test_seed=None, - verbose=True, - version=False): - - self.list_of_targets = list_of_targets - self.test_by_names = test_by_names - self.skip_test = skip_test - self.only_build_tests = only_build_tests - self.skip_yotta_build = skip_yotta_build - self.copy_method = copy_method - self.parallel_test_exec = parallel_test_exec - self.verbose_test_configuration_only = verbose_test_configuration_only - self.build_to_release = build_to_release - self.build_to_debug = build_to_debug - self.list_binaries = list_binaries - self.map_platform_to_yt_target = map_platform_to_yt_target - self.use_target_ids = use_target_ids - self.lock_by_target = lock_by_target - self.digest_source = digest_source - self.json_test_configuration = json_test_configuration - self.run_app = run_app - self.report_junit_file_name = report_junit_file_name - self.report_text_file_name = report_text_file_name - self.report_json = report_json - self.report_fails = report_fails - self.verbose_test_result_only = verbose_test_result_only - self.enum_host_tests = enum_host_tests - self.yotta_search_for_mbed_target = yotta_search_for_mbed_target - self.plain = plain - self.shuffle_test_order = shuffle_test_order - self.shuffle_test_seed = shuffle_test_seed - self.verbose = verbose - self.version = version - - -class OptionParserMock: - static_options = {} - - def __init__(self): - pass - - def add_option(self, *args, **kwargs): - pass - - def parse_args(self): - return (OptionParserMock.static_options, []) - - -class MbedsMock: - def __init__(self): - pass - - def list_mbeds(self): - return [{'target_id_mbed_htm': '02400203A0811E505D7DE3E8', 'mount_point': 'E:', 'target_id': '02400203A0811E505D7DE3E8', 'serial_port': u'COM11', 'target_id_usb_id': '02400203A0811E505D7DE3E8', 'platform_name': 'K64F'}, - {'target_id_mbed_htm': '02400203A0811E505D7DE3D9', 'mount_point': 'F:', 'target_id': '02400203A0811E505D7DE3D9', 'serial_port': u'COM12', 'target_id_usb_id': '02400203A0811E505D7DE3D9', 'platform_name': 'K64F'}] - - def list_mbeds_ext(self): - return [{'target_id_mbed_htm': '02400203A0811E505D7DE3E8', 'mount_point': 'E:', 'target_id': '02400203A0811E505D7DE3E8', 'serial_port': u'COM11', 'target_id_usb_id': '02400203A0811E505D7DE3E8', 'platform_name': 'K64F', 'platform_name_unique': 'K64F[0]'}, - {'target_id_mbed_htm': '02400203A0811E505D7DE3D9', 'mount_point': 'F:', 'target_id': '02400203A0811E505D7DE3D9', 'serial_port': u'COM12', 'target_id_usb_id': '02400203A0811E505D7DE3D9', 'platform_name': 'K64F', 'platform_name_unique': 'K64F[1]'}, - {'target_id_mbed_htm': '02400203A0811E505D7DE3A7', 'mount_point': 'G:', 'target_id': '02400203A0811E505D7DE3A7', 'serial_port': u'COM13', 'target_id_usb_id': '02400203A0811E505D7DE3A7', 'platform_name': 'K64F', 'platform_name_unique': 'K64F[2]'}, - {'target_id_mbed_htm': '09400203A0811E505D7DE3B6', 'mount_point': 'H:', 'target_id': '09400203A0811E505D7DE3B6', 'serial_port': u'COM14', 'target_id_usb_id': '09400203A0811E505D7DE3B6', 'platform_name': 'K100F', 'platform_name_unique': 'K100F[0]'}] - - def list_platforms_ext(self): - return {'K64F': 2} - - -if __name__ == '__main__': - unittest.main() From d24242cce81e98a08d3ef452d9f07fdcc6318f49 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 5 Feb 2016 16:47:41 +0000 Subject: [PATCH 04/29] Merge Test Case reporting functionality - draft --- mbed_greentea/mbed_greentea_cli.py | 48 +++++++++++++------ mbed_greentea/mbed_report_api.py | 76 +++++++++++++++++++++++++++++- mbed_greentea/mbed_test_api.py | 44 ++++++++++++++++- 3 files changed, 150 insertions(+), 18 deletions(-) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 11f57ff2..7f7dc827 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -34,6 +34,7 @@ from mbed_greentea.cmake_handlers import load_ctest_testsuite from mbed_greentea.cmake_handlers import list_binaries_for_targets from mbed_greentea.mbed_report_api import exporter_text +from mbed_greentea.mbed_report_api import exporter_testcase_text from mbed_greentea.mbed_report_api import exporter_json from mbed_greentea.mbed_report_api import exporter_junit from mbed_greentea.mbed_target_info import get_mbed_clasic_target_info @@ -372,25 +373,37 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta enum_host_tests_path=enum_host_tests_path, verbose=verbose) - single_test_result, single_test_output, single_testduration, single_timeout = host_test_result + single_test_result, single_test_output, single_testduration, single_timeout, result_test_cases = host_test_result test_result = single_test_result if single_test_result != TEST_RESULT_OK: test_exec_retcode += 1 # Update report for optional reporting feature - test_name = test['test_bin'].lower() + test_suite_name = test['test_bin'].lower() if yotta_target_name not in test_report: test_report[yotta_target_name] = {} - if test_name not in test_report[yotta_target_name]: - test_report[yotta_target_name][test_name] = {} - test_report[yotta_target_name][test_name]['single_test_result'] = single_test_result - test_report[yotta_target_name][test_name]['single_test_output'] = single_test_output - test_report[yotta_target_name][test_name]['elapsed_time'] = single_testduration - test_report[yotta_target_name][test_name]['platform_name'] = micro - test_report[yotta_target_name][test_name]['copy_method'] = copy_method + if test_suite_name not in test_report[yotta_target_name]: + test_report[yotta_target_name][test_suite_name] = {} - gt_logger.gt_log("test on hardware with target id: %s \n\ttest '%s' %s %s in %.2f sec"% (mut['target_id'], test['test_bin'], '.' * (80 - len(test['test_bin'])), test_result, single_testduration)) + test_report[yotta_target_name][test_suite_name]['single_test_result'] = single_test_result + test_report[yotta_target_name][test_suite_name]['single_test_output'] = single_test_output + test_report[yotta_target_name][test_suite_name]['elapsed_time'] = single_testduration + test_report[yotta_target_name][test_suite_name]['platform_name'] = micro + test_report[yotta_target_name][test_suite_name]['copy_method'] = copy_method + test_report[yotta_target_name][test_suite_name]['testcase_result'] = result_test_cases + + gt_logger.gt_log("test on hardware with target id: %s"% (mut['target_id'])) + gt_logger.gt_log("test suite '%s' %s %s in %.2f sec"% (test['test_bin'], + '.' * (80 - len(test['test_bin'])), + test_result, + single_testduration)) + + for tc_name in sorted(result_test_cases.keys()): + gt_logger.gt_log_tab("test case '%s' %s %s in %.2f sec"% (tc_name, + '.' * (81 - len(tc_name)), + result_test_cases[tc_name]['result_text'], + result_test_cases[tc_name]['duration'])) if single_test_result != 'OK' and not verbose and opts.report_fails: # In some cases we want to print console to see why test failed @@ -442,7 +455,7 @@ def main_cli(opts, args, gt_instance_uuid=None): enum_host_tests_path=enum_host_tests_path, verbose=opts.verbose_test_result_only) - single_test_result, single_test_output, single_testduration, single_timeout = host_test_result + single_test_result, single_test_output, single_testduration, single_timeout, result_test_cases = host_test_result status = TEST_RESULTS.index(single_test_result) if single_test_result in TEST_RESULTS else -1 return (status) @@ -644,7 +657,7 @@ def main_cli(opts, args, gt_instance_uuid=None): enum_host_tests_path=enum_host_tests_path, verbose=True) - single_test_result, single_test_output, single_testduration, single_timeout = host_test_result + single_test_result, single_test_output, single_testduration, single_timeout, result_test_cases = host_test_result status = TEST_RESULTS.index(single_test_result) if single_test_result in TEST_RESULTS else -1 if single_test_result != TEST_RESULT_OK: test_exec_retcode += 1 @@ -750,6 +763,7 @@ def main_cli(opts, args, gt_instance_uuid=None): # Reports (to file) if opts.report_junit_file_name: + gt_logger.gt_log("exporting to JUnit file '%s'..."% gt_logger.gt_bright(opts.report_junit_file_name)) junit_report = exporter_junit(test_report) with open(opts.report_junit_file_name, 'w') as f: f.write(junit_report) @@ -767,10 +781,16 @@ def main_cli(opts, args, gt_instance_uuid=None): else: # Final summary if test_report: - gt_logger.gt_log("test report:") + # Test suite report + gt_logger.gt_log("test suite report:") text_report, text_results = exporter_text(test_report) print text_report - gt_logger.gt_log("results: " + text_results) + gt_logger.gt_log("test suite results: " + text_results) + # test case detailed report + gt_logger.gt_log("test case report:") + text_testcase_report, text_testcase_results = exporter_testcase_text(test_report) + print text_testcase_report + gt_logger.gt_log("test case results: " + text_testcase_results) # This flag guards 'build only' so we expect only yotta errors if test_platforms_match == 0: diff --git a/mbed_greentea/mbed_report_api.py b/mbed_greentea/mbed_report_api.py index 45a29b1d..3090c82e 100644 --- a/mbed_greentea/mbed_report_api.py +++ b/mbed_greentea/mbed_report_api.py @@ -18,6 +18,22 @@ """ +def export_to_file(file_name, payload): + """! Simple file dump used to store reports on disk + @param file_name Report file name (with path if needed) + @param payload Data to store inside file + @return True if report save was successful + """ + result = True + try: + with open(file_name, 'w') as f: + f.write(payload) + except IOError as e: + print "Exporting report to file failed: ", str(e) + result = False + return result + + def exporter_junit(test_result_ext, test_suite_properties=None): """! Export test results in JUnit XML compliant format @details This function will import junit_xml library to perform report conversion @@ -69,13 +85,13 @@ def exporter_text(test_result_ext, test_suite_properties=None): """ from prettytable import PrettyTable #TODO: export to text, preferably to PrettyTable (SQL like) format - cols = ['target', 'platform_name', 'test', 'result', 'elapsed_time (sec)', 'copy_method'] + cols = ['target', 'platform_name', 'test suite', 'result', 'elapsed_time (sec)', 'copy_method'] pt = PrettyTable(cols) for col in cols: pt.align[col] = "l" pt.padding_width = 1 # One space between column edges and contents (default) - result_dict = {} # Used to print mbed 2.0 test result like short summary + result_dict = {} # Used to print test suite results for target_name in sorted(test_result_ext): test_results = test_result_ext[target_name] @@ -101,3 +117,59 @@ def exporter_text(test_result_ext, test_suite_properties=None): result_pt = pt.get_string() result_res = ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()]) return result_pt, result_res + +def exporter_testcase_text(test_result_ext, test_suite_properties=None): + """! Exports test case results to text formatted output + @details This is a human friendly format + @return Tuple with table of results and result quantity summary string + """ + from prettytable import PrettyTable + #TODO: export to text, preferably to PrettyTable (SQL like) format + cols = ['target', 'platform_name', 'test suite', 'test case', 'result', 'elapsed_time (sec)', 'copy_method'] + pt = PrettyTable(cols) + for col in cols: + pt.align[col] = "l" + pt.padding_width = 1 # One space between column edges and contents (default) + + result_testcase_dict = {} # Used to print test case results + + for target_name in test_result_ext: + test_results = test_result_ext[target_name] + row = [] + for test_suite_name in test_results: + test = test_results[test_suite_name] + + # testcase_result stores info about test case results + testcase_result = test['testcase_result'] + # "testcase_result": { + # "STRINGS004": { + # "duration": 0.009999990463256836, + # "time_start": 1453073018.275, + # "time_end": 1453073018.285, + # "result": 1 + # }, + + for tc_name in sorted(testcase_result.keys()): + duration = testcase_result[tc_name]['duration'] + result = int(testcase_result[tc_name]['result']) + result_text = testcase_result[tc_name]['result_text'] + + # Grab quantity of each test result + if result_text in result_testcase_dict: + result_testcase_dict[result_text] += 1 + else: + result_testcase_dict[result_text] = 1 + + row.append(target_name) + row.append(test['platform_name']) + row.append(test_suite_name) + row.append(tc_name) + row.append(result_text) + row.append(round(duration, 2)) + row.append(test['copy_method']) + pt.add_row(row) + row = [] + + result_pt = pt.get_string() + result_res = ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_testcase_dict.items() if v != 0}.iteritems()]) + return result_pt, result_res diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index dc644a3d..a045286d 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -94,7 +94,7 @@ def run_host_test(image_path, enum_host_tests_path=None, run_app=None): """! This function runs host test supervisor (executes mbedhtrun) and checks output from host test process. - @return Tuple with test results, test output and test duration times + @return Tuple with test results, test output, test duration times and test case results @param image_path Path to binary file for flashing @param disk Currently mounted mbed-enabled devices disk (mount point) @param port Currently mounted mbed-enabled devices serial port (console) @@ -179,11 +179,51 @@ def run_command(cmd): testcase_duration = end_time - start_time # Test case duration from reset to {end} result = get_test_result(htrun_output) + result_test_cases = get_testcase_result(htrun_output) if verbose: gt_logger.gt_log("mbed-host-test-runner: stopped") gt_logger.gt_log("mbed-host-test-runner: returned '%s'"% result) - return (result, "".join(htrun_output), testcase_duration, duration) + return (result, "".join(htrun_output), testcase_duration, duration, result_test_cases) + +def get_testcase_result(output): + result_test_cases = {} # Test cases results + re_tc_start = re.compile("^\{\{(__testcase_start);(\w+)\}") + re_tc_finish = re.compile("^\{\{(__testcase_finish);(\w+);(\d+)\}") + + for line in output.split(): + m = re_tc_start.search(line) + if m and len(m.groups()) == 2: + # m1.group(1) == "__testcase_start" + testcase_id = m.group(2) # Test Case ID + if testcase_id not in result_test_cases: + result_test_cases[testcase_id] = {} + result_test_cases[testcase_id]['time_start'] = time() + + m = re_tc_finish.search(line) + if m and len(m.groups()) == 3: + # m.group(1) == "testcase_start" + testcase_id = m.group(2) # Test Case ID + testcase_result = int(m.group(3)) # success code, 0 == success, <0 inconclusive, >0 FAILure + if testcase_id not in result_test_cases: + result_test_cases[testcase_id] = {} + # Setting some info about test case itself + result_test_cases[testcase_id]['time_end'] = time() + result_test_cases[testcase_id]['result'] = testcase_result + + result_test_cases[testcase_id]['result_text'] = 'OK' + if testcase_result > 0: + result_test_cases[testcase_id]['result_text'] = 'FAIL' + elif testcase_result < 0: + result_test_cases[testcase_id]['result_text'] = 'ERROR' + + result_test_cases[testcase_id]['duration'] = 0.0 + if 'time_start' in result_test_cases[testcase_id]: + result_test_cases[testcase_id]['duration'] = result_test_cases[testcase_id]['time_end'] - result_test_cases[testcase_id]['time_start'] + else: + result_test_cases[testcase_id]['duration'] = 0.0 + + return result_test_cases def run_cli_command(cmd, shell=True, verbose=False): """! Runs command from command line From fca7efcfb71191581a3ef18b5dd6d20b7e34fbeb Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 5 Feb 2016 16:51:11 +0000 Subject: [PATCH 05/29] Remove copy_method from test case report --- mbed_greentea/mbed_report_api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mbed_greentea/mbed_report_api.py b/mbed_greentea/mbed_report_api.py index 3090c82e..b6adf9ad 100644 --- a/mbed_greentea/mbed_report_api.py +++ b/mbed_greentea/mbed_report_api.py @@ -125,7 +125,7 @@ def exporter_testcase_text(test_result_ext, test_suite_properties=None): """ from prettytable import PrettyTable #TODO: export to text, preferably to PrettyTable (SQL like) format - cols = ['target', 'platform_name', 'test suite', 'test case', 'result', 'elapsed_time (sec)', 'copy_method'] + cols = ['target', 'platform_name', 'test suite', 'test case', 'result', 'elapsed_time (sec)'] pt = PrettyTable(cols) for col in cols: pt.align[col] = "l" @@ -166,7 +166,6 @@ def exporter_testcase_text(test_result_ext, test_suite_properties=None): row.append(tc_name) row.append(result_text) row.append(round(duration, 2)) - row.append(test['copy_method']) pt.add_row(row) row = [] From 71149d395456774f41f2d3e81c870f9266dbabb2 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 5 Feb 2016 17:53:06 +0000 Subject: [PATCH 06/29] Ported Test Case reporting functionality --- mbed_greentea/mbed_test_api.py | 36 ++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index a045286d..52219a72 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -70,11 +70,15 @@ def get_test_result(output): @details If test result not found returns by default TEST_RESULT_TIMEOUT value @return Returns found test result """ - re_detect = re.compile("\\{(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\\}") - for line in "".join(output).splitlines(): + re_detect = re.compile(r"\{result;(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\}") + + for line in output.split(): search_result = re_detect.search(line) - if search_result and search_result.groups(): - return TEST_RESULT_MAPPING[search_result.groups(0)[0]] + if search_result: + if search_result.group(1) in TEST_RESULT_MAPPING: + return TEST_RESULT_MAPPING[search_result.group(1)] + else: + return TEST_RESULT_UNDEF return TEST_RESULT_TIMEOUT @@ -188,30 +192,28 @@ def run_command(cmd): def get_testcase_result(output): result_test_cases = {} # Test cases results - re_tc_start = re.compile("^\{\{(__testcase_start);(\w+)\}") - re_tc_finish = re.compile("^\{\{(__testcase_finish);(\w+);(\d+)\}") + re_tc_start = re.compile(r"^\[(\d+\.\d+)\][^\{]+\{\{(__testcase_start);(\w+)\}\}") + re_tc_finish = re.compile(r"^\[(\d+\.\d+)\][^\{]+\{\{(__testcase_finish);(\w+);(\d+)\}\}") - for line in output.split(): + for line in output.splitlines(): m = re_tc_start.search(line) - if m and len(m.groups()) == 2: - # m1.group(1) == "__testcase_start" - testcase_id = m.group(2) # Test Case ID + if m: + timestamp, _, testcase_id = m.groups() if testcase_id not in result_test_cases: result_test_cases[testcase_id] = {} - result_test_cases[testcase_id]['time_start'] = time() + result_test_cases[testcase_id]['time_start'] = float(timestamp) m = re_tc_finish.search(line) - if m and len(m.groups()) == 3: - # m.group(1) == "testcase_start" - testcase_id = m.group(2) # Test Case ID - testcase_result = int(m.group(3)) # success code, 0 == success, <0 inconclusive, >0 FAILure + if m: + timestamp, _, testcase_id, testcase_result = m.groups() + testcase_result = int(testcase_result) if testcase_id not in result_test_cases: result_test_cases[testcase_id] = {} # Setting some info about test case itself - result_test_cases[testcase_id]['time_end'] = time() + result_test_cases[testcase_id]['time_end'] = float(timestamp) result_test_cases[testcase_id]['result'] = testcase_result - result_test_cases[testcase_id]['result_text'] = 'OK' + # Assign human readable test case result if testcase_result > 0: result_test_cases[testcase_id]['result_text'] = 'FAIL' elif testcase_result < 0: From 9de37ededdab17abdc74ad278a63c98742b398f9 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 5 Feb 2016 18:22:27 +0000 Subject: [PATCH 07/29] Polishing --- mbed_greentea/mbed_greentea_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 7f7dc827..8ae9d007 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -402,8 +402,8 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta for tc_name in sorted(result_test_cases.keys()): gt_logger.gt_log_tab("test case '%s' %s %s in %.2f sec"% (tc_name, '.' * (81 - len(tc_name)), - result_test_cases[tc_name]['result_text'], - result_test_cases[tc_name]['duration'])) + result_test_cases[tc_name].get('result_text', '_'), + result_test_cases[tc_name].get('duration', 0.0))) if single_test_result != 'OK' and not verbose and opts.report_fails: # In some cases we want to print console to see why test failed From 10d500798b1bc28f9c705f18d1257d95f3494823 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Mon, 8 Feb 2016 08:17:47 +0000 Subject: [PATCH 08/29] Update unit tests --- mbed_greentea/mbed_test_api.py | 2 +- test/mbed_gt_test_api.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index 52219a72..0f5f39dc 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -70,7 +70,7 @@ def get_test_result(output): @details If test result not found returns by default TEST_RESULT_TIMEOUT value @return Returns found test result """ - re_detect = re.compile(r"\{result;(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\}") + re_detect = re.compile(r"\{result;([\w+_]*)\}") for line in output.split(): search_result = re_detect.search(line) diff --git a/test/mbed_gt_test_api.py b/test/mbed_gt_test_api.py index 577a298a..fc8ec431 100644 --- a/test/mbed_gt_test_api.py +++ b/test/mbed_gt_test_api.py @@ -38,7 +38,7 @@ def setUp(self): MBED: Test ID 'Unknown' MBED: UUID 'Unknown' -{{failure}} +{{result;failure}} {{end}} """ @@ -54,7 +54,7 @@ def setUp(self): HOST: Property 'description' = 'Basic' HOST: Property 'test_id' = 'MBED_A1' HOST: Start test... -{{success}} +{{result;success}} {{end}} """ @@ -68,6 +68,7 @@ def setUp(self): HOST: Initialize serial port... ...port ready! HOST: Reset target... +{{result;some_random_value} {{end}} """ From 470c733e9f9acd7c3a4008f71f154b72b837a2b6 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 9 Feb 2016 09:45:10 +0000 Subject: [PATCH 09/29] Add solution for extra new-line when dumping app output to file --- mbed_greentea/mbed_test_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index 0f5f39dc..c2ef7a1d 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -176,7 +176,9 @@ def run_command(cmd): for line in run_command(cmd): htrun_output += line - sys.stdout.write(line) + # When dumping output to file both \r and \n will be a new line + # To avoid this "extra new-line" we only use \n at the end + sys.stdout.write(line.rstrip() + '\n') sys.stdout.flush() end_time = time() From dcfd047ea4074d3dd74da11103fdc73df72812f7 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Wed, 10 Feb 2016 16:10:29 +0000 Subject: [PATCH 10/29] Test case report fixes --- mbed_greentea/mbed_greentea_cli.py | 4 ++-- mbed_greentea/mbed_report_api.py | 12 ++++++++---- mbed_greentea/mbed_test_api.py | 22 ++++++++++++++++------ 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 8ae9d007..12f96062 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -400,8 +400,8 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta single_testduration)) for tc_name in sorted(result_test_cases.keys()): - gt_logger.gt_log_tab("test case '%s' %s %s in %.2f sec"% (tc_name, - '.' * (81 - len(tc_name)), + gt_logger.gt_log_tab("test case: '%s' %s %s in %.2f sec"% (tc_name, + '.' * (80 - len(tc_name)), result_test_cases[tc_name].get('result_text', '_'), result_test_cases[tc_name].get('duration', 0.0))) diff --git a/mbed_greentea/mbed_report_api.py b/mbed_greentea/mbed_report_api.py index b6adf9ad..ae95a557 100644 --- a/mbed_greentea/mbed_report_api.py +++ b/mbed_greentea/mbed_report_api.py @@ -125,7 +125,7 @@ def exporter_testcase_text(test_result_ext, test_suite_properties=None): """ from prettytable import PrettyTable #TODO: export to text, preferably to PrettyTable (SQL like) format - cols = ['target', 'platform_name', 'test suite', 'test case', 'result', 'elapsed_time (sec)'] + cols = ['target', 'platform_name', 'test suite', 'test case', 'passed', 'failed', 'result', 'elapsed_time (sec)'] pt = PrettyTable(cols) for col in cols: pt.align[col] = "l" @@ -150,9 +150,11 @@ def exporter_testcase_text(test_result_ext, test_suite_properties=None): # }, for tc_name in sorted(testcase_result.keys()): - duration = testcase_result[tc_name]['duration'] - result = int(testcase_result[tc_name]['result']) - result_text = testcase_result[tc_name]['result_text'] + duration = testcase_result[tc_name].get('duration', 0.0) + result = testcase_result[tc_name].get('result', 0) + passed = testcase_result[tc_name].get('passed', 0) + failed = testcase_result[tc_name].get('failed', 0) + result_text = testcase_result[tc_name].get('result_text', "UNDEF") # Grab quantity of each test result if result_text in result_testcase_dict: @@ -164,6 +166,8 @@ def exporter_testcase_text(test_result_ext, test_suite_properties=None): row.append(test['platform_name']) row.append(test_suite_name) row.append(tc_name) + row.append(passed) + row.append(failed) row.append(result_text) row.append(round(duration, 2)) pt.add_row(row) diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index c2ef7a1d..f2d1a338 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -194,8 +194,8 @@ def run_command(cmd): def get_testcase_result(output): result_test_cases = {} # Test cases results - re_tc_start = re.compile(r"^\[(\d+\.\d+)\][^\{]+\{\{(__testcase_start);(\w+)\}\}") - re_tc_finish = re.compile(r"^\[(\d+\.\d+)\][^\{]+\{\{(__testcase_finish);(\w+);(\d+)\}\}") + re_tc_start = re.compile(r"^\[(\d+\.\d+)\][^\{]+\{\{(__testcase_start);([^;]+)\}\}") + re_tc_finish = re.compile(r"^\[(\d+\.\d+)\][^\{]+\{\{(__testcase_finish);([^;]+);(\d+);(\d+)\}\}") for line in output.splitlines(): m = re_tc_start.search(line) @@ -204,24 +204,34 @@ def get_testcase_result(output): if testcase_id not in result_test_cases: result_test_cases[testcase_id] = {} result_test_cases[testcase_id]['time_start'] = float(timestamp) + continue m = re_tc_finish.search(line) if m: - timestamp, _, testcase_id, testcase_result = m.groups() - testcase_result = int(testcase_result) + timestamp, _, testcase_id, testcase_passed, testcase_failed = m.groups() + + testcase_passed = int(testcase_passed) + testcase_failed = int(testcase_failed) + + testcase_result = 0 # OK case + if testcase_failed != 0: + testcase_result = testcase_failed # testcase_result > 0 is FAILure + if testcase_id not in result_test_cases: result_test_cases[testcase_id] = {} # Setting some info about test case itself + result_test_cases[testcase_id]['duration'] = 0.0 + result_test_cases[testcase_id]['result_text'] = 'OK' result_test_cases[testcase_id]['time_end'] = float(timestamp) + result_test_cases[testcase_id]['passed'] = testcase_passed + result_test_cases[testcase_id]['failed'] = testcase_failed result_test_cases[testcase_id]['result'] = testcase_result - result_test_cases[testcase_id]['result_text'] = 'OK' # Assign human readable test case result if testcase_result > 0: result_test_cases[testcase_id]['result_text'] = 'FAIL' elif testcase_result < 0: result_test_cases[testcase_id]['result_text'] = 'ERROR' - result_test_cases[testcase_id]['duration'] = 0.0 if 'time_start' in result_test_cases[testcase_id]: result_test_cases[testcase_id]['duration'] = result_test_cases[testcase_id]['time_end'] - result_test_cases[testcase_id]['time_start'] else: From 7210f004044efa49c766169a4d518f590a664f21 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Thu, 11 Feb 2016 09:48:33 +0000 Subject: [PATCH 11/29] Force pySerial v3 usage --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bd5550d2..4db3842f 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def read(fname): "console_scripts": ["mbedgt=mbed_greentea.mbed_greentea_cli:main",], }, install_requires=["PrettyTable>=0.7.2", - "PySerial>=2.7", + "PySerial>=3.0", "mbed-host-tests>=0.1.18", "mbed-ls", "junit-xml", From c1219b21233aa8e55bec66811ca622ee6632ec5a Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Thu, 11 Feb 2016 09:49:00 +0000 Subject: [PATCH 12/29] v0.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4db3842f..4406f008 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() setup(name='mbed-greentea', - version='0.1.18', + version='0.2.0', description=DESCRIPTION, long_description=read('README.md'), author=OWNER_NAMES, From 151734f6e0eab4947e932b5fe416898578032dfa Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Thu, 11 Feb 2016 20:43:24 +0000 Subject: [PATCH 13/29] Add test utest case summary support (+checks) --- mbed_greentea/mbed_greentea_cli.py | 22 +++++++++++++++++++--- mbed_greentea/mbed_test_api.py | 12 +++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 12f96062..14883687 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -373,7 +373,7 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta enum_host_tests_path=enum_host_tests_path, verbose=verbose) - single_test_result, single_test_output, single_testduration, single_timeout, result_test_cases = host_test_result + single_test_result, single_test_output, single_testduration, single_timeout, result_test_cases, test_cases_summary = host_test_result test_result = single_test_result if single_test_result != TEST_RESULT_OK: test_exec_retcode += 1 @@ -399,11 +399,27 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta test_result, single_testduration)) + passes_cnt, failures_cnt = 0, 0 for tc_name in sorted(result_test_cases.keys()): gt_logger.gt_log_tab("test case: '%s' %s %s in %.2f sec"% (tc_name, '.' * (80 - len(tc_name)), result_test_cases[tc_name].get('result_text', '_'), result_test_cases[tc_name].get('duration', 0.0))) + if result_test_cases[tc_name].get('result_text', '_') == 'OK': + passes_cnt += 1 + else: + failures_cnt += 1 + + if test_cases_summary: + passes, failures = test_cases_summary + gt_logger.gt_log("test case summary: %d pass%s, %d failur%s"% (passes, + '' if passes == 1 else 'es', + failures, + '' if failures == 1 else 'es')) + if passes != passes_cnt or failures != failures_cnt: + gt_logger.gt_log_err("test case summary mismatch: reported passes vs failures miscount!") + else: + gt_logger.gt_log_warn("test case summary not found") if single_test_result != 'OK' and not verbose and opts.report_fails: # In some cases we want to print console to see why test failed @@ -455,7 +471,7 @@ def main_cli(opts, args, gt_instance_uuid=None): enum_host_tests_path=enum_host_tests_path, verbose=opts.verbose_test_result_only) - single_test_result, single_test_output, single_testduration, single_timeout, result_test_cases = host_test_result + single_test_result, single_test_output, single_testduration, single_timeout, result_test_cases, test_cases_summary = host_test_result status = TEST_RESULTS.index(single_test_result) if single_test_result in TEST_RESULTS else -1 return (status) @@ -657,7 +673,7 @@ def main_cli(opts, args, gt_instance_uuid=None): enum_host_tests_path=enum_host_tests_path, verbose=True) - single_test_result, single_test_output, single_testduration, single_timeout, result_test_cases = host_test_result + single_test_result, single_test_output, single_testduration, single_timeout, result_test_cases, test_cases_summary = host_test_result status = TEST_RESULTS.index(single_test_result) if single_test_result in TEST_RESULTS else -1 if single_test_result != TEST_RESULT_OK: test_exec_retcode += 1 diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index f2d1a338..7fd79790 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -186,11 +186,21 @@ def run_command(cmd): result = get_test_result(htrun_output) result_test_cases = get_testcase_result(htrun_output) + test_cases_summary = get_testcase_summary(htrun_output) if verbose: gt_logger.gt_log("mbed-host-test-runner: stopped") gt_logger.gt_log("mbed-host-test-runner: returned '%s'"% result) - return (result, "".join(htrun_output), testcase_duration, duration, result_test_cases) + return (result, htrun_output, testcase_duration, duration, result_test_cases, test_cases_summary) + +def get_testcase_summary(output): + re_tc_summary = re.compile(r"^\[(\d+\.\d+)\][^\{]+\{\{(__testcase_summary);(\d+);(\d+)\}\}") + for line in output.splitlines(): + m = re_tc_summary.search(line) + if m: + timestamp, _, passes, failures = m.groups() + return int(passes), int(failures) + return None def get_testcase_result(output): result_test_cases = {} # Test cases results From 0dd25c794234e92331a1f9429563d5f52f191afa Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Thu, 11 Feb 2016 20:48:35 +0000 Subject: [PATCH 14/29] Add verbose flag to htrun on-screen logging --- mbed_greentea/mbed_test_api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index 7fd79790..07a0219b 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -169,7 +169,7 @@ def run_command(cmd): if verbose: gt_logger.gt_log_tab("calling mbedhtrun: %s"% " ".join(cmd)) - gt_logger.gt_log("mbed-host-test-runner: started") + gt_logger.gt_log("mbed-host-test-runner: started") htrun_output = '' start_time = time() @@ -178,8 +178,9 @@ def run_command(cmd): htrun_output += line # When dumping output to file both \r and \n will be a new line # To avoid this "extra new-line" we only use \n at the end - sys.stdout.write(line.rstrip() + '\n') - sys.stdout.flush() + if verbose: + sys.stdout.write(line.rstrip() + '\n') + sys.stdout.flush() end_time = time() testcase_duration = end_time - start_time # Test case duration from reset to {end} From 836e330776d2a9d4193d243f693985c721b95155 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Thu, 11 Feb 2016 21:58:32 +0000 Subject: [PATCH 15/29] Add alias for '-S' for CLI switch '--skip-build' --- mbed_greentea/mbed_greentea_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 14883687..97a67a13 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -148,7 +148,7 @@ def main(): default=False, help="Only build repository and tests, skips actual test procedures (flashing etc.)") - parser.add_option("", "--skip-build", + parser.add_option("-S", "--skip-build", action="store_true", dest="skip_yotta_build", default=False, From dad99602f9fca4e7cba4d92b6d81e921838afe3d Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Thu, 11 Feb 2016 22:11:29 +0000 Subject: [PATCH 16/29] Add alias '-l' for CLI switch '--list' --- mbed_greentea/mbed_greentea_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 97a67a13..7c942bc8 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -187,7 +187,7 @@ def main(): action="store_true", help='If possible force build in debug mode (yotta -d).') - parser.add_option('', '--list', + parser.add_option('-l', '--list', dest='list_binaries', default=False, action="store_true", From 43abb3cfeb077d142c8c0e0fedbc853c58b6cafc Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Mon, 15 Feb 2016 17:40:07 +0000 Subject: [PATCH 17/29] Add test case reporting to junit report Fixed missing text reporting table with test cases --- mbed_greentea/mbed_greentea_cli.py | 21 ++++--- mbed_greentea/mbed_report_api.py | 63 ++++++++++++++++++- ...et_parse.py => mbed_yotta_module_parse.py} | 33 +++++++++- 3 files changed, 108 insertions(+), 9 deletions(-) rename mbed_greentea/{mbed_yotta_target_parse.py => mbed_yotta_module_parse.py} (72%) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 7c942bc8..8c45bdc8 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -36,7 +36,7 @@ from mbed_greentea.mbed_report_api import exporter_text from mbed_greentea.mbed_report_api import exporter_testcase_text from mbed_greentea.mbed_report_api import exporter_json -from mbed_greentea.mbed_report_api import exporter_junit +from mbed_greentea.mbed_report_api import exporter_testcase_junit from mbed_greentea.mbed_target_info import get_mbed_clasic_target_info from mbed_greentea.mbed_target_info import get_mbed_target_from_current_dir from mbed_greentea.mbed_greentea_log import gt_logger @@ -45,7 +45,8 @@ from mbed_greentea.mbed_greentea_dlm import greentea_update_kettle from mbed_greentea.mbed_greentea_dlm import greentea_clean_kettle from mbed_greentea.mbed_yotta_api import build_with_yotta -from mbed_greentea.mbed_yotta_target_parse import YottaConfig +from mbed_greentea.mbed_yotta_module_parse import YottaConfig +from mbed_greentea.mbed_yotta_module_parse import YottaModule try: import mbed_lstools @@ -412,7 +413,7 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta if test_cases_summary: passes, failures = test_cases_summary - gt_logger.gt_log("test case summary: %d pass%s, %d failur%s"% (passes, + gt_logger.gt_log("test case summary: %d pass%s, %d failur%s"% (passes, '' if passes == 1 else 'es', failures, '' if failures == 1 else 'es')) @@ -475,6 +476,10 @@ def main_cli(opts, args, gt_instance_uuid=None): status = TEST_RESULTS.index(single_test_result) if single_test_result in TEST_RESULTS else -1 return (status) + ### Read yotta module basic information + yotta_module = YottaModule() + yotta_module.init() # Read actual yotta module data + ### Selecting yotta targets to process yt_targets = [] # List of yotta targets specified by user used to process during this run if opts.list_of_targets: @@ -780,14 +785,16 @@ def main_cli(opts, args, gt_instance_uuid=None): # Reports (to file) if opts.report_junit_file_name: gt_logger.gt_log("exporting to JUnit file '%s'..."% gt_logger.gt_bright(opts.report_junit_file_name)) - junit_report = exporter_junit(test_report) + junit_report = exporter_testcase_junit(test_report, test_suite_properties=yotta_module.get_data()) with open(opts.report_junit_file_name, 'w') as f: f.write(junit_report) if opts.report_text_file_name: - gt_logger.gt_log("exporting to junit '%s'..."% gt_logger.gt_bright(opts.report_text_file_name)) + gt_logger.gt_log("exporting to text '%s'..."% gt_logger.gt_bright(opts.report_text_file_name)) + text_report, text_results = exporter_text(test_report) + text_testcase_report, text_testcase_results = exporter_testcase_text(test_report) with open(opts.report_text_file_name, 'w') as f: - f.write(text_report) + f.write('\n'.join([text_report, text_results, text_testcase_report, text_testcase_results])) # Reports (to console) if opts.report_json: @@ -804,7 +811,7 @@ def main_cli(opts, args, gt_instance_uuid=None): gt_logger.gt_log("test suite results: " + text_results) # test case detailed report gt_logger.gt_log("test case report:") - text_testcase_report, text_testcase_results = exporter_testcase_text(test_report) + text_testcase_report, text_testcase_results = exporter_testcase_text(test_report, test_suite_properties=yotta_module.get_data()) print text_testcase_report gt_logger.gt_log("test case results: " + text_testcase_results) diff --git a/mbed_greentea/mbed_report_api.py b/mbed_greentea/mbed_report_api.py index ae95a557..1857c504 100644 --- a/mbed_greentea/mbed_report_api.py +++ b/mbed_greentea/mbed_report_api.py @@ -120,6 +120,8 @@ def exporter_text(test_result_ext, test_suite_properties=None): def exporter_testcase_text(test_result_ext, test_suite_properties=None): """! Exports test case results to text formatted output + @param test_result_ext Extended report from Greentea + @param test_suite_properties Data from yotta module.json file @details This is a human friendly format @return Tuple with table of results and result quantity summary string """ @@ -131,6 +133,7 @@ def exporter_testcase_text(test_result_ext, test_suite_properties=None): pt.align[col] = "l" pt.padding_width = 1 # One space between column edges and contents (default) + # ym_name = test_suite_properties.get('name', 'unknown') result_testcase_dict = {} # Used to print test case results for target_name in test_result_ext: @@ -151,7 +154,7 @@ def exporter_testcase_text(test_result_ext, test_suite_properties=None): for tc_name in sorted(testcase_result.keys()): duration = testcase_result[tc_name].get('duration', 0.0) - result = testcase_result[tc_name].get('result', 0) + # result = testcase_result[tc_name].get('result', 0) passed = testcase_result[tc_name].get('passed', 0) failed = testcase_result[tc_name].get('failed', 0) result_text = testcase_result[tc_name].get('result_text', "UNDEF") @@ -176,3 +179,61 @@ def exporter_testcase_text(test_result_ext, test_suite_properties=None): result_pt = pt.get_string() result_res = ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_testcase_dict.items() if v != 0}.iteritems()]) return result_pt, result_res + +def exporter_testcase_junit(test_result_ext, test_suite_properties=None): + """! Export test results in JUnit XML compliant format + @param test_result_ext Extended report from Greentea + @param test_suite_properties Data from yotta module.json file + @details This function will import junit_xml library to perform report conversion + @return String containing Junit XML formatted test result output + """ + from junit_xml import TestSuite, TestCase + + ym_name = test_suite_properties.get('name', 'unknown') + + test_suites = [] + + for target_name in test_result_ext: + test_results = test_result_ext[target_name] + for test_suite_name in test_results: + test = test_results[test_suite_name] + + # tc_elapsed_sec = test['elapsed_time'] + tc_stdout = '' #test['single_test_output'] + tc_stderr = test['single_test_output'] + + # testcase_result stores info about test case results + testcase_result = test['testcase_result'] + # "testcase_result": { + # "STRINGS004": { + # "duration": 0.009999990463256836, + # "time_start": 1453073018.275, + # "time_end": 1453073018.285, + # "result": 1 + # }, + + test_cases = [] + + for tc_name in sorted(testcase_result.keys()): + duration = testcase_result[tc_name].get('duration', 0.0) + # result = testcase_result[tc_name].get('result', 0) + # passed = testcase_result[tc_name].get('passed', 0) + # failed = testcase_result[tc_name].get('failed', 0) + result_text = testcase_result[tc_name].get('result_text', "UNDEF") + + tc_class = ym_name + '.' + target_name + '.' + test_suite_name + tc = TestCase(tc_name, tc_class, duration, tc_stdout, tc_stderr) + + message = '' + if result_text == 'FAIL': + tc.add_failure_info(message, tc_stdout) + elif result_text != 'OK': + tc.add_error_info(message, tc_stdout) + + test_cases.append(tc) + + ts_name = ym_name + '.' + target_name + ts = TestSuite(ts_name, test_cases) + test_suites.append(ts) + + return TestSuite.to_xml_string(test_suites) diff --git a/mbed_greentea/mbed_yotta_target_parse.py b/mbed_greentea/mbed_yotta_module_parse.py similarity index 72% rename from mbed_greentea/mbed_yotta_target_parse.py rename to mbed_greentea/mbed_yotta_module_parse.py index c6122e56..6de216b8 100644 --- a/mbed_greentea/mbed_yotta_target_parse.py +++ b/mbed_greentea/mbed_yotta_module_parse.py @@ -1,6 +1,6 @@ """ mbed SDK -Copyright (c) 2011-2015 ARM Limited +Copyright (c) 2011-2016 ARM Limited Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -77,3 +77,34 @@ def get_test_pins(self): if 'test-pins' in self.yotta_config['hardware']: return self.yotta_config['hardware']['test-pins'] return None + + +class YottaModule(): + + __yotta_module = None + + def __init__(self): + self.MODULE_PATH = '.' + self.YOTTA_CONFIG_NAME = 'module.json' + + def init(self): + """! Loads yotta_module.json as an object from local yotta build directory + @return True if data was successfuly loaded from the file + """ + try: + path = os.path.join(self.MODULE_PATH, self.YOTTA_CONFIG_NAME) + with open(path, 'r') as data_file: + self.__yotta_module = json.load(data_file) + except IOError as e: + print "YottaModule: error - ", str(e) + self.__yotta_module = {} + return bool(len(self.__yotta_module)) + + def set_yotta_config(self, yotta_module): + self.__yotta_module = yotta_module + + def get_data(self): + return self.__yotta_module + + def get_name(self): + return self.__yotta_module.get('name', 'unknown') From c12a3ffe5eaebf1c4f96bd7700dc7fb811ca4e17 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Mon, 15 Feb 2016 18:52:55 +0000 Subject: [PATCH 18/29] Add utest log parsing for JUnit results --- mbed_greentea/mbed_report_api.py | 2 ++ mbed_greentea/mbed_test_api.py | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/mbed_greentea/mbed_report_api.py b/mbed_greentea/mbed_report_api.py index 1857c504..a926599b 100644 --- a/mbed_greentea/mbed_report_api.py +++ b/mbed_greentea/mbed_report_api.py @@ -219,8 +219,10 @@ def exporter_testcase_junit(test_result_ext, test_suite_properties=None): # result = testcase_result[tc_name].get('result', 0) # passed = testcase_result[tc_name].get('passed', 0) # failed = testcase_result[tc_name].get('failed', 0) + utest_log = testcase_result[tc_name].get('utest_log', 0) result_text = testcase_result[tc_name].get('result_text', "UNDEF") + tc_stdout = '\n'.join(utest_log) tc_class = ym_name + '.' + target_name + '.' + test_suite_name tc = TestCase(tc_name, tc_class, duration, tc_stdout, tc_stderr) diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index 07a0219b..e41147d9 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -194,6 +194,45 @@ def run_command(cmd): gt_logger.gt_log("mbed-host-test-runner: returned '%s'"% result) return (result, htrun_output, testcase_duration, duration, result_test_cases, test_cases_summary) +def get_testcase_utest(output, test_case_name): + """ Example test case prints + [1455553765.52][CONN][RXD] >>> Running case #1: 'Simple Test'... + [1455553765.52][CONN][RXD] {{__testcase_start;Simple Test}} + [1455553765.52][CONN][INF] found KV pair in stream: {{__testcase_start;Simple Test}}, queued... + [1455553765.58][CONN][RXD] Simple test called + [1455553765.58][CONN][RXD] {{__testcase_finish;Simple Test;1;0}} + [1455553765.58][CONN][INF] found KV pair in stream: {{__testcase_finish;Simple Test;1;0}}, queued... + [1455553765.70][CONN][RXD] >>> 'Simple Test': 1 passed, 0 failed + """ + + # Return string with all non-alphanumerics backslashed; + # this is useful if you want to match an arbitrary literal + # string that may have regular expression metacharacters in it. + escaped_test_case_name = re.escape(test_case_name) + + re_tc_utest_log_start = re.compile(r"^\[(\d+\.\d+)\]\[(\w+)\]\[(\w+)\] >>> Running case #(\d)+: '(%s)'"% escaped_test_case_name) + re_tc_utest_log_finish = re.compile(r"^\[(\d+\.\d+)\]\[(\w+)\]\[(\w+)\] >>> '(%s)': (\d+) passed, (\d+) failed"% escaped_test_case_name) + + tc_log_lines = [] + for line in output.splitlines(): + + # utest test case start string search + m = re_tc_utest_log_start.search(line) + if m: + tc_log_lines.append(line) + + # If utest test case end string found + m = re_tc_utest_log_finish.search(line) + if m: + tc_log_lines.append(line) + break + + # Continue adding utest log lines + if tc_log_lines: + tc_log_lines.append(line) + + return tc_log_lines + def get_testcase_summary(output): re_tc_summary = re.compile(r"^\[(\d+\.\d+)\][^\{]+\{\{(__testcase_summary);(\d+);(\d+)\}\}") for line in output.splitlines(): @@ -215,6 +254,8 @@ def get_testcase_result(output): if testcase_id not in result_test_cases: result_test_cases[testcase_id] = {} result_test_cases[testcase_id]['time_start'] = float(timestamp) + result_test_cases[testcase_id]['utest_log'] = get_testcase_utest(output, testcase_id) + continue m = re_tc_finish.search(line) From 7d231c4bc22d56280d8a60019503b3284dae9ed3 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 16 Feb 2016 12:06:34 +0000 Subject: [PATCH 19/29] unit test polishing Add YottaModule unit tests Fix for IOTSFW-2070: ``` mbedgt: mbed-host-test-runner: stopped mbedgt: mbed-host-test-runner: returned 'OK' mbedgt: test on hardware with target id: 0240022648e91e7c000000000000000000000000b530e3c4 mbedgt: test suite 'utest-test-case_control_async' ................................................... OK in 16.41 sec test case: 'Control: RepeatAllOnTimeout' ..................................................... OK in 0.09 sec test case: 'Control: RepeatHandlerOnTimeout' ................................................. OK in 1.48 sec test case: 'Control: Timeout (Failure)' ...................................................... OK in 0.00 sec test case: 'Control: Timeout (Success)' ...................................................... OK in 0.00 sec mbedgt: test case summary: 7 passes, 0 failures mbedgt: test case summary mismatch: reported passes vs failures miscount! mbedgt: shuffle seed: 0.8589458441 mbedgt: exporting to JUnit file 'JUnit.xml'... mbedgt: unexpected error: can only join an iterable Traceback (most recent call last): File "C:\jslv3_2\ws\gtt@2\venv\Scripts\mbedgt-script.py", line 9, in load_entry_point('mbed-greentea', 'console_scripts', 'mbedgt')() File "c:\jslv3_2\ws\gtt@2\venv\src\mbed-greentea\mbed_greentea\mbed_greentea_cli.py", line 315, in main cli_ret = main_cli(opts, args) File "c:\jslv3_2\ws\gtt@2\venv\src\mbed-greentea\mbed_greentea\mbed_greentea_cli.py", line 788, in main_cli junit_report = exporter_testcase_junit(test_report, test_suite_properties=yotta_module.get_data()) File "c:\jslv3_2\ws\gtt@2\venv\src\mbed-greentea\mbed_greentea\mbed_report_api.py", line 225, in exporter_testcase_junit tc_stdout = '\n'.join(utest_log) ... --- mbed_greentea/mbed_greentea_cli.py | 4 +- mbed_greentea/mbed_report_api.py | 2 +- mbed_greentea/mbed_yotta_module_parse.py | 2 +- test/mbed_gt_yotta_config.py | 3 +- test/mbed_gt_yotta_module.py | 70 ++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 test/mbed_gt_yotta_module.py diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 8c45bdc8..c1324971 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -818,11 +818,11 @@ def main_cli(opts, args, gt_instance_uuid=None): # This flag guards 'build only' so we expect only yotta errors if test_platforms_match == 0: # No tests were executed - gt_logger.gt_log("no platform/target matching tests were found!") + gt_logger.gt_log_warn("no platform/target matching tests were found!") test_exec_retcode += -10 if target_platforms_match == 0: # No platforms were tested - gt_logger.gt_log("no target matching platforms were found!") + gt_logger.gt_log_warn("no target matching platforms were found!") test_exec_retcode += -100 return (test_exec_retcode) diff --git a/mbed_greentea/mbed_report_api.py b/mbed_greentea/mbed_report_api.py index a926599b..8b776238 100644 --- a/mbed_greentea/mbed_report_api.py +++ b/mbed_greentea/mbed_report_api.py @@ -219,7 +219,7 @@ def exporter_testcase_junit(test_result_ext, test_suite_properties=None): # result = testcase_result[tc_name].get('result', 0) # passed = testcase_result[tc_name].get('passed', 0) # failed = testcase_result[tc_name].get('failed', 0) - utest_log = testcase_result[tc_name].get('utest_log', 0) + utest_log = testcase_result[tc_name].get('utest_log', '') result_text = testcase_result[tc_name].get('result_text', "UNDEF") tc_stdout = '\n'.join(utest_log) diff --git a/mbed_greentea/mbed_yotta_module_parse.py b/mbed_greentea/mbed_yotta_module_parse.py index 6de216b8..c8d3b805 100644 --- a/mbed_greentea/mbed_yotta_module_parse.py +++ b/mbed_greentea/mbed_yotta_module_parse.py @@ -100,7 +100,7 @@ def init(self): self.__yotta_module = {} return bool(len(self.__yotta_module)) - def set_yotta_config(self, yotta_module): + def set_yotta_module(self, yotta_module): self.__yotta_module = yotta_module def get_data(self): diff --git a/test/mbed_gt_yotta_config.py b/test/mbed_gt_yotta_config.py index 8bfd9cfd..191a36c1 100644 --- a/test/mbed_gt_yotta_config.py +++ b/test/mbed_gt_yotta_config.py @@ -17,7 +17,8 @@ """ import unittest -from mbed_greentea.mbed_yotta_target_parse import YottaConfig +from mbed_greentea.mbed_yotta_module_parse import YottaConfig + class YOttaConfigurationParse(unittest.TestCase): diff --git a/test/mbed_gt_yotta_module.py b/test/mbed_gt_yotta_module.py new file mode 100644 index 00000000..f83564ec --- /dev/null +++ b/test/mbed_gt_yotta_module.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +""" +mbed SDK +Copyright (c) 2011-2015 ARM Limited + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import unittest +from mbed_greentea.mbed_yotta_module_parse import YottaModule + + +class YOttaConfigurationParse(unittest.TestCase): + + def setUp(self): + self.YOTTA_MODULE_LONG = { + "name": "utest", + "version": "1.9.1", + "description": "Simple test harness with unity and greentea integration.", + "keywords": [ + "greentea", + "testing", + "unittest", + "unity", + "unit", + "test", + "asynchronous", + "async", + "mbed-official" + ], + "author": "Niklas Hauser ", + "license": "Apache-2.0", + "dependencies": { + "minar": "^1.0.0", + "core-util": "^1.0.1", + "compiler-polyfill": "^1.2.0", + "mbed-drivers": "~0.12.0", + "greentea-client": "^0.1.2" + }, + "testDependencies": { + "unity": "^2.0.1", + "greentea-client": "^0.1.2" + } + } + + self.yotta_module = YottaModule() + self.yotta_module.set_yotta_module(self.YOTTA_MODULE_LONG) + + def tearDown(self): + pass + + def test_get_name(self): + self.assertEqual('utest', self.yotta_module.get_name()) + + def test_get_dict_items(self): + self.assertEqual('Simple test harness with unity and greentea integration.', self.yotta_module.get_data().get('description')) + self.assertEqual('Apache-2.0', self.yotta_module.get_data().get('license')) + +if __name__ == '__main__': + unittest.main() From 7280188ffaa489eec9e1dd60cadc372111be7dee Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Wed, 17 Feb 2016 17:08:20 +0000 Subject: [PATCH 20/29] Remove extra line in Junit stdout log cut Polishing of "test case summary mismatch" message --- mbed_greentea/mbed_greentea_cli.py | 1 + mbed_greentea/mbed_test_api.py | 1 + 2 files changed, 2 insertions(+) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index c1324971..022631f5 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -419,6 +419,7 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta '' if failures == 1 else 'es')) if passes != passes_cnt or failures != failures_cnt: gt_logger.gt_log_err("test case summary mismatch: reported passes vs failures miscount!") + gt_logger.gt_log_tab("(%d, %d) vs (%d, %d)"% (passes, failures, passes_cnt, failures_cnt)) else: gt_logger.gt_log_warn("test case summary not found") diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index e41147d9..1541e748 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -220,6 +220,7 @@ def get_testcase_utest(output, test_case_name): m = re_tc_utest_log_start.search(line) if m: tc_log_lines.append(line) + continue # If utest test case end string found m = re_tc_utest_log_finish.search(line) From cd2bd4aae0b3ac25ade97e76c2ed306e5bc537e8 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Mon, 22 Feb 2016 16:53:48 +0000 Subject: [PATCH 21/29] Typo --- mbed_greentea/mbed_greentea_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 022631f5..251aec42 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -416,7 +416,7 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta gt_logger.gt_log("test case summary: %d pass%s, %d failur%s"% (passes, '' if passes == 1 else 'es', failures, - '' if failures == 1 else 'es')) + 'e' if failures == 1 else 'es')) if passes != passes_cnt or failures != failures_cnt: gt_logger.gt_log_err("test case summary mismatch: reported passes vs failures miscount!") gt_logger.gt_log_tab("(%d, %d) vs (%d, %d)"% (passes, failures, passes_cnt, failures_cnt)) From 65b7236919ed8ea341f47067ada47b17829d17db Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 23 Feb 2016 16:21:02 +0000 Subject: [PATCH 22/29] Add Test Suite -> Test Case mapping in case of no utest present This feature is used to assume that some binaries will operate without utest instrumentation. For example tests for MINAR, some tests in CORE-UTIL etc. may require special binary preparation. --- mbed_greentea/mbed_greentea_cli.py | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 251aec42..d8b28c73 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -422,6 +422,50 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta gt_logger.gt_log_tab("(%d, %d) vs (%d, %d)"% (passes, failures, passes_cnt, failures_cnt)) else: gt_logger.gt_log_warn("test case summary not found") + if not result_test_cases: + gt_logger.gt_log_tab("no test case report, assuming test suite to be a single test case!") + # We will map test suite result to test case to + # output valid test case in report + + # Generate "artificial" test case name from test suite name# + # E.g: + # mbed-drivers-test-dev_null -> dev_null + test_case_name = test_suite_name + test_str_idx = test_suite_name.find("-test") + if test_str_idx != -1: + test_case_name = test_case_name[test_str_idx + 6:] + + # Test case result: OK, FAIL or ERROR + tc_result_text = { + "OK": "OK", + "FAIL": "FAIL", + }.get(single_test_result, 'ERROR') + + # Test case integer success code OK, FAIL and ERROR: (0, >0, <0) + tc_result = { + "OK": 0, + "FAIL": 1024, + "ERROR": -1024, + }.get(tc_result_text, '-2048') + + # Test case passes and failures: (1 pass, 0 failures) or (0 passes, 1 failure) + tc_passed, tc_failed = { + 0: (1, 0), + }.get(tc_result, (0, 1)) + + # Add test case made from test suite result to test case report + test_report[yotta_target_name][test_suite_name]['testcase_result'] = { + test_case_name: { + 'duration': single_testduration, + 'time_start': 0.0, + 'time_end': 0.0, + 'utest_log': single_test_output.splitlines(), + 'result_text': tc_result_text, + 'passed': tc_passed, + 'failed': tc_failed, + 'result': tc_result, + } + } if single_test_result != 'OK' and not verbose and opts.report_fails: # In some cases we want to print console to see why test failed From 3cda48676269c4f6f671d0f71a05477f83860d67 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 23 Feb 2016 16:58:33 +0000 Subject: [PATCH 23/29] Refactore test suite -> test case mapping feature to be more generic --- mbed_greentea/mbed_greentea_cli.py | 110 +++++++++++++++-------------- 1 file changed, 58 insertions(+), 52 deletions(-) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index d8b28c73..e4cf9d6d 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -387,12 +387,56 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta if test_suite_name not in test_report[yotta_target_name]: test_report[yotta_target_name][test_suite_name] = {} - test_report[yotta_target_name][test_suite_name]['single_test_result'] = single_test_result - test_report[yotta_target_name][test_suite_name]['single_test_output'] = single_test_output - test_report[yotta_target_name][test_suite_name]['elapsed_time'] = single_testduration - test_report[yotta_target_name][test_suite_name]['platform_name'] = micro - test_report[yotta_target_name][test_suite_name]['copy_method'] = copy_method - test_report[yotta_target_name][test_suite_name]['testcase_result'] = result_test_cases + if not test_cases_summary and not result_test_cases: + gt_logger.gt_log_warn("test case summary event not found") + gt_logger.gt_log_tab("no test case report present, assuming test suite to be a single test case!") + + # We will map test suite result to test case to + # output valid test case in report + + # Generate "artificial" test case name from test suite name# + # E.g: + # mbed-drivers-test-dev_null -> dev_null + test_case_name = test_suite_name + test_str_idx = test_suite_name.find("-test") + if test_str_idx != -1: + test_case_name = test_case_name[test_str_idx + 6:] + + # Test case result: OK, FAIL or ERROR + tc_result_text = { + "OK": "OK", + "FAIL": "FAIL", + }.get(single_test_result, 'ERROR') + + # Test case integer success code OK, FAIL and ERROR: (0, >0, <0) + tc_result = { + "OK": 0, + "FAIL": 1024, + "ERROR": -1024, + }.get(tc_result_text, '-2048') + + # Test case passes and failures: (1 pass, 0 failures) or (0 passes, 1 failure) + tc_passed, tc_failed = { + 0: (1, 0), + }.get(tc_result, (0, 1)) + + # Test case report build for whole binary + # Add test case made from test suite result to test case report + result_test_cases = { + test_case_name: { + 'duration': single_testduration, + 'time_start': 0.0, + 'time_end': 0.0, + 'utest_log': single_test_output.splitlines(), + 'result_text': tc_result_text, + 'passed': tc_passed, + 'failed': tc_failed, + 'result': tc_result, + } + } + + # Test summary build for whole binary (as a test case) + test_cases_summary = (tc_passed, tc_failed, ) gt_logger.gt_log("test on hardware with target id: %s"% (mut['target_id'])) gt_logger.gt_log("test suite '%s' %s %s in %.2f sec"% (test['test_bin'], @@ -400,6 +444,14 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta test_result, single_testduration)) + # Test report build for whole binary + test_report[yotta_target_name][test_suite_name]['single_test_result'] = single_test_result + test_report[yotta_target_name][test_suite_name]['single_test_output'] = single_test_output + test_report[yotta_target_name][test_suite_name]['elapsed_time'] = single_testduration + test_report[yotta_target_name][test_suite_name]['platform_name'] = micro + test_report[yotta_target_name][test_suite_name]['copy_method'] = copy_method + test_report[yotta_target_name][test_suite_name]['testcase_result'] = result_test_cases + passes_cnt, failures_cnt = 0, 0 for tc_name in sorted(result_test_cases.keys()): gt_logger.gt_log_tab("test case: '%s' %s %s in %.2f sec"% (tc_name, @@ -420,52 +472,6 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta if passes != passes_cnt or failures != failures_cnt: gt_logger.gt_log_err("test case summary mismatch: reported passes vs failures miscount!") gt_logger.gt_log_tab("(%d, %d) vs (%d, %d)"% (passes, failures, passes_cnt, failures_cnt)) - else: - gt_logger.gt_log_warn("test case summary not found") - if not result_test_cases: - gt_logger.gt_log_tab("no test case report, assuming test suite to be a single test case!") - # We will map test suite result to test case to - # output valid test case in report - - # Generate "artificial" test case name from test suite name# - # E.g: - # mbed-drivers-test-dev_null -> dev_null - test_case_name = test_suite_name - test_str_idx = test_suite_name.find("-test") - if test_str_idx != -1: - test_case_name = test_case_name[test_str_idx + 6:] - - # Test case result: OK, FAIL or ERROR - tc_result_text = { - "OK": "OK", - "FAIL": "FAIL", - }.get(single_test_result, 'ERROR') - - # Test case integer success code OK, FAIL and ERROR: (0, >0, <0) - tc_result = { - "OK": 0, - "FAIL": 1024, - "ERROR": -1024, - }.get(tc_result_text, '-2048') - - # Test case passes and failures: (1 pass, 0 failures) or (0 passes, 1 failure) - tc_passed, tc_failed = { - 0: (1, 0), - }.get(tc_result, (0, 1)) - - # Add test case made from test suite result to test case report - test_report[yotta_target_name][test_suite_name]['testcase_result'] = { - test_case_name: { - 'duration': single_testduration, - 'time_start': 0.0, - 'time_end': 0.0, - 'utest_log': single_test_output.splitlines(), - 'result_text': tc_result_text, - 'passed': tc_passed, - 'failed': tc_failed, - 'result': tc_result, - } - } if single_test_result != 'OK' and not verbose and opts.report_fails: # In some cases we want to print console to see why test failed From 962599155fe0ef6d83a3ed7bd75fe1115b18c6db Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 23 Feb 2016 23:26:06 +0000 Subject: [PATCH 24/29] Add unicode strip from JUnit report text --- mbed_greentea/mbed_report_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mbed_greentea/mbed_report_api.py b/mbed_greentea/mbed_report_api.py index 8b776238..78bdf8b4 100644 --- a/mbed_greentea/mbed_report_api.py +++ b/mbed_greentea/mbed_report_api.py @@ -200,7 +200,7 @@ def exporter_testcase_junit(test_result_ext, test_suite_properties=None): # tc_elapsed_sec = test['elapsed_time'] tc_stdout = '' #test['single_test_output'] - tc_stderr = test['single_test_output'] + tc_stderr = test['single_test_output'].decode('unicode_escape').encode('ascii','ignore') # testcase_result stores info about test case results testcase_result = test['testcase_result'] @@ -222,7 +222,7 @@ def exporter_testcase_junit(test_result_ext, test_suite_properties=None): utest_log = testcase_result[tc_name].get('utest_log', '') result_text = testcase_result[tc_name].get('result_text', "UNDEF") - tc_stdout = '\n'.join(utest_log) + tc_stdout = '\n'.join(utest_log).decode('unicode_escape').encode('ascii','ignore') tc_class = ym_name + '.' + target_name + '.' + test_suite_name tc = TestCase(tc_name, tc_class, duration, tc_stdout, tc_stderr) From e9efa7b928734ba44a48cabebe777502487da0dd Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 23 Feb 2016 23:27:01 +0000 Subject: [PATCH 25/29] Add default test case report values for __testcase_finish --- mbed_greentea/mbed_test_api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index 1541e748..b2ce0d70 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -254,9 +254,18 @@ def get_testcase_result(output): timestamp, _, testcase_id = m.groups() if testcase_id not in result_test_cases: result_test_cases[testcase_id] = {} + + # Data collected when __testcase_start is fetched result_test_cases[testcase_id]['time_start'] = float(timestamp) result_test_cases[testcase_id]['utest_log'] = get_testcase_utest(output, testcase_id) + # Data collected when __testcase_finish is fetched + result_test_cases[testcase_id]['duration'] = 0.0 + result_test_cases[testcase_id]['result_text'] = 'ERROR' + result_test_cases[testcase_id]['time_end'] = float(timestamp) + result_test_cases[testcase_id]['passed'] = 0 + result_test_cases[testcase_id]['failed'] = 0 + result_test_cases[testcase_id]['result'] = -4096 continue m = re_tc_finish.search(line) From 6cdc12b90e06915641a5775bb70e77dc1936ffd0 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 26 Feb 2016 11:12:02 +0000 Subject: [PATCH 26/29] Test suite -> test case feature polishing --- mbed_greentea/mbed_greentea_cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index e4cf9d6d..5e87bff5 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -398,10 +398,13 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta # E.g: # mbed-drivers-test-dev_null -> dev_null test_case_name = test_suite_name - test_str_idx = test_suite_name.find("-test") + test_str_idx = test_suite_name.find("-test-") if test_str_idx != -1: test_case_name = test_case_name[test_str_idx + 6:] + gt_logger.gt_log_tab("test suite: %s"% test_suite_name) + gt_logger.gt_log_tab("test case: %s"% test_case_name) + # Test case result: OK, FAIL or ERROR tc_result_text = { "OK": "OK", From c7c5f5407023c9f02c8732499a05850695f11bf9 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 26 Feb 2016 15:13:19 +0000 Subject: [PATCH 27/29] Add sorted results for test cases report --- mbed_greentea/mbed_report_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mbed_greentea/mbed_report_api.py b/mbed_greentea/mbed_report_api.py index 78bdf8b4..769987a3 100644 --- a/mbed_greentea/mbed_report_api.py +++ b/mbed_greentea/mbed_report_api.py @@ -136,10 +136,10 @@ def exporter_testcase_text(test_result_ext, test_suite_properties=None): # ym_name = test_suite_properties.get('name', 'unknown') result_testcase_dict = {} # Used to print test case results - for target_name in test_result_ext: + for target_name in sorted(test_result_ext): test_results = test_result_ext[target_name] row = [] - for test_suite_name in test_results: + for test_suite_name in sorted(test_results): test = test_results[test_suite_name] # testcase_result stores info about test case results @@ -152,7 +152,7 @@ def exporter_testcase_text(test_result_ext, test_suite_properties=None): # "result": 1 # }, - for tc_name in sorted(testcase_result.keys()): + for tc_name in sorted(testcase_result): duration = testcase_result[tc_name].get('duration', 0.0) # result = testcase_result[tc_name].get('result', 0) passed = testcase_result[tc_name].get('passed', 0) From 8cf5dd0e6158a1148cb45a110ae5269e36f6a998 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 1 Mar 2016 14:56:12 +0000 Subject: [PATCH 28/29] Add Hooks & GCOV support --- mbed_greentea/mbed_coverage_api.py | 65 ++++++++++ mbed_greentea/mbed_greentea_cli.py | 73 ++++++++++- mbed_greentea/mbed_greentea_hooks.py | 175 +++++++++++++++++++++++++++ mbed_greentea/mbed_test_api.py | 19 +++ test/mbed_gt_hooks.py | 105 ++++++++++++++++ 5 files changed, 434 insertions(+), 3 deletions(-) create mode 100644 mbed_greentea/mbed_coverage_api.py create mode 100644 mbed_greentea/mbed_greentea_hooks.py create mode 100644 test/mbed_gt_hooks.py diff --git a/mbed_greentea/mbed_coverage_api.py b/mbed_greentea/mbed_coverage_api.py new file mode 100644 index 00000000..cb6191d9 --- /dev/null +++ b/mbed_greentea/mbed_coverage_api.py @@ -0,0 +1,65 @@ +""" +mbed SDK +Copyright (c) 2011-2016 ARM Limited + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Przemyslaw Wirkus +""" + +import os + +""" +def __default_coverage_start_callback(self, key, value, timestamp): + # {{__coverage_start;PATH;PAYLOAD}} + # PAYLAODED is HEX coded string + lcov_path, lcov_payload = value.split(';') + try: + bin_payload = coverage_pack_hex_payload(lcov_payload) + coverage_dump_file(lcov_path, bin_payload) + + self.log("dumped %d bytes to '%s'"% (len(bin_payload), lcov_path)) + except Exception as e: + self.log("LCOV:" + str(e)) +""" + +def coverage_pack_hex_payload(payload): + """! Convert a block of hex string data back to binary and return the binary data + @param payload String with hex encoded ascii data, e.g.: '6164636772...' + @return bytearray with payload with data + """ + # This payload might be packed with dot compression + # where byte value 0x00 is coded as ".", and not as "00" + payload = payload.replace('.', '00') + + hex_pairs = map(''.join, zip(*[iter(payload)] * 2)) # ['61', '64', '63', '67', '72', ... ] + bin_payload = bytearray([int(s, 16) for s in hex_pairs]) + return bin_payload + +def coverage_dump_file(path, payload): + """! Creates file and dumps payload to it on specified path (even if path doesn't exist) + @param path Path to file + @param payload Binary data to store in a file + @return True if operation was completed + """ + result = True + try: + d = os.path.dirname(path) + if not os.path.exists(d): + os.makedirs(d) + with open(path, "wb") as f: + f.write(payload) + except IOError as e: + print str(e) + result = False + return result diff --git a/mbed_greentea/mbed_greentea_cli.py b/mbed_greentea/mbed_greentea_cli.py index 5e87bff5..7ff5e4bf 100644 --- a/mbed_greentea/mbed_greentea_cli.py +++ b/mbed_greentea/mbed_greentea_cli.py @@ -30,7 +30,7 @@ from mbed_greentea.mbed_test_api import run_host_test from mbed_greentea.mbed_test_api import TEST_RESULTS -from mbed_greentea.mbed_test_api import TEST_RESULT_OK +from mbed_greentea.mbed_test_api import TEST_RESULT_OK, TEST_RESULT_FAIL from mbed_greentea.cmake_handlers import load_ctest_testsuite from mbed_greentea.cmake_handlers import list_binaries_for_targets from mbed_greentea.mbed_report_api import exporter_text @@ -45,6 +45,7 @@ from mbed_greentea.mbed_greentea_dlm import greentea_update_kettle from mbed_greentea.mbed_greentea_dlm import greentea_clean_kettle from mbed_greentea.mbed_yotta_api import build_with_yotta +from mbed_greentea.mbed_greentea_hooks import GreenteaHooks from mbed_greentea.mbed_yotta_module_parse import YottaConfig from mbed_greentea.mbed_yotta_module_parse import YottaModule @@ -223,6 +224,10 @@ def main(): dest='digest_source', help='Redirect input from where test suite should take console input. You can use stdin or file name to get test case console output') + parser.add_option('-H', '--hooks', + dest='hooks_json', + help='Load hooks used drive extra functionality') + parser.add_option('', '--test-cfg', dest='json_test_configuration', help='Pass to host test data with host test configuration') @@ -330,7 +335,7 @@ def main(): return(cli_ret) -def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_target_name): +def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_target_name, greentea_hooks): test_exec_retcode = 0 test_platforms_match = 0 test_report = {} @@ -376,9 +381,27 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta single_test_result, single_test_output, single_testduration, single_timeout, result_test_cases, test_cases_summary = host_test_result test_result = single_test_result + + build_path = os.path.join("./build", yotta_target_name) + build_path_abs = os.path.abspath(build_path) + if single_test_result != TEST_RESULT_OK: test_exec_retcode += 1 + if single_test_result in [TEST_RESULT_OK, TEST_RESULT_FAIL]: + if greentea_hooks: + # Test was successful + # We can execute test hook just after test is finished ('hook_test_end') + format = { + "test_name": test['test_bin'], + "test_bin_name": os.path.basename(test['image_path']), + "image_path": test['image_path'], + "build_path": build_path, + "build_path_abs": build_path_abs, + "yotta_target_name": yotta_target_name, + } + greentea_hooks.run_hook_ext('hook_test_end', format) + # Update report for optional reporting feature test_suite_name = test['test_bin'].lower() if yotta_target_name not in test_report: @@ -455,6 +478,11 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, mut_info, yotta_ta test_report[yotta_target_name][test_suite_name]['copy_method'] = copy_method test_report[yotta_target_name][test_suite_name]['testcase_result'] = result_test_cases + test_report[yotta_target_name][test_suite_name]['build_path'] = build_path + test_report[yotta_target_name][test_suite_name]['build_path_abs'] = build_path_abs + test_report[yotta_target_name][test_suite_name]['image_path'] = test['image_path'] + test_report[yotta_target_name][test_suite_name]['test_bin_name'] = os.path.basename(test['image_path']) + passes_cnt, failures_cnt = 0, 0 for tc_name in sorted(result_test_cases.keys()): gt_logger.gt_log_tab("test case: '%s' %s %s in %.2f sec"% (tc_name, @@ -516,12 +544,16 @@ def main_cli(opts, args, gt_instance_uuid=None): print_version() return (0) + # We will load hooks from JSON file to support extra behaviour during test execution + greentea_hooks = GreenteaHooks(opts.hooks_json) if opts.hooks_json else None + # Capture alternative test console inputs, used e.g. in 'yotta test command' if opts.digest_source: enum_host_tests_path = get_local_host_tests_dir(opts.enum_host_tests) host_test_result = run_host_test(image_path=None, disk=None, port=None, + hooks=greentea_hooks, digest_source=opts.digest_source, enum_host_tests_path=enum_host_tests_path, verbose=opts.verbose_test_result_only) @@ -796,7 +828,7 @@ def main_cli(opts, args, gt_instance_uuid=None): # Experimental, parallel test execution ################################################################# if number_of_threads < parallel_test_exec: - args = (test_result_queue, test_queue, opts, mut, mut_info, yotta_target_name) + args = (test_result_queue, test_queue, opts, mut, mut_info, yotta_target_name, greentea_hooks) t = Thread(target=run_test_thread, args=args) execute_threads.append(t) number_of_threads += 1 @@ -828,6 +860,41 @@ def main_cli(opts, args, gt_instance_uuid=None): print "Example: execute 'mbedgt --target=TARGET_NAME' to start testing for TARGET_NAME target" return (0) + gt_logger.gt_log("all tests finished!") + + # We will execute post test hooks on tests + for yotta_target in test_report: + test_name_list = [] # All test case names for particular yotta target + for test_name in test_report[yotta_target]: + test = test_report[yotta_target][test_name] + # Test was successful + if test['single_test_result'] in [TEST_RESULT_OK, TEST_RESULT_FAIL]: + test_name_list.append(test_name) + # Call hook executed for each test, just after all tests are finished + if greentea_hooks: + # We can execute this test hook just after all tests are finished ('hook_post_test_end') + format = { + "test_name": test_name, + "test_bin_name": test['test_bin_name'], + "image_path": test['image_path'], + "build_path": test['build_path'], + "build_path_abs": test['build_path_abs'], + "yotta_target_name": yotta_target, + } + greentea_hooks.run_hook_ext('hook_post_test_end', format) + if greentea_hooks: + # Call hook executed for each yotta target, just after all tests are finished + build_path = os.path.join("./build", yotta_target) + build_path_abs = os.path.abspath(build_path) + # We can execute this test hook just after all tests are finished ('hook_post_test_end') + format = { + "build_path": build_path, + "build_path_abs": build_path_abs, + "test_name_list": test_name_list, + "yotta_target_name": yotta_target, + } + greentea_hooks.run_hook_ext('hook_post_all_test_end', format) + # This tool is designed to work in CI # We want to return success codes based on tool actions, # only if testes were executed and all passed we want to diff --git a/mbed_greentea/mbed_greentea_hooks.py b/mbed_greentea/mbed_greentea_hooks.py new file mode 100644 index 00000000..1f5a5c2d --- /dev/null +++ b/mbed_greentea/mbed_greentea_hooks.py @@ -0,0 +1,175 @@ +""" +mbed SDK +Copyright (c) 2011-2015 ARM Limited +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Przemyslaw Wirkus +""" + +import re +import json +from subprocess import Popen, PIPE +from mbed_greentea.mbed_greentea_log import gt_logger + +""" +List of available hooks: +""" + + +class GreenteaTestHook(): + """! Class used to define + """ + name = None + + def __init__(self, name): + self.name = name + + def run(self, format=None): + pass + +class GreenteaCliTestHook(GreenteaTestHook): + """! Class used to define a hook which will call command line program + """ + cmd = None + + def __init__(self, name, cmd): + GreenteaTestHook.__init__(self, name) + self.cmd = cmd + + def run_cli_process(self, cmd): + """! Runs command as a process and return stdout, stderr and ret code + @param cmd Command to execute + @return Tuple of (stdout, stderr, returncode) + """ + _stdout, _stderr, ret = None, None, -1 + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) + _stdout, _stderr = p.communicate() + ret = p.returncode + except OSError as e: + gt_logger.gt_log_err(str(e)) + ret = -1 + return _stdout, _stderr, ret + + def run(self, format=None): + """! Runs hook after command is formated with in-place {tags} + @format Pass format dictionary to replace hook {tags} with real values + @param format Used to format string with cmd, notation used is e.g: {build_name} + """ + gt_logger.gt_log("hook '%s' execution"% self.name) + cmd = self.format_before_run(self.cmd, format) + gt_logger.gt_log_tab("hook command: %s"% cmd) + (_stdout, _stderr, ret) = self.run_cli_process(cmd) + if _stdout: + print _stdout + if ret: + gt_logger.gt_log_err("hook exited with error: %d, dumping stderr..."% ret) + print _stderr + return ret + + @staticmethod + def format_before_run(cmd, format, verbose=False): + if format: + # We will expand first + cmd_expand = GreenteaCliTestHook.expand_parameters(cmd, format) + if cmd_expand: + cmd = cmd_expand + if verbose: + gt_logger.gt_log_tab("hook expanded: %s"% cmd) + + cmd = cmd.format(**format) + if verbose: + gt_logger.gt_log_tab("hook formated: %s"% cmd) + return cmd + + @staticmethod + def expand_parameters(expr, expandables, delimiter=' '): + """! Expands lists for multiple parameters in hook command + @param expr Expression to expand + @param expandables Dictionary of token: list_to_expand See details for more info + @param delimiter Delimiter used to combine expanded strings, space by default + @details + test_name_list = ['mbed-drivers-test-basic', 'mbed-drivers-test-hello', 'mbed-drivers-test-time_us'] + build_path_list = ['./build/frdm-k64f-gcc', './build/frdm-k64f-armcc'] + expandables = { + "{test_name_list}": test_name_list, + "{build_path_list}": build_path_list + } + expr = "lcov --gcov-tool arm-none-eabi-gcov [-a {build_path_list}/test/{test_name_list}.info] --output-file result.info" + 'expr' expression [-a {build_path_list}/test/{test_name_list}.info] will expand to: + [ + "-a ./build/frdm-k64f-gcc/test/mbed-drivers-test-basic.info", + "-a ./build/frdm-k64f-armcc/test/mbed-drivers-test-basic.info", + "-a ./build/frdm-k64f-gcc/test/mbed-drivers-test-hello.info", + "-a ./build/frdm-k64f-armcc/test/mbed-drivers-test-hello.info", + "-a ./build/frdm-k64f-gcc/test/mbed-drivers-test-time_us.info", + "-a ./build/frdm-k64f-armcc/test/mbed-drivers-test-time_us.info" + ] + """ + result = None + if expandables: + expansion_result = [] + m = re.search('\[.*?\]', expr) + if m: + expr_str_orig = m.group(0) + expr_str_base = m.group(0)[1:-1] + expr_str_list = [expr_str_base] + for token in expandables: + # We will expand only values which are lists (of strings) + if type(expandables[token]) is list: + # Use tokens with curly braces (Python string format like) + format_token = '{' + token + '}' + for expr_str in expr_str_list: + if format_token in expr_str: + patterns = expandables[token] + for pattern in patterns: + s = expr_str + s = s.replace(format_token, pattern) + expr_str_list.append(s) + # Nothing to extend/change in this string + if not any('{' + p + '}' in s for p in expandables.keys() if type(expandables[p]) is list): + expansion_result.append(s) + expansion_result.sort() + result = expr.replace(expr_str_orig, delimiter.join(expansion_result)) + return result + +class GreenteaHooks(): + """! Class used to store all hooks + @details Hooks command starts with '$' dollar sign + """ + HOOKS = {} + def __init__(self, path_to_hooks): + """! Opens JSON file with + """ + try: + with open(path_to_hooks, 'r') as data_file: + hooks = json.load(data_file) + if 'hooks' in hooks: + for hook in hooks['hooks']: + hook_name = hook + hook_expression = hooks['hooks'][hook] + # This is a command line hook + if hook_expression.startswith('$'): + self.HOOKS[hook_name] = GreenteaCliTestHook(hook_name, hook_expression[1:]) + except IOError as e: + print str(e) + self.HOOKS = None + + def is_hooked_to(self, hook_name): + return hook_name in self.HOOKS + + def run_hook(self, hook_name, format): + if hook_name in self.HOOKS: + return self.HOOKS[hook_name].run(format) + + def run_hook_ext(self, hook_name, format): + if self.is_hooked_to(hook_name): + # We can execute this test hook just after all tests are finished ('hook_post_test_end') + self.run_hook(hook_name, format) diff --git a/mbed_greentea/mbed_test_api.py b/mbed_greentea/mbed_test_api.py index b2ce0d70..aaa4f04c 100644 --- a/mbed_greentea/mbed_test_api.py +++ b/mbed_greentea/mbed_test_api.py @@ -23,6 +23,8 @@ from subprocess import call, Popen, PIPE, STDOUT from mbed_greentea.mbed_greentea_log import gt_logger +from mbed_greentea.mbed_coverage_api import coverage_dump_file +from mbed_greentea.mbed_coverage_api import coverage_pack_hex_payload # Return codes for test script @@ -188,6 +190,7 @@ def run_command(cmd): result = get_test_result(htrun_output) result_test_cases = get_testcase_result(htrun_output) test_cases_summary = get_testcase_summary(htrun_output) + get_coverage_data(htrun_output) if verbose: gt_logger.gt_log("mbed-host-test-runner: stopped") @@ -234,6 +237,22 @@ def get_testcase_utest(output, test_case_name): return tc_log_lines +def get_coverage_data(output): + # Example GCOV output + # [1456840876.73][CONN][RXD] {{__coverage_start;c:\Work\core-util/source/PoolAllocator.cpp.gcda;6164636772393034c2733f32...a33e...b9}} + gt_logger.gt_log("checking for GCOV data...") + re_gcov = re.compile(r"^\[(\d+\.\d+)\][^\{]+\{\{(__coverage_start);([^;]+);([^}]+)\}\}$") + for line in output.splitlines(): + m = re_gcov.search(line) + if m: + timestamp, _, gcov_path, gcov_payload = m.groups() + try: + bin_gcov_payload = coverage_pack_hex_payload(gcov_payload) + coverage_dump_file(gcov_path, bin_gcov_payload) + except Exception as e: + gt_logger.gt_log_err("error while handling GCOV data: " + str(e)) + gt_logger.gt_log_tab("storing %d bytes in '%s'"% (len(bin_gcov_payload), gcov_path)) + def get_testcase_summary(output): re_tc_summary = re.compile(r"^\[(\d+\.\d+)\][^\{]+\{\{(__testcase_summary);(\d+);(\d+)\}\}") for line in output.splitlines(): diff --git a/test/mbed_gt_hooks.py b/test/mbed_gt_hooks.py new file mode 100644 index 00000000..4a3d40bd --- /dev/null +++ b/test/mbed_gt_hooks.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +""" +mbed SDK +Copyright (c) 2011-2015 ARM Limited +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import unittest +from mbed_greentea.mbed_greentea_hooks import GreenteaCliTestHook + +class GreenteaCliTestHookTest(unittest.TestCase): + + def setUp(self): + self.cli_hooks = GreenteaCliTestHook('test_hook', 'some command') + + def tearDown(self): + pass + + def test_expand_parameters_with_1_list(self): + # Simple list + self.assertEqual('new_value_1 new_value_2', + self.cli_hooks.expand_parameters('[{token_list}]', { + "token_list" : ['new_value_1', 'new_value_2'] + })) + + # List with prefix + self.assertEqual('-a new_value_1 -a new_value_2', + self.cli_hooks.expand_parameters('[-a {token_list}]', { + "token_list" : ['new_value_1', 'new_value_2'] + })) + + # List with prefix and extra text + self.assertEqual('-a /path/to/new_value_1 -a /path/to/new_value_2', + self.cli_hooks.expand_parameters('[-a /path/to/{token_list}]', { + "token_list" : ['new_value_1', 'new_value_2'] + })) + + def test_expand_parameters_with_2_lists(self): + self.assertEqual('ytA T1', + self.cli_hooks.expand_parameters('[{yt_target_list} {test_list}]', { + "test_list" : ['T1'], + "yt_target_list" : ['ytA'], + })) + + self.assertEqual('ytA T1 ytA T2 ytA T3 ytB T1 ytB T2 ytB T3 ytC T1 ytC T2 ytC T3', + self.cli_hooks.expand_parameters('[{yt_target_list} {test_list}]', { + "test_list" : ['T1', 'T2', 'T3'], + "yt_target_list" : ['ytA', 'ytB', 'ytC'], + })) + + # In this test we expect {yotta_target_name} token to stay untouched because it is not declared as a list + self.assertEqual('lcov -a /build/{yotta_target_name}/mbed-drivers-test-basic.info -a /build/{yotta_target_name}/mbed-drivers-test-hello.info', + self.cli_hooks.expand_parameters('lcov [-a /build/{yotta_target_name}/{test_list}.info]', { + "test_list" : ['mbed-drivers-test-basic', 'mbed-drivers-test-hello'], + "yotta_target_name" : 'frdm-k64f-gcc' + })) + + def test_expand_parameters_exceptions(self): + # Here [] is reduced to empty string + self.assertEqual('', + self.cli_hooks.expand_parameters('[]', { + "test_list" : ['mbed-drivers-test-basic', 'mbed-drivers-test-hello'], + "yotta_target_name" : 'frdm-k64f-gcc' + })) + + # Here [some string] is reduced to empty string + self.assertEqual('', + self.cli_hooks.expand_parameters('[some string]', { + "test_list" : ['mbed-drivers-test-basic', 'mbed-drivers-test-hello'], + "yotta_target_name" : 'frdm-k64f-gcc' + })) + + # Here [some string] is reduced to empty string + self.assertEqual('-+=abc', + self.cli_hooks.expand_parameters('-+=[some string]abc', { + "test_list" : ['mbed-drivers-test-basic', 'mbed-drivers-test-hello'], + "yotta_target_name" : 'frdm-k64f-gcc' + })) + + self.assertEqual(None, + self.cli_hooks.expand_parameters('some text without square brackets but with tokes: {test_list} and {yotta_target_name}', { + "test_list" : ['mbed-drivers-test-basic', 'mbed-drivers-test-hello'], + "yotta_target_name" : 'frdm-k64f-gcc' + })) + + def test_format_before_run_with_1_list_1_string(self): + # {test_name_list} should expand [] list twice + # {yotta_target_name} should not be used to expand, only to replace + self.assertEqual('build path = -a /build/frdm-k64f-gcc/mbed-drivers-test-basic -a /build/frdm-k64f-gcc/mbed-drivers-test-hello', + self.cli_hooks.format_before_run('build path = [-a /build/{yotta_target_name}/{test_name_list}]', { + "test_name_list" : ['mbed-drivers-test-basic', 'mbed-drivers-test-hello'], + "yotta_target_name" : 'frdm-k64f-gcc', + })) + + +if __name__ == '__main__': + unittest.main() From ce8f693dfb925011f0a7346ead3adfe96b4ca002 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Wed, 2 Mar 2016 11:38:18 +0000 Subject: [PATCH 29/29] Add htrun v0.2.0 dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4406f008..b3caa8ac 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ def read(fname): }, install_requires=["PrettyTable>=0.7.2", "PySerial>=3.0", - "mbed-host-tests>=0.1.18", + "mbed-host-tests>=0.2.0", "mbed-ls", "junit-xml", "lockfile",