From 63f7a6a7e4677bc052630b4aa755d8e10e297a47 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 19 Sep 2023 10:59:33 +0100 Subject: [PATCH] add exit if there are failed tests --- .github/workflows/linux.yml | 2 +- .../layer_tests_summary/run_conformance.py | 81 +++++++++++-------- 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4b229896f3c15b..cfa800f4288f6a 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -358,7 +358,7 @@ jobs: -t=${{ matrix.TEST_TYPE }} \ -w=${{ env.CONFORMANCE_ARTIFACTS_DIR }} \ -f=${{ env.CONFORMANCE_ARTIFACTS_DIR }}/expected_failures_cpu.csv \ - ${{ github.ref_name == 'master' && '-u' || '' }} + ${{ github.ref_name == 'master' && '-u' || '' }} # Required due to: https://github.com/actions/cache/tree/main/restore#ensuring-proper-restores-and-save-happen-across-the-actions - name: Rename Expected Failures List diff --git a/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_conformance.py b/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_conformance.py index 8c64a0eeb9d3d7..294a8bbee88cfc 100644 --- a/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_conformance.py +++ b/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_conformance.py @@ -1,23 +1,23 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +import csv +import os +import urllib.request as ur from argparse import ArgumentParser -from subprocess import Popen -from shutil import copytree, rmtree -from summarize import create_summary, create_api_summary -from merge_xmls import merge_xml -from run_parallel import TestParallelRunner from pathlib import Path +from shutil import copytree, rmtree +from subprocess import Popen +from urllib.parse import urlparse import defusedxml.ElementTree as ET -from urllib.parse import urlparse -import os -import csv -import urllib.request as ur +from merge_xmls import merge_xml +from run_parallel import TestParallelRunner +from summarize import create_summary, create_api_summary from utils import constants -from utils.conformance_utils import get_logger from utils import file_utils +from utils.conformance_utils import get_logger logger = get_logger('conformance_runner') has_python_api = True @@ -34,10 +34,12 @@ SCRIPT_DIR_PATH, SCRIPT_NAME = os.path.split(os.path.abspath(__file__)) NO_MODEL_CONSTANT = os.path.join(SCRIPT_DIR_PATH, "data", "models.lst") + def get_default_working_dir(): path = Path(__file__).parent.resolve() return os.path.join(path, "temp") + def parse_arguments(): parser = ArgumentParser() @@ -56,27 +58,31 @@ def parse_arguments(): cache_path_help = "Path to the cache file with test_name list sorted by execution time as `.lst` file!" expected_failures_update_help = "Overwrite expected failures list in case same failures were fixed" - parser.add_argument("-d", "--device", help= device_help, type=str, required=False, default="CPU") + parser.add_argument("-d", "--device", help=device_help, type=str, required=False, default="CPU") parser.add_argument("-t", "--type", help=type_help, type=str, required=False, default=constants.OP_CONFORMANCE) parser.add_argument("--gtest_filter", help=gtest_filter_helper, type=str, required=False, default="*") - parser.add_argument("-w", "--working_dir", help=working_dir_help, type=str, required=False, default=get_default_working_dir()) - parser.add_argument("-m", "--models_path", help=models_path_help, type=str, required=False, default=NO_MODEL_CONSTANT) + parser.add_argument("-w", "--working_dir", help=working_dir_help, type=str, required=False, + default=get_default_working_dir()) + parser.add_argument("-m", "--models_path", help=models_path_help, type=str, required=False, + default=NO_MODEL_CONSTANT) parser.add_argument("-ov", "--ov_path", help=ov_help, type=str, required=False, default="") - parser.add_argument("-j", "--workers", help=workers_help, type=int, required=False, default=os.cpu_count()-1) + parser.add_argument("-j", "--workers", help=workers_help, type=int, required=False, default=os.cpu_count() - 1) parser.add_argument("-c", "--ov_config_path", help=ov_config_path_helper, type=str, required=False, default="") parser.add_argument("-s", "--dump_graph", help=dump_graph_help, type=int, required=False, default=0) parser.add_argument("-sm", "--special_mode", help=special_mode_help, type=str, required=False, default="") parser.add_argument("-p", "--parallel_devices", help=parallel_help, type=bool, required=False, default=False) parser.add_argument("-f", "--expected_failures", help=expected_failures_help, type=str, required=False, default="") - parser.add_argument("-u", "--expected_failures_update", help=expected_failures_update_help, type=bool, required=False, default=False) + parser.add_argument("-u", "--expected_failures_update", help=expected_failures_update_help, type=bool, + required=False, default=False) parser.add_argument("--cache_path", help=cache_path_help, type=str, required=False, default="") return parser.parse_args() + class Conformance: - def __init__(self, device:str, model_path:os.path, ov_path:os.path, type:str, workers:int, - gtest_filter:str, working_dir:os.path, ov_config_path:os.path, special_mode:str, - cache_path:str, parallel_devices:bool, expected_failures_file: str, + def __init__(self, device: str, model_path: os.path, ov_path: os.path, type: str, workers: int, + gtest_filter: str, working_dir: os.path, ov_config_path: os.path, special_mode: str, + cache_path: str, parallel_devices: bool, expected_failures_file: str, expected_failures_update: bool): self._device = device self._model_path = model_path @@ -107,7 +113,8 @@ def __init__(self, device:str, model_path:os.path, ov_path:os.path, type:str, wo logger.error(f'Incorrect value to set API scope: {special_mode}. Please check to get possible values') exit(-1) else: - logger.error(f"Incorrect conformance type: {type}. Please use '{constants.OP_CONFORMANCE}' or '{constants.API_CONFORMANCE}'") + logger.error( + f"Incorrect conformance type: {type}. Please use '{constants.OP_CONFORMANCE}' or '{constants.API_CONFORMANCE}'") exit(-1) self._type = type self._workers = workers @@ -124,6 +131,8 @@ def __init__(self, device:str, model_path:os.path, ov_path:os.path, type:str, wo logger.warning(f"Expected failures testlist `{self._expected_failures_file}` does not exist!") self._expected_failures_update = expected_failures_update + self.is_successful_run = False + def __download_models(self, url_to_download, path_to_save): _, file_name = os.path.split(urlparse(url_to_download).path) download_path = os.path.join(path_to_save, file_name) @@ -138,11 +147,10 @@ def __download_models(self, url_to_download, path_to_save): logger.error(f"{download_path} is not a file. Exit!") exit(-1) if file_utils.is_archieve(download_path): - logger.info(f"The file {download_path} is archieve. Should be unzip to {path_to_save}") + logger.info(f"The file {download_path} is archived. Should be unzipped to {path_to_save}") return file_utils.unzip_archieve(download_path, path_to_save) return download_path - def __dump_subgraph(self): subgraph_dumper_path = os.path.join(self._ov_path, f'{SUBGRAPH_DUMPER_BIN_NAME}{constants.OS_BIN_FILE_EXT}') if not os.path.isfile(subgraph_dumper_path): @@ -153,7 +161,9 @@ def __dump_subgraph(self): logger.info(f"Remove directory {conformance_ir_path}") rmtree(conformance_ir_path) os.mkdir(conformance_ir_path) - self._model_path = file_utils.prepare_filelist(self._model_path, ["*.onnx", "*.pdmodel", "*.__model__", "*.pb", "*.xml", "*.tflite"]) + self._model_path = file_utils.prepare_filelist(self._model_path, + ["*.onnx", "*.pdmodel", "*.__model__", "*.pb", "*.xml", + "*.tflite"]) logger.info(f"Stating model dumping from {self._model_path}") cmd = f'{subgraph_dumper_path} --input_folders="{self._model_path}" --output_folder="{conformance_ir_path}"' process = Popen(cmd, shell=True) @@ -172,11 +182,12 @@ def __dump_subgraph(self): save_rel_weights(Path(self._model_path), op_rel_weight) logger.info(f"All conformance IRs in {self._model_path} were renamed based on hash") else: - logger.warning("The OV Python was not built or Environment was not updated to requirments. Skip the step to rename Conformance IR based on a hash") - + logger.warning( + "The OV Python was not built or Environment was not updated to requirements. " + "Skip the step to rename Conformance IR based on a hash") @staticmethod - def __get_failed_test_from_csv(csv_file:str): + def __get_failed_test_from_csv(csv_file: str): failures = set() with open(csv_file, "r") as failures_file: for row in csv.reader(failures_file, delimiter=','): @@ -212,7 +223,7 @@ def __run_conformance(self): conformance_path = os.path.join(self._ov_path, f'{API_CONFORMANCE_BIN_NAME}{constants.OS_BIN_FILE_EXT}') if not os.path.isfile(conformance_path): - logger.error(f"{conformance_path} is not exist!") + logger.error(f"{conformance_path} does not exist!") exit(-1) logs_dir = os.path.join(self._working_dir, f'{self._device}_logs') @@ -241,7 +252,7 @@ def __run_conformance(self): is_parallel_devices=self._is_parallel_over_devices, excluded_tests=self._expected_failures if not self._expected_failures_update else set()) conformance.run() - conformance.postprocess_logs() + self.is_successful_run = conformance.postprocess_logs() if os.path.isfile(self._expected_failures_file): self.__check_expected_failures() @@ -249,13 +260,15 @@ def __run_conformance(self): final_report_name = f'report_{self._type.lower()}' merge_xml([parallel_report_dir], report_dir, final_report_name, self._type, True) - logger.info(f"Conformance is successful. XML reportwas saved to {report_dir}") - return (os.path.join(report_dir, final_report_name + ".xml"), report_dir) + logger.info(f"Conformance is successful. XML report was saved to {report_dir}") + return os.path.join(report_dir, final_report_name + ".xml"), report_dir - def __summarize(self, xml_report_path:os.path, report_dir: os.path): + def __summarize(self, xml_report_path: os.path, report_dir: os.path): if self._type == constants.OP_CONFORMANCE: summary_root = ET.parse(xml_report_path).getroot() - rel_weights_path = os.path.join(self._model_path, constants.REL_WEIGHTS_FILENAME.replace(constants.REL_WEIGHTS_REPLACE_STR, self._special_mode)) + rel_weights_path = os.path.join(self._model_path, + constants.REL_WEIGHTS_FILENAME.replace(constants.REL_WEIGHTS_REPLACE_STR, + self._special_mode)) create_summary(summary_root, report_dir, [], "", "", True, True, rel_weights_path) else: create_api_summary([xml_report_path], report_dir, [], "", "") @@ -303,7 +316,7 @@ def run(self, dump_models: bool): if dump_models: self.__dump_subgraph() if not os.path.exists(self._model_path): - logger.error(f"The model direstory {self._model_path} does not exist!") + logger.error(f"The model directory {self._model_path} does not exist!") exit(-1) if not os.path.exists(self._model_path): logger.error(f"Directory {self._model_path} does not exist") @@ -311,6 +324,7 @@ def run(self, dump_models: bool): xml_report, report_dir = self.__run_conformance() self.__summarize(xml_report, report_dir) + if __name__ == "__main__": args = parse_arguments() conformance = Conformance(args.device, args.models_path, @@ -321,4 +335,5 @@ def run(self, dump_models: bool): args.parallel_devices, args.expected_failures, args.expected_failures_update) conformance.run(args.dump_graph) - + if not conformance.is_successful_run: + exit(-1)