From ceb5a4d19f90043c6be21f65eb3193f93f53cff6 Mon Sep 17 00:00:00 2001 From: "alexander.shchepetov" Date: Thu, 20 May 2021 15:21:34 +0300 Subject: [PATCH] Conditional compilation test can run from open source --- inference-engine/thirdparty/mkl-dnn | 2 +- tests/conditional_compilation/conftest.py | 84 ++++++++++++++----- tests/conditional_compilation/test_collect.py | 69 --------------- tests/conditional_compilation/test_infer.py | 18 ---- .../test_size_tracking.py | 41 --------- tests/conditional_compilation/test_verify.py | 29 ------- tests/conditional_compilation/tests_utils.py | 43 ---------- tests/lib/proc_utils.py | 3 +- 8 files changed, 68 insertions(+), 221 deletions(-) delete mode 100644 tests/conditional_compilation/test_collect.py delete mode 100644 tests/conditional_compilation/test_infer.py delete mode 100644 tests/conditional_compilation/test_size_tracking.py delete mode 100644 tests/conditional_compilation/test_verify.py delete mode 100644 tests/conditional_compilation/tests_utils.py diff --git a/inference-engine/thirdparty/mkl-dnn b/inference-engine/thirdparty/mkl-dnn index a5fffb52b012b3..2dd787262134c2 160000 --- a/inference-engine/thirdparty/mkl-dnn +++ b/inference-engine/thirdparty/mkl-dnn @@ -1 +1 @@ -Subproject commit a5fffb52b012b31c65ace894638ecfb8948de9ec +Subproject commit 2dd787262134c20f91f222bfa776225d2dddbc9a diff --git a/tests/conditional_compilation/conftest.py b/tests/conditional_compilation/conftest.py index a11cd981667c26..69fb2620afd1df 100644 --- a/tests/conditional_compilation/conftest.py +++ b/tests/conditional_compilation/conftest.py @@ -5,25 +5,24 @@ # pylint: disable=line-too-long -""" Pytest configuration for compilation tests. - -Sample usage: -python3 -m pytest --test_conf= \ - --sea_runtool=./thirdparty/itt_collector/runtool/sea_runtool.py --artifacts ./compiled test_collect.py \ - --collector_dir=./bin/intel64/Release --artifacts= \ - --openvino_ref= -""" +"""Pytest configuration for compilation tests.""" +import logging import sys -import pytest -import yaml from inspect import getsourcefile from pathlib import Path # add ../lib to imports sys.path.insert(0, str((Path(getsourcefile(lambda: 0)) / ".." / ".." / "lib").resolve(strict=True))) +import yaml +import pytest + from path_utils import expand_env_vars # pylint: disable=import-error +from test_utils import make_build, write_session_info, SESSION_INFO_FILE # pylint: disable=import-error + + +log = logging.getLogger() def pytest_addoption(parser): @@ -32,13 +31,9 @@ def pytest_addoption(parser): "--test_conf", type=Path, default=Path(__file__).parent / "test_config.yml", - help="Path to models root directory" - ) - parser.addoption( - "--sea_runtool", - type=Path, - help="Path to sea_runtool.py" + help="Path to models root directory", ) + parser.addoption("--sea_runtool", type=Path, help="Path to sea_runtool.py") parser.addoption( "--collector_dir", type=Path, @@ -56,6 +51,11 @@ def pytest_addoption(parser): type=Path, help="Path to root directory with installed OpenVINO", ) + parser.addoption( + "--openvino_root_dir", + type=Path, + help="Path to OpenVINO repository root directory", + ) def pytest_generate_tests(metafunc): @@ -63,7 +63,7 @@ def pytest_generate_tests(metafunc): params = [] ids = [] - with open(metafunc.config.getoption('test_conf'), "r") as file: + with open(metafunc.config.getoption("test_conf"), "r") as file: test_cases = yaml.safe_load(file) for test in test_cases: @@ -72,7 +72,7 @@ def pytest_generate_tests(metafunc): if "marks" in test: extra_args["marks"] = test["marks"] - test_id = model_path.replace('$', '').replace('{', '').replace('}', '') + test_id = model_path.replace("$", "").replace("{", "").replace("}", "") params.append(pytest.param(test_id, Path(expand_env_vars(model_path)), **extra_args)) ids = ids + [test_id] metafunc.parametrize("test_id, model", params, ids=ids) @@ -99,4 +99,50 @@ def artifacts(request): @pytest.fixture(scope="session") def openvino_root_dir(request): """Fixture function for command-line option.""" - return request.config.getoption("openvino_ref") + return request.config.getoption("openvino_root_dir") + + +@pytest.fixture(scope="session") +def openvino_ref(request, openvino_root_dir, artifacts): + """Fixture function for command-line option. + Return path to root directory with installed OpenVINO. + If --openvino_ref command-line option is not specified firstly build and install + instrumented package with OpenVINO repository specified in --openvino_root_dir option. + """ + openvino_ref_path = request.config.getoption("openvino_ref") + if openvino_ref_path: + return openvino_ref_path + + build_dir = openvino_root_dir / "build_instrumented" + openvino_ref_path = artifacts / "ref_pkg" + + log.info("--openvino_ref is not specified. " "Building instrumented build at %s", build_dir) + + return_code, output = make_build( + openvino_root_dir, + build_dir, + openvino_ref_path, + cmake_additional_args=["-DSELECTIVE_BUILD=COLLECT"], + log=log + ) + assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}" + + return openvino_ref_path + +@pytest.fixture(scope="function") +def test_info(request, pytestconfig): + """Fixture function for getting the additional attributes of the current test.""" + setattr(request.node._request, "test_info", {}) + if not hasattr(pytestconfig, "session_info"): + setattr(pytestconfig, "session_info", []) + + yield request.node._request.test_info + + pytestconfig.session_info.append(request.node._request.test_info) + + +@pytest.fixture(scope="session") +def save_session_info(pytestconfig, artifacts): + """Fixture function for saving additional attributes to configuration file.""" + yield + write_session_info(path=artifacts / SESSION_INFO_FILE, data=pytestconfig.session_info) diff --git a/tests/conditional_compilation/test_collect.py b/tests/conditional_compilation/test_collect.py deleted file mode 100644 index aa5b0edd52d12e..00000000000000 --- a/tests/conditional_compilation/test_collect.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -""" Test conditional compilation statistics collection. -""" - -import glob -import os -import sys -import pytest - -from proc_utils import cmd_exec # pylint: disable=import-error -from tests_utils import write_session_info, SESSION_INFO_FILE, infer_tool - - -@pytest.fixture(scope="function") -def test_info(request, pytestconfig): - """Fixture function for getting the additional attributes of the current test.""" - setattr(request.node._request, "test_info", {}) - if not hasattr(pytestconfig, "session_info"): - setattr(pytestconfig, "session_info", []) - - yield request.node._request.test_info - - pytestconfig.session_info.append(request.node._request.test_info) - - -@pytest.fixture(scope="session") -def save_session_info(pytestconfig, artifacts): - """Fixture function for saving additional attributes to configuration file.""" - yield - write_session_info(path=artifacts / SESSION_INFO_FILE, data=pytestconfig.session_info) - - -def test_cc_collect(test_id, model, sea_runtool, collector_dir, artifacts, test_info, save_session_info): - """ Test conditional compilation statistics collection - :param test_info: custom `test_info` field of built-in `request` pytest fixture. - contain a dictionary to store test metadata. - """ - out = artifacts / test_id - test_info["test_id"] = test_id - # cleanup old data if any - prev_result = glob.glob(f"{out}.pid*.csv") - for path in prev_result: - os.remove(path) - # run use case - sys_executable = os.path.join(sys.prefix, 'python.exe') if sys.platform == "win32" \ - else os.path.join(sys.prefix, 'bin', 'python') - return_code, output = cmd_exec( - [ - sys_executable, - str(sea_runtool), - f"--output={out}", - f"--bindir={collector_dir}", - "!", - sys_executable, - infer_tool, - f"-m={model}", - "-d=CPU", - f"-r={out}", - ] - ) - out_csv = glob.glob(f"{out}.pid*.csv") - test_info["out_csv"] = out_csv - - assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}" - assert (len(out_csv) == 1), f'Multiple or none "{out}.pid*.csv" files' diff --git a/tests/conditional_compilation/test_infer.py b/tests/conditional_compilation/test_infer.py deleted file mode 100644 index 2bf174127e266e..00000000000000 --- a/tests/conditional_compilation/test_infer.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -""" Test inference with conditional compiled binaries. -""" - -from tests_utils import run_infer - - -def test_infer(test_id, model, artifacts): - """ Test inference with conditional compiled binaries - """ - install_prefix = artifacts / test_id / "install_pkg" - out = artifacts / test_id - returncode, output = run_infer(model, f"{out}_cc.npz", install_prefix) - assert returncode == 0, f"Command exited with non-zero status {returncode}:\n {output}" diff --git a/tests/conditional_compilation/test_size_tracking.py b/tests/conditional_compilation/test_size_tracking.py deleted file mode 100644 index c6185531baec3f..00000000000000 --- a/tests/conditional_compilation/test_size_tracking.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -from pathlib import Path -import logging - -from path_utils import get_lib_path # pylint: disable=import-error - - -def get_lib_sizes(path, libraries): - """Function for getting lib sizes by lib names""" - assert Path.exists(path), f'Directory {path} isn\'t created' - result = {} - error_lib = [] - for lib in libraries: - try: - result[lib] = Path(path).joinpath(get_lib_path(lib)).stat().st_size - except FileNotFoundError as error: - error_lib.append(str(error)) - assert len(error_lib) == 0, 'Following libraries couldn\'t be found: \n{}'.format('\n'.join(error_lib)) - return result - - -def test_size_tracking_libs(openvino_root_dir, test_id, model, artifacts): - log = logging.getLogger('size_tracking') - libraries = ['inference_engine_transformations', 'MKLDNNPlugin', 'ngraph'] - - ref_libs_size = get_lib_sizes(openvino_root_dir, libraries) - install_prefix = artifacts / test_id / 'install_pkg' - lib_sizes = get_lib_sizes(install_prefix, libraries) - - for lib in libraries: - lib_size_diff = ref_libs_size[lib] - lib_sizes[lib] - lib_size_diff_percent = lib_size_diff / ref_libs_size[lib] * 100 - log.info('{}: old - {}kB; new - {}kB; diff = {}kB({:.2f}%)'.format(lib, - ref_libs_size[lib] / 1024, - lib_sizes[lib] / 1024, - lib_size_diff / 1024, - lib_size_diff_percent)) - res = [lib for lib in libraries if lib_sizes[lib] > ref_libs_size[lib]] - assert len(res) == 0, f'These libraries: {res} have increased in size!' diff --git a/tests/conditional_compilation/test_verify.py b/tests/conditional_compilation/test_verify.py deleted file mode 100644 index 685d4f5268716d..00000000000000 --- a/tests/conditional_compilation/test_verify.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -""" Test to verify inference results. -""" - -import numpy as np -from tests_utils import run_infer - - -def test_verify(test_id, model, artifacts, openvino_root_dir, tolerance=1e-6): # pylint: disable=too-many-arguments - """ Test verifying that inference results are equal - """ - out = artifacts / test_id - install_prefix = artifacts / test_id / "install_pkg" - out_file = f"{out}.npz" - out_file_cc = f"{out}_cc.npz" - returncode, output = run_infer(model, out_file, openvino_root_dir) - assert returncode == 0, f"Command exited with non-zero status {returncode}:\n {output}" - returncode, output = run_infer(model, out_file_cc, install_prefix) - assert returncode == 0, f"Command exited with non-zero status {returncode}:\n {output}" - reference_results = dict(np.load(out_file)) - inference_results = dict(np.load(out_file_cc)) - assert sorted(reference_results.keys()) == sorted(inference_results.keys()), \ - "Results have different number of layers" - for layer in reference_results.keys(): - assert np.allclose(reference_results[layer], inference_results[layer], tolerance), \ - "Reference and inference results differ" diff --git a/tests/conditional_compilation/tests_utils.py b/tests/conditional_compilation/tests_utils.py deleted file mode 100644 index 0c6148e4af95ab..00000000000000 --- a/tests/conditional_compilation/tests_utils.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -""" Utility functions for work with json test configuration file. -""" -import os -import json -import sys -from inspect import getsourcefile -from pathlib import Path -from proc_utils import cmd_exec # pylint: disable=import-error -from install_pkg import get_openvino_environment # pylint: disable=import-error - -SESSION_INFO_FILE = "cc_tests.json" -infer_tool = str((Path(getsourcefile(lambda: 0)) / ".." / "tools" / "infer_tool.py").resolve()) - - -def read_session_info(path: Path = Path(getsourcefile(lambda: 0)).parent / SESSION_INFO_FILE): - with open(path, 'r') as json_file: - cc_tests_ids = json.load(json_file) - return cc_tests_ids - - -def write_session_info(path: Path = Path(getsourcefile(lambda: 0)).parent / SESSION_INFO_FILE, - data: dict = None): - with open(path, "w") as json_file: - json.dump(data, json_file, indent=4) - - -def run_infer(model, out_file, install_dir): - """ Function running inference - """ - sys_executable = os.path.join(sys.prefix, 'python.exe') if sys.platform == "win32" \ - else os.path.join(sys.prefix, 'bin', 'python') - returncode, output = cmd_exec( - [sys_executable, - infer_tool, - "-d=CPU", f"-m={model}", f"-r={out_file}" - ], - env=get_openvino_environment(install_dir), - ) - return returncode, output diff --git a/tests/lib/proc_utils.py b/tests/lib/proc_utils.py index 1eaf6a6bed94f6..4b5be22e443fbf 100644 --- a/tests/lib/proc_utils.py +++ b/tests/lib/proc_utils.py @@ -10,7 +10,7 @@ import subprocess -def cmd_exec(args, env=None, log=None, verbose=True): +def cmd_exec(args, env=None, log=None, verbose=True, shell=False): """ Run cmd using subprocess with logging and other improvements """ if log is None: @@ -28,6 +28,7 @@ def cmd_exec(args, env=None, log=None, verbose=True): stderr=subprocess.STDOUT, encoding="utf-8", universal_newlines=True, + shell=shell, ) output = [] for line in iter(proc.stdout.readline, ""):