Conditional compilation test can run from open source (#5629)

tests/conditional_compilation/README.md

@@ -0,0 +1,31 @@
+# Conditional compilation tests
+
+This folder contains conditional compilation (CC) test framework code and CC tests file.
+
+## Environment preparation:
+Install Python modules required for tests:
+```bash
+pip3 install -r requirements.txt 
+```
+
+## Run tests
+
+```bash
+pytest test_cc.py
+```
+**Test parameters:**
+- `sea_runtool` - path to `sea_runtool.py` file.
+- `collector_dir` - path to collector file parent folder.
+- `artifacts` - Path to directory where test write output or read input.
+- `openvino_root_dir` - Path to OpenVINO repo root directory.
+
+**Optional:**
+- `test_conf` - path to test cases .yml config.
+- `openvino_ref` - Path to root directory with installed OpenVINO. If the option is not specified, CC test firstly build and install
+    instrumented package at `<artifacts>/ref_pkg` folder with OpenVINO repository specified in `--openvino_root_dir` option.
+    > If OpenVINO instrumented package has been successfuly installed, in the future you can set `--openvino_ref` parameter as `<artifacts>/ref_pkg` for better performance.
+
+**Sample usage:**
+```bash
+pytest test_cc.py --sea_runtool=./thirdparty/itt_collector/runtool/sea_runtool.py --collector_dir=./bin/intel64/Release --artifacts=../artifacts --openvino_root_dir=.
+```

tests/conditional_compilation/conftest.py

@@ -5,25 +5,24 @@
 
 # pylint: disable=line-too-long
 
-""" Pytest configuration for compilation tests.
-
-Sample usage:
-python3 -m pytest --test_conf=<path to test config> \
-    --sea_runtool=./thirdparty/itt_collector/runtool/sea_runtool.py --artifacts ./compiled test_collect.py \
-    --collector_dir=./bin/intel64/Release --artifacts=<path to directory where tests write output or read input> \
-    --openvino_ref=<Path to root directory with installed OpenVINO>
-"""
+"""Pytest configuration for compilation tests."""
 
+import logging
 import sys
-import pytest
-import yaml
 from inspect import getsourcefile
 from pathlib import Path
 
 # add ../lib to imports
 sys.path.insert(0, str((Path(getsourcefile(lambda: 0)) / ".." / ".." / "lib").resolve(strict=True)))
 
+import yaml
+import pytest
+
 from path_utils import expand_env_vars  # pylint: disable=import-error
+from test_utils import make_build, validate_path_arg, write_session_info, SESSION_INFO_FILE  # pylint: disable=import-error
+
+
+log = logging.getLogger()
 
 
 def pytest_addoption(parser):
@@ -32,7 +31,7 @@ def pytest_addoption(parser):
         "--test_conf",
         type=Path,
         default=Path(__file__).parent / "test_config.yml",
-        help="Path to models root directory"
+        help="Path to models root directory",
     )
     parser.addoption(
         "--sea_runtool",
@@ -56,14 +55,19 @@ def pytest_addoption(parser):
         type=Path,
         help="Path to root directory with installed OpenVINO",
     )
+    parser.addoption(
+        "--openvino_root_dir",
+        type=Path,
+        help="Path to OpenVINO repository root directory",
+    )
 
 
 def pytest_generate_tests(metafunc):
     """Generate tests depending on command line options."""
     params = []
     ids = []
 
-    with open(metafunc.config.getoption('test_conf'), "r") as file:
+    with open(metafunc.config.getoption("test_conf"), "r") as file:
         test_cases = yaml.safe_load(file)
 
     for test in test_cases:
@@ -72,7 +76,7 @@ def pytest_generate_tests(metafunc):
         if "marks" in test:
             extra_args["marks"] = test["marks"]
 
-        test_id = model_path.replace('$', '').replace('{', '').replace('}', '')
+        test_id = model_path.replace("$", "").replace("{", "").replace("}", "")
         params.append(pytest.param(test_id, Path(expand_env_vars(model_path)), **extra_args))
         ids = ids + [test_id]
     metafunc.parametrize("test_id, model", params, ids=ids)
@@ -81,13 +85,19 @@ def pytest_generate_tests(metafunc):
 @pytest.fixture(scope="session")
 def sea_runtool(request):
     """Fixture function for command-line option."""
-    return request.config.getoption("sea_runtool")
+    sea_runtool = request.config.getoption("sea_runtool", skip=True)
+    validate_path_arg(sea_runtool)
+
+    return sea_runtool
 
 
 @pytest.fixture(scope="session")
 def collector_dir(request):
     """Fixture function for command-line option."""
-    return request.config.getoption("collector_dir")
+    collector_dir = request.config.getoption("collector_dir", skip=True)
+    validate_path_arg(collector_dir, is_dir=True)
+
+    return collector_dir
 
 
 @pytest.fixture(scope="session")
@@ -99,4 +109,59 @@ def artifacts(request):
 @pytest.fixture(scope="session")
 def openvino_root_dir(request):
     """Fixture function for command-line option."""
-    return request.config.getoption("openvino_ref")
+    openvino_root_dir = request.config.getoption("openvino_root_dir", skip=True)
+    validate_path_arg(openvino_root_dir, is_dir=True)
+
+    return openvino_root_dir
+
+
+@pytest.fixture(scope="session")
+def openvino_ref(request, artifacts):
+    """Fixture function for command-line option.
+    Return path to root directory with installed OpenVINO.
+    If --openvino_ref command-line option is not specified firstly build and install
+    instrumented package with OpenVINO repository specified in --openvino_root_dir option.
+    """
+    openvino_ref = request.config.getoption("openvino_ref")
+    if openvino_ref:
+        validate_path_arg(openvino_ref, is_dir=True)
+
+        return openvino_ref
+
+    openvino_root_dir = request.config.getoption("openvino_root_dir", skip=True)
+    validate_path_arg(openvino_root_dir, is_dir=True)
+
+    build_dir = openvino_root_dir / "build_instrumented"
+    openvino_ref_path = artifacts / "ref_pkg"
+
+    log.info("--openvino_ref is not specified. Preparing instrumented build at %s", build_dir)
+
+    return_code, output = make_build(
+        openvino_root_dir,
+        build_dir,
+        openvino_ref_path,
+        cmake_additional_args=["-DSELECTIVE_BUILD=COLLECT"],
+        log=log
+    )
+    assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
+
+    return openvino_ref_path
+
+
+@pytest.fixture(scope="function")
+def test_info(request, pytestconfig):
+    """Fixture function for getting the additional attributes of the current test."""
+    setattr(request.node._request, "test_info", {})
+    if not hasattr(pytestconfig, "session_info"):
+        setattr(pytestconfig, "session_info", [])
+
+    yield request.node._request.test_info
+
+    pytestconfig.session_info.append(request.node._request.test_info)
+
+
+@pytest.fixture(scope="session")
+def save_session_info(pytestconfig, artifacts):
+    """Fixture function for saving additional attributes to configuration file."""
+    yield
+    write_session_info(path=artifacts / SESSION_INFO_FILE, data=pytestconfig.session_info)

tests/conditional_compilation/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = --ignore-unknown-dependency

tests/conditional_compilation/requirements.txt

@@ -0,0 +1 @@
+pytest-dependency==0.5.1

tests/conditional_compilation/test_cc.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2018-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+""" Test conditional compilation.
+"""
+
+import glob
+import logging
+import os
+import sys
+import numpy as np
+import pytest
+
+from proc_utils import cmd_exec  # pylint: disable=import-error
+from test_utils import get_lib_sizes, infer_tool, make_build, run_infer  # pylint: disable=import-error
+
+
+log = logging.getLogger()
+
+
+@pytest.mark.dependency(name="cc_collect")
+def test_cc_collect(test_id, model, openvino_ref, test_info,
+                    save_session_info, sea_runtool, collector_dir, artifacts):  # pylint: disable=unused-argument
+    """Test conditional compilation statistics collection
+    :param test_info: custom `test_info` field of built-in `request` pytest fixture.
+                      contain a dictionary to store test metadata.
+    """
+    out = artifacts / test_id
+    test_info["test_id"] = test_id
+    # cleanup old data if any
+    prev_result = glob.glob(f"{out}.pid*.csv")
+    for path in prev_result:
+        os.remove(path)
+    # run use case
+    sys_executable = (
+        os.path.join(sys.prefix, "python.exe")
+        if sys.platform == "win32"
+        else os.path.join(sys.prefix, "bin", "python")
+    )
+    return_code, output = cmd_exec(
+        [
+            sys_executable,
+            str(sea_runtool),
+            f"--output={out}",
+            f"--bindir={collector_dir}",
+            "!",
+            sys_executable,
+            infer_tool,
+            f"-m={model}",
+            "-d=CPU",
+            f"-r={out}",
+        ]
+    )
+    out_csv = glob.glob(f"{out}.pid*.csv")
+    test_info["out_csv"] = out_csv
+
+    assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
+    assert len(out_csv) == 1, f'Multiple or none "{out}.pid*.csv" files'
+
+
+@pytest.mark.dependency(depends=["cc_collect"])
+def test_minimized_pkg(test_id, model, openvino_root_dir, artifacts):  # pylint: disable=unused-argument
+    """Build and install OpenVINO package with collected conditional compilation statistics."""
+    out = artifacts / test_id
+    install_prefix = out / "install_pkg"
+    build_dir = openvino_root_dir / "build_minimized"
+
+    out_csv = glob.glob(f"{out}.pid*.csv")
+    assert len(out_csv) == 1, f'Multiple or none "{out}.pid*.csv" files'
+
+    log.info("Building minimized build at %s", build_dir)
+
+    return_code, output = make_build(
+        openvino_root_dir,
+        build_dir,
+        install_prefix,
+        cmake_additional_args=[f"-DSELECTIVE_BUILD_STAT={out_csv[0]}"],
+        log=log,
+    )
+    assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
+
+
+@pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"])
+def test_infer(test_id, model, artifacts):
+    """Test inference with conditional compiled binaries."""
+    out = artifacts / test_id
+    minimized_pkg = out / "install_pkg"
+    return_code, output = run_infer(model, f"{out}_cc.npz", minimized_pkg)
+    assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
+
+
+@pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"])
+def test_verify(test_id, model, openvino_ref, artifacts, tolerance=1e-6):  # pylint: disable=too-many-arguments
+    """Test verifying that inference results are equal."""
+    out = artifacts / test_id
+    minimized_pkg = out / "install_pkg"
+    out_file = f"{out}.npz"
+    out_file_cc = f"{out}_cc.npz"
+    return_code, output = run_infer(model, out_file, openvino_ref)
+    assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
+    return_code, output = run_infer(model, out_file_cc, minimized_pkg)
+    assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
+    reference_results = dict(np.load(out_file))
+    inference_results = dict(np.load(out_file_cc))
+    assert sorted(reference_results.keys()) == sorted(
+        inference_results.keys()
+    ), "Results have different number of layers"
+    for layer in reference_results.keys():
+        assert np.allclose(
+            reference_results[layer], inference_results[layer], tolerance
+        ), "Reference and inference results differ"
+
+
+@pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"])
+def test_libs_size(test_id, model, openvino_ref, artifacts):  # pylint: disable=unused-argument
+    """Test if libraries haven't increased in size after conditional compilation."""
+    libraries = ["inference_engine_transformations", "MKLDNNPlugin", "ngraph"]
+    minimized_pkg = artifacts / test_id / "install_pkg"
+    ref_libs_size = get_lib_sizes(openvino_ref, libraries)
+    lib_sizes = get_lib_sizes(minimized_pkg, libraries)
+
+    for lib in libraries:
+        lib_size_diff = ref_libs_size[lib] - lib_sizes[lib]
+        lib_size_diff_percent = lib_size_diff / ref_libs_size[lib] * 100
+        log.info(
+            "{}: old - {}kB; new - {}kB; diff = {}kB({:.2f}%)".format(
+                lib,
+                ref_libs_size[lib] / 1024,
+                lib_sizes[lib] / 1024,
+                lib_size_diff / 1024,
+                lib_size_diff_percent,
+            )
+        )
+    res = [lib for lib in libraries if lib_sizes[lib] > ref_libs_size[lib]]
+    assert len(res) == 0, f"These libraries: {res} have increased in size!"