From cf64636ec76ad4ad3ac8b614378af37927fb834e Mon Sep 17 00:00:00 2001 From: Sergey Lyubimtsev Date: Tue, 1 Jun 2021 16:18:34 +0300 Subject: [PATCH 1/6] enable make clean to remove ie_wheel artifacts --- .../ie_bridges/python/wheel/CMakeLists.txt | 7 ++++++- .../ie_bridges/python/wheel/setup.py | 20 +++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/inference-engine/ie_bridges/python/wheel/CMakeLists.txt b/inference-engine/ie_bridges/python/wheel/CMakeLists.txt index bd6d6e80d72b57..99e378b039734c 100644 --- a/inference-engine/ie_bridges/python/wheel/CMakeLists.txt +++ b/inference-engine/ie_bridges/python/wheel/CMakeLists.txt @@ -54,10 +54,15 @@ endif() add_custom_command(TARGET ie_wheel POST_BUILD COMMAND ${PYTHON_EXECUTABLE} ${SETUP_PY} bdist_wheel - --dist-dir ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/wheels + --dist-dir ${OUTPUT_ROOT}/wheels --build=${WHEEL_BUILD} --plat-name=${WHEEL_PLATFORM} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" COMMENT "Building Python wheel ${WHEEL_PACKAGE_NAME}" VERBATIM ) + +set_property(TARGET ie_wheel + APPEND + PROPERTY ADDITIONAL_CLEAN_FILES ${OUTPUT_ROOT}/wheels +) diff --git a/inference-engine/ie_bridges/python/wheel/setup.py b/inference-engine/ie_bridges/python/wheel/setup.py index f0d88da3753db9..932a9ba8d7cca5 100644 --- a/inference-engine/ie_bridges/python/wheel/setup.py +++ b/inference-engine/ie_bridges/python/wheel/setup.py @@ -8,7 +8,7 @@ import subprocess # nosec import typing from pathlib import Path -from shutil import copyfile +from shutil import copyfile, rmtree from distutils.command.install import install from distutils.command.build import build from distutils.errors import DistutilsSetupError @@ -30,6 +30,7 @@ NGRAPH_LIBS_DIR = config('NGRAPH_LIBS_DIR', '') TBB_LIBS_DIR = config('TBB_LIBS_DIR', '') PY_PACKAGES_DIR = config('PY_PACKAGES_DIR', '') +SITE_PACKAGES_DIR = 'site-packages' LIBS_RPATH = '$ORIGIN' if sys.platform == 'linux' else '@loader_path' LIB_INSTALL_CFG = { @@ -128,15 +129,27 @@ def run(self): self.run_command('build_clib') install.run(self) - class PrepareLibs(build_clib): """Prepare prebuilt libraries""" def run(self): + self.clean({'build_dir': {'prefix': 'build'}, + 'dist_dir': {'prefix': 'dist'}}) + self.clean(LIB_INSTALL_CFG) + self.clean(PY_INSTALL_CFG) + self.configure(LIB_INSTALL_CFG) self.configure(PY_INSTALL_CFG) self.generate_package(get_dir_list(LIB_INSTALL_CFG)) + def clean(self, install_cfg): + """Clean up staging directories""" + for comp, comp_data in install_cfg.items(): + install_prefix = comp_data.get('prefix') + self.announce(f'Cleaning {comp}: {install_prefix}', level=3) + if os.path.exists(install_prefix): + rmtree(install_prefix) + def configure(self, install_cfg): """Collect prebuilt libraries. Install them to the temp directories, set rpath.""" for comp, comp_data in install_cfg.items(): @@ -330,6 +343,9 @@ def get_package_dir(install_cfg): if os.path.exists(package_license): copyfile(package_license, 'LICENSE') +if os.path.exists(SITE_PACKAGES_DIR): + rmtree(SITE_PACKAGES_DIR) + packages = find_namespace_packages(','.join(get_dir_list(PY_INSTALL_CFG))) package_data: typing.Dict[str, list] = {} From dc26d38a00fa5a9d6df61b02abc3d8681ba4b778 Mon Sep 17 00:00:00 2001 From: Sergey Lyubimtsev Date: Tue, 1 Jun 2021 16:25:43 +0300 Subject: [PATCH 2/6] ./setup.py:132:1: E302 expected 2 blank lines --- inference-engine/ie_bridges/python/wheel/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/inference-engine/ie_bridges/python/wheel/setup.py b/inference-engine/ie_bridges/python/wheel/setup.py index 932a9ba8d7cca5..e85f48df95ea7b 100644 --- a/inference-engine/ie_bridges/python/wheel/setup.py +++ b/inference-engine/ie_bridges/python/wheel/setup.py @@ -129,6 +129,7 @@ def run(self): self.run_command('build_clib') install.run(self) + class PrepareLibs(build_clib): """Prepare prebuilt libraries""" From e139c357567dabcdde02dde7cc387f8f0582028e Mon Sep 17 00:00:00 2001 From: Sergey Lyubimtsev Date: Tue, 1 Jun 2021 19:46:01 +0300 Subject: [PATCH 3/6] fix CI build issue --- .../ie_bridges/python/wheel/CMakeLists.txt | 2 +- .../ie_bridges/python/wheel/setup.py | 36 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/inference-engine/ie_bridges/python/wheel/CMakeLists.txt b/inference-engine/ie_bridges/python/wheel/CMakeLists.txt index 99e378b039734c..05b935ed44d955 100644 --- a/inference-engine/ie_bridges/python/wheel/CMakeLists.txt +++ b/inference-engine/ie_bridges/python/wheel/CMakeLists.txt @@ -53,7 +53,7 @@ endif() add_custom_command(TARGET ie_wheel POST_BUILD - COMMAND ${PYTHON_EXECUTABLE} ${SETUP_PY} bdist_wheel + COMMAND ${PYTHON_EXECUTABLE} ${SETUP_PY} clean bdist_wheel --dist-dir ${OUTPUT_ROOT}/wheels --build=${WHEEL_BUILD} --plat-name=${WHEEL_PLATFORM} diff --git a/inference-engine/ie_bridges/python/wheel/setup.py b/inference-engine/ie_bridges/python/wheel/setup.py index e85f48df95ea7b..ca468b138be48e 100644 --- a/inference-engine/ie_bridges/python/wheel/setup.py +++ b/inference-engine/ie_bridges/python/wheel/setup.py @@ -11,6 +11,7 @@ from shutil import copyfile, rmtree from distutils.command.install import install from distutils.command.build import build +from distutils.command.clean import clean from distutils.errors import DistutilsSetupError from distutils.file_util import copy_file from distutils import log @@ -30,7 +31,6 @@ NGRAPH_LIBS_DIR = config('NGRAPH_LIBS_DIR', '') TBB_LIBS_DIR = config('TBB_LIBS_DIR', '') PY_PACKAGES_DIR = config('PY_PACKAGES_DIR', '') -SITE_PACKAGES_DIR = 'site-packages' LIBS_RPATH = '$ORIGIN' if sys.platform == 'linux' else '@loader_path' LIB_INSTALL_CFG = { @@ -134,23 +134,10 @@ class PrepareLibs(build_clib): """Prepare prebuilt libraries""" def run(self): - self.clean({'build_dir': {'prefix': 'build'}, - 'dist_dir': {'prefix': 'dist'}}) - self.clean(LIB_INSTALL_CFG) - self.clean(PY_INSTALL_CFG) - self.configure(LIB_INSTALL_CFG) self.configure(PY_INSTALL_CFG) self.generate_package(get_dir_list(LIB_INSTALL_CFG)) - def clean(self, install_cfg): - """Clean up staging directories""" - for comp, comp_data in install_cfg.items(): - install_prefix = comp_data.get('prefix') - self.announce(f'Cleaning {comp}: {install_prefix}', level=3) - if os.path.exists(install_prefix): - rmtree(install_prefix) - def configure(self, install_cfg): """Collect prebuilt libraries. Install them to the temp directories, set rpath.""" for comp, comp_data in install_cfg.items(): @@ -211,6 +198,24 @@ def run(self): copy_file(src, dst, verbose=self.verbose, dry_run=self.dry_run) +class CustomClean(clean): + """Clean up staging directories""" + + def clean(self, install_cfg): + for comp, comp_data in install_cfg.items(): + install_prefix = comp_data.get('prefix') + self.announce(f'Cleaning {comp}: {install_prefix}', level=3) + if os.path.exists(install_prefix): + rmtree(install_prefix) + + def run(self): + self.clean({'build_dir': {'prefix': 'build'}, + 'dist_dir': {'prefix': 'dist'}}) + self.clean(LIB_INSTALL_CFG) + self.clean(PY_INSTALL_CFG) + clean.run(self) + + def is_tool(name): """Check if the command-line tool is available""" try: @@ -344,8 +349,6 @@ def get_package_dir(install_cfg): if os.path.exists(package_license): copyfile(package_license, 'LICENSE') -if os.path.exists(SITE_PACKAGES_DIR): - rmtree(SITE_PACKAGES_DIR) packages = find_namespace_packages(','.join(get_dir_list(PY_INSTALL_CFG))) package_data: typing.Dict[str, list] = {} @@ -367,6 +370,7 @@ def get_package_dir(install_cfg): 'install': CustomInstall, 'build_clib': PrepareLibs, 'build_ext': CopyExt, + 'clean': CustomClean, }, ext_modules=find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG)), packages=packages, From 69c8071b5e1ef2f55c0856b2e4438be9e476aeaf Mon Sep 17 00:00:00 2001 From: Sergey Lyubimtsev Date: Wed, 2 Jun 2021 18:42:14 +0300 Subject: [PATCH 4/6] set ouput dir for whl file to CMAKE_BINARY_DIR --- inference-engine/ie_bridges/python/wheel/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/inference-engine/ie_bridges/python/wheel/CMakeLists.txt b/inference-engine/ie_bridges/python/wheel/CMakeLists.txt index 05b935ed44d955..93830d68d40b8f 100644 --- a/inference-engine/ie_bridges/python/wheel/CMakeLists.txt +++ b/inference-engine/ie_bridges/python/wheel/CMakeLists.txt @@ -53,8 +53,9 @@ endif() add_custom_command(TARGET ie_wheel POST_BUILD + COMMAND ${CMAKE_COMMAND} -E rm -rf "${CMAKE_CURRENT_BINARY_DIR}/site-packages" COMMAND ${PYTHON_EXECUTABLE} ${SETUP_PY} clean bdist_wheel - --dist-dir ${OUTPUT_ROOT}/wheels + --dist-dir ${CMAKE_BINARY_DIR}/wheels --build=${WHEEL_BUILD} --plat-name=${WHEEL_PLATFORM} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" @@ -64,5 +65,5 @@ add_custom_command(TARGET ie_wheel set_property(TARGET ie_wheel APPEND - PROPERTY ADDITIONAL_CLEAN_FILES ${OUTPUT_ROOT}/wheels + PROPERTY ADDITIONAL_CLEAN_FILES ${CMAKE_BINARY_DIR}/wheels ) From b0dde313cf87da0590b72f45767fad8d09791bbb Mon Sep 17 00:00:00 2001 From: Sergey Lyubimtsev Date: Wed, 2 Jun 2021 18:55:07 +0300 Subject: [PATCH 5/6] cleanup --- inference-engine/ie_bridges/python/wheel/setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/inference-engine/ie_bridges/python/wheel/setup.py b/inference-engine/ie_bridges/python/wheel/setup.py index ca468b138be48e..9e24ea7a17eef8 100644 --- a/inference-engine/ie_bridges/python/wheel/setup.py +++ b/inference-engine/ie_bridges/python/wheel/setup.py @@ -161,6 +161,7 @@ def generate_package(self, src_dirs): # additional blacklist filter, just to fix cmake install issues blacklist = ['.lib', '.pdb', '_debug.dll', '_debug.dylib'] package_dir = os.path.join(get_package_dir(PY_INSTALL_CFG), WHEEL_LIBS_INSTALL_DIR) + for src_dir in src_dirs: local_base_dir = Path(src_dir) for file_path in local_base_dir.rglob('*'): @@ -209,8 +210,6 @@ def clean(self, install_cfg): rmtree(install_prefix) def run(self): - self.clean({'build_dir': {'prefix': 'build'}, - 'dist_dir': {'prefix': 'dist'}}) self.clean(LIB_INSTALL_CFG) self.clean(PY_INSTALL_CFG) clean.run(self) From 94517ad0f438bfeef7c2b8c6b32633f0f57f529b Mon Sep 17 00:00:00 2001 From: Sergey Lyubimtsev Date: Wed, 2 Jun 2021 18:58:27 +0300 Subject: [PATCH 6/6] blank line --- inference-engine/ie_bridges/python/wheel/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference-engine/ie_bridges/python/wheel/setup.py b/inference-engine/ie_bridges/python/wheel/setup.py index 9e24ea7a17eef8..dc177be7d31375 100644 --- a/inference-engine/ie_bridges/python/wheel/setup.py +++ b/inference-engine/ie_bridges/python/wheel/setup.py @@ -161,7 +161,7 @@ def generate_package(self, src_dirs): # additional blacklist filter, just to fix cmake install issues blacklist = ['.lib', '.pdb', '_debug.dll', '_debug.dylib'] package_dir = os.path.join(get_package_dir(PY_INSTALL_CFG), WHEEL_LIBS_INSTALL_DIR) - + for src_dir in src_dirs: local_base_dir = Path(src_dir) for file_path in local_base_dir.rglob('*'):