[Build] Added PYI files check (openvinotoolkit#1214)

ilya-lavrenov · Dec 3, 2024 · 76bf49e · 76bf49e
1 parent a7cda8f
commit 76bf49e
Show file tree

Hide file tree

Showing 7 changed files with 203 additions and 19 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -25,12 +25,30 @@ if(POLICY CMP0169)
     cmake_policy(SET CMP0169 OLD)
 endif()
 
+if(UNIX AND NOT (APPLE OR ANDROID OR CYGWIN))
+    set(LINUX ON)
+endif()
+
 project(OpenVINOGenAI
         VERSION 2024.5.0.0
         DESCRIPTION "OpenVINO GenAI"
         HOMEPAGE_URL "https://github.com/openvinotoolkit/openvino.genai"
         LANGUAGES CXX C)
 
+if(NOT DEFINED Python3_FIND_VIRTUALENV)
+    set(Python3_FIND_VIRTUALENV FIRST)
+endif()
+
+# Looking for OpenVINO in the python distribution. It doesn't work for cross-compiling build
+if(NOT CMAKE_CROSSCOMPILING)
+    find_package(Python3 REQUIRED)
+    execute_process(
+        COMMAND ${Python3_EXECUTABLE} -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')"
+        OUTPUT_VARIABLE OpenVINO_DIR_PY
+        ERROR_QUIET
+    )
+endif()
+
 # Find OpenVINODeveloperPackage first to compile with SDL flags
 find_package(OpenVINODeveloperPackage ${OpenVINOGenAI_VERSION} QUIET
              COMPONENTS Runtime Threading
@@ -44,7 +62,7 @@ include(cmake/features.cmake)
 
 if(ENABLE_PYTHON)
     # the following two calls are required for cross-compilation
-    if(OpenVINODeveloperPackage_DIR)
+    if(OpenVINODeveloperPackage_FOUND)
         ov_find_python3(REQUIRED)
         ov_detect_python_module_extension()
     else()
@@ -62,9 +80,15 @@ endif()
 
 add_subdirectory(thirdparty)
 add_subdirectory(src)
-add_subdirectory(samples)
-add_subdirectory(tools/continuous_batching)
-add_subdirectory(tests/cpp)
+if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/samples")
+    add_subdirectory(samples)
+endif()
+if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/tools/continuous_batching")
+    add_subdirectory(tools/continuous_batching)
+endif()
+if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/tests/cpp")
+    add_subdirectory(tests/cpp)
+endif()
 
 install(FILES LICENSE DESTINATION docs/licensing COMPONENT licensing_genai RENAME LICENSE-GENAI)
 install(FILES third-party-programs.txt DESTINATION docs/licensing COMPONENT licensing_genai RENAME third-party-programs-genai.txt)

diff --git a/pyproject.toml b/pyproject.toml
@@ -22,22 +22,24 @@ dependencies = [
 directory = "src/python"
 
 [tool.py-build-cmake.sdist]
-exclude = ["tools", "samples", "tests", "thirdparty"]
+include = ["CMakeLists.txt", "LICENSE", "third-party-programs.txt", "SECURITY.md", "cmake", "src", "thirdparty"]
 
 [tool.py-build-cmake.cmake]
 minimum_version = "3.23"
 build_type = "Release"
 config = ["Release"]
 find_python3 = true
-build_args = ["--parallel", "--target", "py_openvino_genai"]
+build_args = ["--parallel", "--target", "py_openvino_genai_stub"]
 install_args = ["--strip"]
 install_components = ["wheel_genai"]
 options = {"BUILD_TOKENIZERS" = "OFF"}
 
 [build-system]
 requires = [
     "py-build-cmake@git+https://github.com/tttapa/py-build-cmake@7ab73da351c7140f06d727a8705bece4cf544cd9",
-    "cmake~=3.23"
+    "openvino~=2025.0.0.0.dev",
+    "pybind11-stubgen==2.5.1",
+    "cmake~=3.23.0"
 ]
 build-backend = "py_build_cmake.build"
 

diff --git a/requirements-build.txt b/requirements-build.txt
@@ -1 +1,2 @@
-cmake~=3.30
+cmake~=3.23.0
+pybind11-stubgen==2.5.1
diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt
@@ -96,6 +96,21 @@ else()
         SOVERSION ${MAJOR_SUFFIX}${OpenVINOGenAI_VERSION_MINOR}${OpenVINOGenAI_VERSION_PATCH})
 endif()
 
+if(OpenVINODeveloperPackage_FOUND)
+    # must be called after all target_link_libraries
+    # ov_add_api_validator_post_build_step(TARGET ${TARGET_NAME})
+
+    ov_ncc_naming_style(FOR_TARGET ${TARGET_NAME}
+                        SOURCE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/include")
+
+    # TODO: override versions as currently they come from OpenVINO
+    # ov_add_vs_version_file(NAME ${TARGET_NAME}
+    #                        FILEDESCRIPTION "OpenVINO GenAI library")
+
+    # TODO: commit changes separately
+    # ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
+endif()
+
 # - Windows: `<openvino_dir>\runtime\bin\intel64\Release\`
 # - MacOS_x86: `<openvino_dir>/runtime/lib/intel64/Release`
 # - MacOS_arm64: `<openvino_dir>/runtime/lib/arm64/Release/`

diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt
@@ -20,18 +20,28 @@ endif()
 
 file(GLOB python_sources "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp")
 
-pybind11_add_module(py_openvino_genai ${python_sources})
-target_link_libraries(py_openvino_genai PRIVATE openvino::genai)
-target_include_directories(py_openvino_genai PRIVATE "${OpenVINOGenAI_SOURCE_DIR}/src/cpp/src") # for tokenizers_path.hpp
-set_target_properties(py_openvino_genai PROPERTIES
+set(TARGET_NAME py_openvino_genai)
+pybind11_add_module(${TARGET_NAME} ${python_sources})
+
+target_link_libraries(${TARGET_NAME} PRIVATE openvino::genai)
+target_include_directories(${TARGET_NAME} PRIVATE "${OpenVINOGenAI_SOURCE_DIR}/src/cpp/src") # for tokenizers_path.hpp
+set_target_properties(${TARGET_NAME} PROPERTIES
     ARCHIVE_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
     LIBRARY_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
 )
-file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/__init__.py" DESTINATION "${CMAKE_BINARY_DIR}/openvino_genai/")
+file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/__init__.py"
+          "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/__init__.pyi"
+          "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/py_openvino_genai.pyi"
+     DESTINATION "${CMAKE_BINARY_DIR}/openvino_genai/")
 
 configure_file("${OpenVINOGenAI_SOURCE_DIR}/cmake/templates/__version__.py.in"
                "${CMAKE_BINARY_DIR}/openvino_genai/__version__.py" @ONLY)
 
+if(OpenVINODeveloperPackage_FOUND)
+    # TODO: commit changes separately
+    # ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
+endif()
+
 if(DEFINED PY_BUILD_CMAKE_PACKAGE_NAME)
     # RPATH for wheel is mandatory to find openvino_genai library. It
     # must be forced because GenAI may be built with OpenVINO targeting
@@ -53,14 +63,16 @@ elseif(APPLE)
 endif()
 
 if(rpaths)
-    set_target_properties(py_openvino_genai PROPERTIES INSTALL_RPATH "${rpaths}")
+    set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "${rpaths}")
 endif()
 
 install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/__init__.py"
+              "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/__init__.pyi"
+              "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/py_openvino_genai.pyi"
               "${CMAKE_BINARY_DIR}/openvino_genai/__version__.py"
         DESTINATION python/openvino_genai
         COMPONENT pygenai_${Python3_VERSION_MAJOR}_${Python3_VERSION_MINOR})
-install(TARGETS py_openvino_genai
+install(TARGETS ${TARGET_NAME}
         LIBRARY DESTINATION python/openvino_genai
         COMPONENT pygenai_${Python3_VERSION_MAJOR}_${Python3_VERSION_MINOR})
 
@@ -78,6 +90,104 @@ install(FILES "${OpenVINOGenAI_SOURCE_DIR}/LICENSE"
 
 # wheel_genai component is used for wheel generation in pyproject.toml.
 # Exclude wheel_genai from normal packaging because there's pygenai_X_Y component for that.
-install(TARGETS openvino_genai py_openvino_genai
+install(TARGETS openvino_genai ${TARGET_NAME}
         LIBRARY DESTINATION openvino_genai COMPONENT wheel_genai EXCLUDE_FROM_ALL
         RUNTIME DESTINATION openvino_genai COMPONENT wheel_genai EXCLUDE_FROM_ALL)
+
+# Generate or check .pyi stub files generated by pybind11-stub
+
+set(pyproject_toml "${OpenVINOGenAI_SOURCE_DIR}/pyproject.toml")
+file(STRINGS ${pyproject_toml} pybind11_stubgen_dep REGEX "pybind11-stubgen")
+
+if(pybind11_stubgen_dep MATCHES "pybind11-stubgen==[0-9\.]+")
+    set(pybind11_stubgen_dep "${CMAKE_MATCH_0}")
+else()
+    message(FATAL_ERROR "Internal error: failed to parse pybind11-stubgen version from from '${pyproject_toml}'")
+endif()
+
+if(OpenVINODeveloperPackage_FOUND)
+    ov_check_pip_package(REQUIREMENT ${pybind11_stubgen_dep}
+                         RESULT_VAR pybind11_stubgen_AVAILABLE
+                         WARNING_MESSAGE "Please, install ${pybind11_stubgen_dep} if you plan to develop Python OpenVINO GenAI API"
+                         MESSAGE_MODE WARNING)
+elseif(DEFINED PY_BUILD_CMAKE_PACKAGE_NAME AND NOT WIN32)
+    # in case of wheel build, pybind11-stubgen is always available via pyproject.toml's build-system
+    # except Win32 where we have issues with pybind11_stubgen executable which cannot import its own module
+    set(pybind11_stubgen_AVAILABLE ON)
+endif()
+
+# but we also need to check whether OpenVINO is installed
+if(CMAKE_CROSSCOMPILING)
+    # we cannot check OpenVINO during cross-compile
+    set(pybind11_stubgen_AVAILABLE OFF)
+else()
+    execute_process(
+        COMMAND ${Python3_EXECUTABLE} -c "import openvino"
+        RESULT_VARIABLE EXIT_CODE
+        OUTPUT_VARIABLE OUTPUT_TEXT
+        ERROR_VARIABLE ERROR_TEXT)
+
+    # OpenVINO is not available because of import error
+    if(NOT EXIT_CODE EQUAL 0)
+        set(pybind11_stubgen_AVAILABLE OFF)
+    endif()
+endif()
+
+if(pybind11_stubgen_AVAILABLE)
+    if(DEFINED ENV{CI} OR DEFINED ENV{TF_BUILD} OR DEFINED ENV{JENKINS_URL})
+        set(ci_run ON)
+    endif()
+
+    set(stub_files_location "${OpenVINOGenAI_BINARY_DIR}/src/python")
+    set(generated_files ${stub_files_location}/openvino_genai/__init__.pyi
+                        ${stub_files_location}/openvino_genai/py_openvino_genai.pyi)
+    set_source_files_properties(${generated_files} PROPERTIES GENERATED ON)
+
+    if(COMMAND find_host_program)
+        find_host_program(pybind11_stubgen NAMES pybind11-stubgen NO_CACHE REQUIRED)
+    else()
+        find_program(pybind11_stubgen NAMES pybind11-stubgen NO_CACHE REQUIRED)
+    endif()
+
+    if(ci_run)
+        set(validation_command
+            COMMAND "${CMAKE_COMMAND}"
+                -D generated_pyi_files_location=${stub_files_location}
+                -D source_pyi_files_location=${CMAKE_CURRENT_SOURCE_DIR}
+                -P "${CMAKE_CURRENT_SOURCE_DIR}/compare_pyi.cmake")
+        set(validation_dependencies
+            "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/__init__.pyi"
+            "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/py_openvino_genai.pyi")
+    else()
+        set(copy_to_source_command
+            COMMAND "${CMAKE_COMMAND}" -E copy ${generated_files} "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/")
+    endif()
+
+    set(output_file pybind11_stub_gen_completed.txt)
+    add_custom_command(OUTPUT ${output_file}
+        COMMAND "${CMAKE_COMMAND}" -E rm -f "${CMAKE_BINARY_DIR}/openvino_genai/__init__.pyi"
+                                            "${CMAKE_BINARY_DIR}/openvino_genai/py_openvino_genai.pyi"
+        COMMAND "${CMAKE_COMMAND}" -E env PYTHONPATH=${CMAKE_BINARY_DIR}:${openvino_pythonpath}:$ENV{PYTHONPATH}
+                ${pybind11_stubgen} --output-dir ${stub_files_location} openvino_genai
+        ${validation_command}
+        ${copy_to_source_command}
+        COMMAND "${CMAKE_COMMAND}" -E copy ${generated_files} "${CMAKE_BINARY_DIR}/openvino_genai/"
+        COMMAND "${CMAKE_COMMAND}" -E touch ${output_file}
+        DEPENDS
+            ${python_sources}
+            ${validation_dependencies}
+            "${CMAKE_CURRENT_SOURCE_DIR}/openvino_genai/__init__.py"
+            "${CMAKE_CURRENT_SOURCE_DIR}/compare_pyi.cmake"
+        COMMENT "[${pybind11_stubgen_dep}] Generate .pyi files"
+        VERBATIM)
+
+    add_custom_target(${TARGET_NAME}_stub ALL DEPENDS ${output_file})
+else()
+    # Produce warning message at build time as well
+    add_custom_command(OUTPUT pybind11_stub_gen_not_found.txt
+        COMMAND ${CMAKE_COMMAND}
+            -E cmake_echo_color --red "Warning: Please, install ${pybind11_stubgen_dep}")
+    add_custom_target(${TARGET_NAME}_stub ALL DEPENDS pybind11_stub_gen_not_found.txt)
+endif()
+
+add_dependencies(${TARGET_NAME}_stub ${TARGET_NAME})
diff --git a/src/python/compare_pyi.cmake b/src/python/compare_pyi.cmake
@@ -0,0 +1,28 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+foreach(var IN ITEMS generated_pyi_files_location source_pyi_files_location)
+    if(NOT DEFINED ${var})
+        message(FATAL_ERROR "Variable ${var} is not defined")
+    endif()
+endforeach()
+
+file(GLOB_RECURSE pyi_files ${generated_pyi_files_location}/*.pyi)
+
+# perform comparison of generated files with committed ones
+foreach(pyi_file IN LISTS pyi_files)
+    string(REPLACE ${generated_pyi_files_location} ${source_pyi_files_location} commited_pyi_file "${pyi_file}")
+    if(NOT EXISTS "${commited_pyi_file}")
+        message(FATAL_ERROR "${commited_pyi_file} does not exists. Please, install pybind11-stubgen and generate .pyi files")
+    else()
+        execute_process(COMMAND "${CMAKE_COMMAND}" -E compare_files "${pyi_file}" "${commited_pyi_file}"
+                        OUTPUT_VARIABLE output_message
+                        ERROR_VARIABLE error_message
+                        RESULT_VARIABLE exit_code
+                        OUTPUT_STRIP_TRAILING_WHITESPACE)
+        if(NOT exit_code EQUAL 0)
+            message(FATAL_ERROR "File ${commited_pyi_file} is outdated and need to be regenerated with pybind11-stubgen")
+        endif()
+    endif()
+endforeach()
diff --git a/src/python/openvino_genai/py_openvino_genai.pyi b/src/python/openvino_genai/py_openvino_genai.pyi
@@ -706,6 +706,7 @@ class ImageGenerationConfig:
     generator: Generator
     guidance_scale: float
     height: int
+    max_sequence_length: int
     negative_prompt: str | None
     negative_prompt_2: str | None
     negative_prompt_3: str | None
@@ -1267,9 +1268,10 @@ class Text2ImagePipeline:
             height: int - height of resulting images,
             width: int - width of resulting images,
             num_inference_steps: int - number of inference steps,
-            generator: openvino_genai.CppStdGenerator or class inherited from openvino_genai.Generator - random generator
-            adapters: LoRA adapters
-            strength: strength for image to image generation. 1.0f means initial image is fully noised
+            generator: openvino_genai.CppStdGenerator or class inherited from openvino_genai.Generator - random generator,
+            adapters: LoRA adapters,
+            strength: strength for image to image generation. 1.0f means initial image is fully noised,
+            max_sequence_length: int - length of t5_encoder_model input
         
             :return: ov.Tensor with resulting images
             :rtype: ov.Tensor
@@ -1379,6 +1381,8 @@ class UNet2DConditionModel:
                         device (str): Device to run the model on (e.g., CPU, GPU).
                         kwargs: Device properties.
         """
+    def do_classifier_free_guidance(self, guidance_scale: float) -> bool:
+        ...
     def get_config(self) -> UNet2DConditionModel.Config:
         ...
     def infer(self, sample: openvino._pyopenvino.Tensor, timestep: openvino._pyopenvino.Tensor) -> openvino._pyopenvino.Tensor: