From 1d58ce72c9f2acc6f3cbc2b4e2c3dcaf8ee58665 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Sun, 17 Sep 2023 02:05:35 +0400 Subject: [PATCH] Removed testdata repo usage --- .ci/azure/linux.yml | 14 -- .ci/azure/linux_arm64.yml | 1 - .ci/azure/linux_cuda.yml | 8 - .ci/azure/linux_debian.yml | 21 --- .ci/azure/linux_lohika.yml | 8 - .ci/azure/mac.yml | 18 --- .ci/azure/windows.yml | 12 -- .github/workflows/code_snippets.yml | 5 +- .github/workflows/code_style.yml | 6 +- .github/workflows/coverage.yml | 4 +- .github/workflows/linux.yml | 16 +- .github/workflows/linux_android_arm64.yml | 10 +- .github/workflows/linux_arm64.yml | 5 +- .../linux_conditional_compilation.yml | 6 +- .github/workflows/linux_debian.yml | 12 +- .github/workflows/mo.yml | 2 +- .github/workflows/py_checks.yml | 2 +- .github/workflows/windows.yml | 23 +-- .../windows_conditional_compilation.yml | 6 +- cmake/dependencies.cmake | 16 -- .../IEDevScriptsConfig.cmake | 1 - .../download/dependency_solver.cmake | 8 - cmake/developer_package/models.cmake | 76 --------- cmake/features.cmake | 2 - install_build_dependencies.sh | 3 +- src/bindings/c/tests/test_model_repo.cpp | 12 ++ src/bindings/c/tests/test_model_repo.hpp | 15 +- .../tests/deprecated/helpers/CMakeLists.txt | 8 - .../deprecated/helpers/test_model_path.cpp | 150 ------------------ .../deprecated/helpers/test_model_path.hpp | 62 -------- .../tests/deprecated/helpers/tests_common.hpp | 1 - .../intel_gna/tests/functional/CMakeLists.txt | 4 - .../functional/helpers/test_model_repo.cpp | 18 +-- .../functional/helpers/test_model_repo.hpp | 27 ---- 34 files changed, 52 insertions(+), 530 deletions(-) delete mode 100644 cmake/developer_package/models.cmake delete mode 100644 src/plugins/intel_gna/tests/deprecated/helpers/test_model_path.cpp delete mode 100644 src/plugins/intel_gna/tests/deprecated/helpers/test_model_path.hpp diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml index 91c6ee5814e6f0..3519952ecd4101 100644 --- a/.ci/azure/linux.yml +++ b/.ci/azure/linux.yml @@ -34,12 +34,6 @@ resources: name: openvinotoolkit/openvino_contrib ref: master - - repository: testdata - type: github - endpoint: openvinotoolkit - name: openvinotoolkit/testdata - ref: master - variables: - group: github @@ -83,7 +77,6 @@ jobs: BUILD_TYPE: Release REPO_DIR: $(Build.Repository.LocalPath) OPENVINO_CONTRIB_REPO_DIR: $(REPO_DIR)/../openvino_contrib - MODELS_PATH: $(REPO_DIR)/../testdata WORK_DIR: $(Pipeline.Workspace)/_w BUILD_DIR: $(WORK_DIR)/build BUILD_SAMPLES_DIR: $(WORK_DIR)/build_samples @@ -216,13 +209,6 @@ jobs: echo SourceBranch: $(Build.SourceBranch) displayName: 'System info' - - # Should be after 'Install dependencies' because Git lfs is not installed - - checkout: testdata - clean: 'true' - lfs: 'true' - path: testdata - - task: CMake@1 inputs: # CMake must get Python 3.x version by default diff --git a/.ci/azure/linux_arm64.yml b/.ci/azure/linux_arm64.yml index 618af65e537ea2..62840d46200de5 100644 --- a/.ci/azure/linux_arm64.yml +++ b/.ci/azure/linux_arm64.yml @@ -195,7 +195,6 @@ jobs: -DPYTHON_MODULE_EXTENSION=$(aarch64-linux-gnu-python3-config --extension-suffix) \ -DPYTHON_LIBRARY=/usr/lib/aarch64-linux-gnu/libc-2.31.so \ -DPYTHON_INCLUDE_DIR=$(Agent.ToolsDirectory)/Python/$(OV_PYTHON_VERSION)/x64/include/python$(OV_PYTHON_VERSION_MAJOR_MINOR) \ - -DENABLE_DATA=OFF \ -DENABLE_TESTS=ON \ -DENABLE_SYSTEM_TBB=ON \ -DENABLE_SYSTEM_PROTOBUF=ON \ diff --git a/.ci/azure/linux_cuda.yml b/.ci/azure/linux_cuda.yml index b92e6e0ba70b79..09e75560937229 100644 --- a/.ci/azure/linux_cuda.yml +++ b/.ci/azure/linux_cuda.yml @@ -39,12 +39,6 @@ resources: name: openvinotoolkit/openvino_contrib ref: master - - repository: testdata - type: github - endpoint: openvinotoolkit - name: openvinotoolkit/testdata - ref: master - jobs: - job: CUDAPlugin_Lin timeoutInMinutes: '60' @@ -60,7 +54,6 @@ jobs: HOME_DIR: $(Agent.HomeDirectory) REPO_DIR: $(Build.Repository.LocalPath) OPENVINO_REPO_DIR: $(REPO_DIR)/../openvino - MODELS_PATH: $(REPO_DIR)/../testdata WORK_DIR: $(Pipeline.Workspace)/_w BUILD_DIR: $(WORK_DIR)/build BIN_DIR: $(OPENVINO_REPO_DIR)/bin/intel64/$(BUILD_TYPE) @@ -137,7 +130,6 @@ jobs: -DENABLE_OV_ONNX_FRONTEND=OFF \ -DENABLE_PYTHON=OFF \ -DENABLE_TESTS=ON \ - -DENABLE_DATA=OFF \ -S /root/repos/openvino \ -B /root/w/build && cmake --build /root/w/build --parallel --config Release --verbose -- ov_nvidia_func_tests ov_nvidia_unit_tests" diff --git a/.ci/azure/linux_debian.yml b/.ci/azure/linux_debian.yml index 690ad5c2c95cb1..96db6129c72ee1 100644 --- a/.ci/azure/linux_debian.yml +++ b/.ci/azure/linux_debian.yml @@ -28,14 +28,6 @@ pr: - '*/conformance/*' - 'tests/layer_tests/*' -resources: - repositories: - - repository: testdata - type: github - endpoint: openvinotoolkit - name: openvinotoolkit/testdata - ref: master - jobs: - job: Lin_Debian # About 150% of total time @@ -50,7 +42,6 @@ jobs: VSTS_HTTP_TIMEOUT: 200 BUILD_TYPE: Release REPO_DIR: $(Build.Repository.LocalPath) - MODELS_PATH: $(REPO_DIR)/../testdata WORK_DIR: $(Pipeline.Workspace)/_w BUILD_DIR: $(WORK_DIR)/build BUILD_SAMPLES_DIR: $(WORK_DIR)/build_samples @@ -149,12 +140,6 @@ jobs: git clone https://github.com/google/gtest-parallel.git displayName: 'Install build dependencies' - # Should be after 'Install dependencies' because Git lfs is not installed - - checkout: testdata - clean: 'true' - lfs: 'true' - path: testdata - - task: CMake@1 inputs: # CMake must get Python 3.x version by default @@ -338,15 +323,9 @@ jobs: displayName: 'TEMPLATE FuncTests' - script: $(INSTALL_TEST_DIR)/InferenceEngineCAPITests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-InferenceEngineCAPITests.xml - env: - DATA_PATH: $(MODELS_PATH) - MODELS_PATH: $(MODELS_PATH) displayName: 'IE CAPITests' - script: $(INSTALL_TEST_DIR)/ov_capi_test --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ov_capi_test.xml - env: - DATA_PATH: $(MODELS_PATH) - MODELS_PATH: $(MODELS_PATH) displayName: 'OV CAPITests' # Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time diff --git a/.ci/azure/linux_lohika.yml b/.ci/azure/linux_lohika.yml index c9be14cc217dc0..5fb3b59129efa3 100644 --- a/.ci/azure/linux_lohika.yml +++ b/.ci/azure/linux_lohika.yml @@ -20,7 +20,6 @@ jobs: # VSTS_HTTP_TIMEOUT: 200 # BUILD_TYPE: Release # REPO_DIR: $(Build.Repository.LocalPath) -# MODELS_PATH: $(REPO_DIR)/../testdata # WORK_DIR: $(Pipeline.Workspace)/_w # BUILD_DIR: $(WORK_DIR)/build @@ -38,13 +37,6 @@ jobs: - script: git -C ~/work/openvino checkout -m $(Build.SourceVersion) && git -C ~/work/openvino submodule update --init --recursive displayName: checkout - # Should be after 'Install dependencies' because Git lfs is not installed -# - checkout: testdata -# clean: 'true' -# submodules: 'true' -# lfs: 'true' -# path: testdata - - script: env -C ~/work ./configreleasenolto.sh displayName: CMake diff --git a/.ci/azure/mac.yml b/.ci/azure/mac.yml index d39b10e536ed57..f1a20fb49b3055 100644 --- a/.ci/azure/mac.yml +++ b/.ci/azure/mac.yml @@ -37,12 +37,6 @@ resources: name: openvinotoolkit/openvino_contrib ref: master - - repository: testdata - type: github - endpoint: openvinotoolkit - name: openvinotoolkit/testdata - ref: master - variables: - group: github @@ -61,7 +55,6 @@ jobs: BUILD_TYPE: Release REPO_DIR: $(Build.Repository.LocalPath) OPENVINO_CONTRIB_REPO_DIR: $(REPO_DIR)/../openvino_contrib - MODELS_PATH: $(REPO_DIR)/../testdata WORK_DIR: $(Pipeline.Workspace)/_w BUILD_DIR: $(WORK_DIR)/build INSTALL_DIR: $(WORK_DIR)/install_pkg @@ -109,11 +102,6 @@ jobs: submodules: 'true' path: openvino_contrib - - checkout: testdata - clean: 'true' - lfs: 'true' - path: testdata - - script: | set -e brew install cython automake @@ -219,17 +207,11 @@ jobs: - script: | $(SETUPVARS) && $(INSTALL_TEST_DIR)/InferenceEngineCAPITests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-InferenceEngineCAPITests.xml - env: - DATA_PATH: $(MODELS_PATH) - MODELS_PATH: $(MODELS_PATH) displayName: 'IE CAPITests' enabled: 'false' - script: | $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_capi_test --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ov_capi_test.xml - env: - DATA_PATH: $(MODELS_PATH) - MODELS_PATH: $(MODELS_PATH) displayName: 'IE CAPITests' enabled: 'false' diff --git a/.ci/azure/windows.yml b/.ci/azure/windows.yml index 0c1383b4e7cfe4..f0217b36c10df9 100644 --- a/.ci/azure/windows.yml +++ b/.ci/azure/windows.yml @@ -34,12 +34,6 @@ resources: name: openvinotoolkit/openvino_contrib ref: master - - repository: testdata - type: github - endpoint: openvinotoolkit - name: openvinotoolkit/testdata - ref: master - jobs: - job: Win strategy: @@ -63,7 +57,6 @@ jobs: BUILD_TYPE: Release REPO_DIR: $(Build.Repository.LocalPath) OPENVINO_CONTRIB_REPO_DIR: $(REPO_DIR)\..\openvino_contrib - MODELS_PATH: $(REPO_DIR)\..\testdata WORK_DIR: $(Pipeline.Workspace)\_w BUILD_DIR: $(WORK_DIR)\build BUILD_SAMPLES_DIR: $(WORK_DIR)\build_samples @@ -130,11 +123,6 @@ jobs: submodules: 'true' path: openvino_contrib - - checkout: testdata - clean: 'true' - lfs: 'true' - path: testdata - - script: | python -m pip install --upgrade pip rem For running Python API tests diff --git a/.github/workflows/code_snippets.yml b/.github/workflows/code_snippets.yml index 5b014563d94475..581a584b88805f 100644 --- a/.github/workflows/code_snippets.yml +++ b/.github/workflows/code_snippets.yml @@ -27,8 +27,7 @@ jobs: - name: Clone OpenVINO uses: actions/checkout@v4 with: - submodules: recursive - lfs: true + submodules: 'true' - name: Install OpenCL uses: awalsh128/cache-apt-pkgs-action@v1.3.0 @@ -38,7 +37,7 @@ jobs: version: 3.0 - name: CMake configure - run: cmake -DCMAKE_BUILD_TYPE=Release -B build + run: cmake -DCMAKE_BUILD_TYPE=Release -DTHREADING=SEQ -B build - name: Get number of CPU cores uses: SimenB/github-actions-cpu-cores@v2 diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index dbc3245055af47..164a5454465bf5 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -13,7 +13,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: recursive + submodules: 'true' - name: Install clang-format-9 run: | @@ -49,7 +49,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: recursive + submodules: 'true' - name: Install ShellCheck run: | @@ -80,7 +80,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: recursive + submodules: 'true' - name: Install Clang dependency run: | diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 8e375e63cbacc5..d89666b898fa80 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -30,7 +30,7 @@ jobs: - name: Clone OpenVINO uses: actions/checkout@v4 with: - submodules: recursive + submodules: 'true' - name: Install dependencies run: | @@ -108,7 +108,7 @@ jobs: run: ${{ github.workspace }}/bin/intel64/Release/ov_hetero_func_tests - name: Run IR frontend tests - run: ${{ github.workspace }}/bin/intel64/Release/ov_ir_frontend_tests # --gtest_print_time=1 --gtest_output=xml:${{ github.workspace }}/testdata/TEST-IRFrontend.xml + run: ${{ github.workspace }}/bin/intel64/Release/ov_ir_frontend_tests - name: Run ONNX frontend tests run: ${{ github.workspace }}/bin/intel64/Release/ov_onnx_frontend_tests --gtest_filter=-*IE_GPU* diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 22d9037988634b..55111be2453ee6 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -46,8 +46,6 @@ jobs: LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests BUILD_DIR: ${{ github.workspace }}/build - DATA_PATH: ${{ github.workspace }}/testdata - MODELS_PATH: ${{ github.workspace }}/testdata OV_TEMP: ${{ github.workspace }}/openvino_temp PYTHON_STATIC_ARGS: -m "not dynamic_library" steps: @@ -55,22 +53,13 @@ jobs: uses: actions/checkout@v4 with: path: 'openvino' - submodules: 'recursive' + submodules: 'true' - name: Clone OpenVINO Contrib uses: actions/checkout@v4 with: repository: 'openvinotoolkit/openvino_contrib' path: 'openvino_contrib' - submodules: 'recursive' - - - name: Clone testdata for C API tests - uses: actions/checkout@v4 - with: - repository: 'openvinotoolkit/testdata' - path: 'testdata' - submodules: 'recursive' - lfs: 'true' # # Dependencies @@ -493,8 +482,6 @@ jobs: LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests BUILD_DIR: ${{ github.workspace }}/build - DATA_PATH: ${{ github.workspace }}/testdata - MODELS_PATH: ${{ github.workspace }}/testdata OV_TEMP: ${{ github.workspace }}/openvino_temp PYTHON_STATIC_ARGS: -m "not dynamic_library" @@ -507,7 +494,6 @@ jobs: uses: actions/checkout@v4 with: path: 'openvino' - submodules: 'recursive' # # Dependencies diff --git a/.github/workflows/linux_android_arm64.yml b/.github/workflows/linux_android_arm64.yml index d2106b22b63b55..4d0e40640ac5b5 100644 --- a/.github/workflows/linux_android_arm64.yml +++ b/.github/workflows/linux_android_arm64.yml @@ -55,7 +55,14 @@ jobs: uses: actions/checkout@v4 with: path: 'openvino' - submodules: 'recursive' + + - name: Init submodules for non vcpkg dependencies + run: | + pushd ${{ env.OPENVINO_REPO }} + git submodule update --init -- ${{ env.OPENVINO_REPO }}/src/plugins + git submodule update --init -- ${{ env.OPENVINO_REPO }}/thirdparty/gtest + git submodule update --init -- ${{ env.OPENVINO_REPO }}/thirdparty/open_model_zoo + popd - name: Clone VCPKG uses: actions/checkout@v4 @@ -63,7 +70,6 @@ jobs: repository: 'microsoft/vcpkg' path: 'vcpkg' fetch-depth: '0' - submodules: 'recursive' - name: Setup Python 3.11 uses: actions/setup-python@v4 diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index a598ca0b5ebe46..3ed58d0ec02de9 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -44,8 +44,6 @@ jobs: OPENVINO_REPO: ${{ github.workspace }}/openvino BUILD_DIR: ${{ github.workspace }}/build INSTALL_DIR: ${{ github.workspace }}/install - DATA_PATH: ${{ github.workspace }}/testdata - MODELS_PATH: ${{ github.workspace }}/testdata OV_TEMP: ${{ github.workspace }}/openvino_temp steps: - name: Clone OpenVINO @@ -152,7 +150,6 @@ jobs: - name: CMake configure run: | source ${{ env.BUILD_DIR }}/dependencies/conanbuild.sh - # TODO: return tests building once GPU plugin migrates to Plugin API 2.0 cmake \ -G Ninja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -166,7 +163,7 @@ jobs: -DPYTHON_MODULE_EXTENSION=$(aarch64-linux-gnu-python3-config --extension-suffix) \ -DPYTHON_LIBRARY=/usr/lib/aarch64-linux-gnu/libc-2.31.so \ -DPYTHON_INCLUDE_DIR=$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))") \ - -DENABLE_DATA=OFF \ + -DENABLE_TESTS=ON \ -DENABLE_SYSTEM_TBB=ON \ -DENABLE_SYSTEM_PROTOBUF=ON \ -DENABLE_SYSTEM_SNAPPY=ON \ diff --git a/.github/workflows/linux_conditional_compilation.yml b/.github/workflows/linux_conditional_compilation.yml index 5522dce84e9c52..8ebad71c82af8d 100644 --- a/.github/workflows/linux_conditional_compilation.yml +++ b/.github/workflows/linux_conditional_compilation.yml @@ -43,7 +43,6 @@ jobs: OPENVINO_REPO: ${{ github.workspace }}/openvino OPENVINO_CONTRIB_REPO: ${{ github.workspace }}/openvino_contrib BUILD_DIR: ${{ github.workspace }}/build - DATA_PATH: ${{ github.workspace }}/testdata MODELS_PATH: ${{ github.workspace }}/testdata OV_TEMP: ${{ github.workspace }}/openvino_temp PYTHON_STATIC_ARGS: -m "not dynamic_library and not template_plugin" @@ -52,14 +51,13 @@ jobs: uses: actions/checkout@v3 with: path: 'openvino' - submodules: 'recursive' + submodules: 'true' - - name: Clone testdata for C API tests + - name: Clone test models uses: actions/checkout@v3 with: repository: 'openvinotoolkit/testdata' path: 'testdata' - submodules: 'recursive' lfs: 'true' # diff --git a/.github/workflows/linux_debian.yml b/.github/workflows/linux_debian.yml index 6e65a344764582..b517a368c46ced 100644 --- a/.github/workflows/linux_debian.yml +++ b/.github/workflows/linux_debian.yml @@ -48,8 +48,6 @@ jobs: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests - DATA_PATH: ${{ github.workspace }}/testdata - MODELS_PATH: ${{ github.workspace }}/testdata OV_TEMP: ${{ github.workspace }}/openvino_temp SAMPLES_INSTALL_DIR: /usr/share/openvino/samples PYTHON_STATIC_ARGS: -m "not dynamic_library and not template_plugin" @@ -58,15 +56,7 @@ jobs: uses: actions/checkout@v4 with: path: 'openvino' - submodules: 'recursive' - - - name: Clone testdata for C API tests - uses: actions/checkout@v4 - with: - repository: 'openvinotoolkit/testdata' - path: 'testdata' - submodules: 'recursive' - lfs: 'true' + submodules: 'true' - name: Create Directories run: | diff --git a/.github/workflows/mo.yml b/.github/workflows/mo.yml index eab7664a776333..4341af10f400db 100644 --- a/.github/workflows/mo.yml +++ b/.github/workflows/mo.yml @@ -22,7 +22,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: recursive + path: 'openvino' - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/py_checks.yml b/.github/workflows/py_checks.yml index 399c118b40a40c..e1a0d3356c26f4 100644 --- a/.github/workflows/py_checks.yml +++ b/.github/workflows/py_checks.yml @@ -27,7 +27,7 @@ jobs: - name: Code checkout uses: actions/checkout@v4 with: - submodules: recursive + path: 'openvino' - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a6c775c42c2d67..680f2686ac63cf 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -36,8 +36,6 @@ env: SAMPLES_INSTALL_DIR: "${{ github.workspace }}\\install\\samples" LAYER_TESTS_INSTALL_DIR: "${{ github.workspace }}\\install\\tests\\layer_tests" BUILD_DIR: "${{ github.workspace }}\\build" - DATA_PATH: "${{ github.workspace }}\\testdata" - MODELS_PATH: "${{ github.workspace }}\\testdata" OV_TEMP: "${{ github.workspace }}\\openvino_temp" PYTHON_STATIC_ARGS: -m "not dynamic_library and not template_plugin" VCVARSPATH: "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Auxiliary\\Build\\vcvarsall.bat" @@ -53,22 +51,13 @@ jobs: uses: actions/checkout@v4 with: path: 'openvino' - submodules: 'recursive' + submodules: 'true' - name: Clone OpenVINO Contrib uses: actions/checkout@v4 with: repository: 'openvinotoolkit/openvino_contrib' path: 'openvino_contrib' - submodules: 'recursive' - - - name: Clone testdata for C API tests - uses: actions/checkout@v4 - with: - repository: 'openvinotoolkit/testdata' - path: 'testdata' - submodules: 'recursive' - lfs: 'true' # # Dependencies @@ -265,8 +254,6 @@ jobs: SAMPLES_INSTALL_DIR: "${{ github.workspace }}\\install\\samples" LAYER_TESTS_INSTALL_DIR: "${{ github.workspace }}\\install\\tests\\layer_tests" BUILD_DIR: "${{ github.workspace }}\\build" - DATA_PATH: "${{ github.workspace }}\\testdata" - MODELS_PATH: "${{ github.workspace }}\\testdata" PYTHON_STATIC_ARGS: -m "not dynamic_library and not template_plugin" steps: @@ -306,14 +293,6 @@ jobs: uses: actions/checkout@v4 with: path: 'openvino' - submodules: 'recursive' - - - name: Clone OpenVINO Contrib - uses: actions/checkout@v4 - with: - repository: 'openvinotoolkit/openvino_contrib' - path: 'openvino_contrib' - submodules: 'recursive' - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/windows_conditional_compilation.yml b/.github/workflows/windows_conditional_compilation.yml index 94c4ec773c268f..1b884811c43316 100644 --- a/.github/workflows/windows_conditional_compilation.yml +++ b/.github/workflows/windows_conditional_compilation.yml @@ -40,7 +40,6 @@ env: LAYER_TESTS_INSTALL_DIR: "${{ github.workspace }}\\install\\tests\\layer_tests" BUILD_DIR: "${{ github.workspace }}\\build" BUILD_DIR_2: "${{ github.workspace }}\\build_s" - DATA_PATH: "${{ github.workspace }}\\testdata" MODELS_PATH: "${{ github.workspace }}\\testdata" OV_TEMP: "${{ github.workspace }}\\openvino_temp" BUILD_TYPE: "Release" @@ -60,14 +59,13 @@ jobs: uses: actions/checkout@v3 with: path: 'openvino' - submodules: 'recursive' + submodules: 'true' - - name: Clone testdata for sample test + - name: Clone test models uses: actions/checkout@v4 with: repository: 'openvinotoolkit/testdata' path: 'testdata' - submodules: 'recursive' lfs: 'true' # diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 0f9683377e18e9..1d4210f300b058 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -7,22 +7,6 @@ cmake_policy(SET CMP0054 NEW) # TODO: fix it, outside of source dir MO cannot find TBB dependency set_temp_directory(TEMP "${CMAKE_SOURCE_DIR}") -if(ENABLE_SAME_BRANCH_FOR_MODELS) - branchName(MODELS_BRANCH) -else() - set(MODELS_BRANCH "master") -endif() - -if(ENABLE_DATA) - add_models_repo(${ENABLE_DATA} "data:https://github.com/openvinotoolkit/testdata.git") - set(MODELS_PATH "${TEMP}/models/src/data") - set(DATA_PATH "${MODELS_PATH}") -endif() - -message(STATUS "MODELS_PATH=" ${MODELS_PATH}) - -fetch_models_and_validation_set() - ## Intel OMP package if(THREADING STREQUAL "OMP") reset_deps_cache(OMP) diff --git a/cmake/developer_package/IEDevScriptsConfig.cmake b/cmake/developer_package/IEDevScriptsConfig.cmake index 4f6a195b293bc9..2a5fc78dc6ba22 100644 --- a/cmake/developer_package/IEDevScriptsConfig.cmake +++ b/cmake/developer_package/IEDevScriptsConfig.cmake @@ -238,7 +238,6 @@ include(cross_compile/cross_compiled_func) include(faster_build) include(whole_archive) include(linux_name) -include(models) include(api_validator/api_validator) include(vs_version/vs_version) diff --git a/cmake/developer_package/download/dependency_solver.cmake b/cmake/developer_package/download/dependency_solver.cmake index 0fa2ad53bfbf6d..410a81754cae22 100644 --- a/cmake/developer_package/download/dependency_solver.cmake +++ b/cmake/developer_package/download/dependency_solver.cmake @@ -135,14 +135,6 @@ function (RESOLVE_DEPENDENCY NAME_OF_CMAKE_VAR) endfunction(RESOLVE_DEPENDENCY) -function (resolve_model_dependency network archive network_model_path) - RESOLVE_DEPENDENCY(${network_model_path} - ARCHIVE "models_archives/${archive}" - TARGET_PATH "${MODELS_PATH}/${network}") - string (REPLACE ${MODELS_PATH} "" relative_path ${${network_model_path}}) - set(${network_model_path} ".${relative_path}" PARENT_SCOPE) -endfunction() - function(reset_deps_cache) # # Reset the dependencies cache if it was set by dependency solver diff --git a/cmake/developer_package/models.cmake b/cmake/developer_package/models.cmake deleted file mode 100644 index 9c6f3fa7fc02a8..00000000000000 --- a/cmake/developer_package/models.cmake +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (C) 2018-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -if(ENABLE_DATA) - find_package(Git REQUIRED) -endif() - -set(MODELS_LST "") -set(MODELS_LST_TO_FETCH "") - -function (add_models_repo add_to_fetcher model_name) - list(LENGTH ARGV add_models_args) - if (add_models_args EQUAL 3) - list(GET ARGV 2 branch_name) - else() - set(branch_name ${MODELS_BRANCH}) - endif() - if (add_to_fetcher) - set(model_name "${model_name}:${branch_name}") - list(APPEND MODELS_LST_TO_FETCH ${model_name}) - endif() - - list(APPEND MODELS_LST ${model_name}) - - set(MODELS_LST_TO_FETCH ${MODELS_LST_TO_FETCH} PARENT_SCOPE) - set(MODELS_LST ${MODELS_LST} PARENT_SCOPE) -endfunction() - -function(add_lfs_repo name prefix url tag) - if(TARGET ${name}) - return() - endif() - - include(ExternalProject) - ExternalProject_Add(${name} - PREFIX ${prefix} - GIT_REPOSITORY ${url} - GIT_TAG ${tag} - GIT_CONFIG "http.sslverify=false" - GIT_PROGRESS 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - LOG_DOWNLOAD ON) - - execute_process( - COMMAND ${GIT_EXECUTABLE} lfs install --local --force - WORKING_DIRECTORY ${prefix}/src/${name} - OUTPUT_VARIABLE lfs_output - RESULT_VARIABLE lfs_var) - if(lfs_var) - message(FATAL_ERROR [=[ - Failed to setup Git LFS: ${lfs_output} - Git lfs must be installed in order to fetch models - Please install it from https://git-lfs.github.com/ - ]=]) - endif() -endfunction() - -function (fetch_models_and_validation_set) - foreach(loop_var ${MODELS_LST_TO_FETCH}) - string(REPLACE ":" ";" MODEL_CONFIG_LST ${loop_var}) - - list(GET MODEL_CONFIG_LST 0 folder_name) - list(GET MODEL_CONFIG_LST 1 git_url) - list(GET MODEL_CONFIG_LST 2 repo_name) - list(GET MODEL_CONFIG_LST 3 branch_name) - - add_lfs_repo( - "${folder_name}" - "${TEMP}/models" - "${git_url}:${repo_name}" - "${branch_name}") - endforeach(loop_var) -endfunction() diff --git a/cmake/features.cmake b/cmake/features.cmake index 9fdee1f1f8277b..5dca5b7b21bef1 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -107,8 +107,6 @@ ie_dependent_option (GAPI_TEST_PERF "if GAPI unit tests should examine performan ie_dependent_option (ENABLE_FUNCTIONAL_TESTS "functional tests" ON "ENABLE_TESTS" OFF) -ie_dependent_option (ENABLE_DATA "fetch models from testdata repo" ON "ENABLE_FUNCTIONAL_TESTS;NOT ANDROID" OFF) - ie_option (ENABLE_SAMPLES "console samples are part of OpenVINO Runtime package" ON) set(OPENVINO_EXTRA_MODULES "" CACHE STRING "Extra paths for extra modules to include into OpenVINO build") diff --git a/install_build_dependencies.sh b/install_build_dependencies.sh index 21faba35e5aa2b..0bba1601a1ed16 100755 --- a/install_build_dependencies.sh +++ b/install_build_dependencies.sh @@ -65,6 +65,7 @@ if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ] ; then libpython3-dev \ pybind11-dev \ libffi-dev \ + cython3 \ `# spell checking for MO sources` \ python3-enchant \ `# samples and tools` \ @@ -106,7 +107,6 @@ elif [ -f /etc/redhat-release ] || grep -q "rhel" /etc/os-release ; then make \ `# to determine openvino version via git` \ git \ - git-lfs \ `# to build and check pip packages` \ patchelf \ fdupes \ @@ -145,7 +145,6 @@ elif [ -f /etc/os-release ] && grep -q "SUSE" /etc/os-release ; then make \ `# to determine openvino version via git` \ git \ - git-lfs \ `# to build and check pip packages` \ patchelf \ fdupes \ diff --git a/src/bindings/c/tests/test_model_repo.cpp b/src/bindings/c/tests/test_model_repo.cpp index 1a7e64975a63bb..c221df9e3f8fe9 100644 --- a/src/bindings/c/tests/test_model_repo.cpp +++ b/src/bindings/c/tests/test_model_repo.cpp @@ -7,6 +7,18 @@ #include namespace TestDataHelpers { + +const std::string model_bin_name = "test_model.bin"; +const std::string model_xml_name = "test_model.xml"; +const std::string model_exported_name = "test_exported_model.blob"; + +void generate_test_model() { + ov::pass::Manager manager; + manager.register_pass(model_xml_name, model_bin_name); + auto function = ngraph::builder::subgraph::makeConvPoolReluNoReshapes({1, 3, 227, 227}); + manager.run_passes(function); +} + std::string generate_test_xml_file() { // Create the file std::string plugin_xml = "plugin_test.xml"; diff --git a/src/bindings/c/tests/test_model_repo.hpp b/src/bindings/c/tests/test_model_repo.hpp index b9a595c9577e24..b6f016f27ed3c2 100644 --- a/src/bindings/c/tests/test_model_repo.hpp +++ b/src/bindings/c/tests/test_model_repo.hpp @@ -15,16 +15,11 @@ namespace TestDataHelpers { -static const std::string model_bin_name = "test_model.bin"; -static const std::string model_xml_name = "test_model.xml"; -static const std::string model_exported_name = "test_exported_model.blob"; - -inline void generate_test_model() { - ov::pass::Manager manager; - manager.register_pass(model_xml_name, model_bin_name); - auto function = ngraph::builder::subgraph::makeConvPoolReluNoReshapes({1, 3, 227, 227}); - manager.run_passes(function); -} +extern const std::string model_bin_name; +extern const std::string model_xml_name; +extern const std::string model_exported_name; + +void generate_test_model(); inline std::string get_model_xml_file_name() { return model_xml_name; diff --git a/src/plugins/intel_gna/tests/deprecated/helpers/CMakeLists.txt b/src/plugins/intel_gna/tests/deprecated/helpers/CMakeLists.txt index 845af496642ce1..3423cf8f90ac86 100644 --- a/src/plugins/intel_gna/tests/deprecated/helpers/CMakeLists.txt +++ b/src/plugins/intel_gna/tests/deprecated/helpers/CMakeLists.txt @@ -13,9 +13,6 @@ file (GLOB HELPERS_INCLUDES ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp ) -## Enable Models multiple search paths -message("configuring file: ${CMAKE_CURRENT_BINARY_DIR}/test_model_repo.h") - function(add_helpers target_name) add_library(${target_name} STATIC ${HELPERS_SRC}) @@ -37,8 +34,3 @@ add_helpers(${TARGET_NAME}_s USE_STATIC_IE) target_link_libraries(${TARGET_NAME}_s PUBLIC common_test_utils_s PRIVATE inference_engine_legacy_s) - -if (ENABLE_DATA) - add_dependencies(${TARGET_NAME} data) - add_dependencies(${TARGET_NAME}_s data) -endif() diff --git a/src/plugins/intel_gna/tests/deprecated/helpers/test_model_path.cpp b/src/plugins/intel_gna/tests/deprecated/helpers/test_model_path.cpp deleted file mode 100644 index fe183cae6e7a88..00000000000000 --- a/src/plugins/intel_gna/tests/deprecated/helpers/test_model_path.cpp +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -// -// Created by user on 19.10.18. -// - -#include "helpers/test_model_repo.hpp" -#include "test_model_path.hpp" - -#ifndef _WIN32 -# include -# include -#else -# include -#endif - -#include -#include -#include -#include -#include -#include - -#ifndef _WIN32 -static std::string getDirname(std::string filePath) { - std::vector input(filePath.begin(), filePath.end()); - input.push_back(0); - return dirname(&*input.begin()); -} -#else -static std::string getDirname(std::string filePath) { - char dirname[_MAX_DIR] = {}; - char drive[_MAX_DRIVE] = {}; - _splitpath(filePath.c_str(), drive, dirname, nullptr, nullptr); // preserve drive letter if possible - return std::string(drive) + dirname; -} -#endif - -static std::string get_models_path() { - const char* models_path = TestDataHelpers::get_model_path_non_fatal(); - - if (nullptr == models_path) { - ::testing::AssertionFailure() << "MODELS_PATH not defined"; - } - - return std::string(models_path); -} - -static bool exist(const std::string& name) { - std::ifstream file(name); - if(!file) // If the file was not found, then file is 0, i.e. !file=1 or true. - return false; // The file was not found. - else // If the file was found, then file is non-0. - return true; // The file was found. -} - -static std::vector getModelsDirs() { - auto repo_list = get_model_repo(); - int last_delimiter = 0; - std::vector folders; - for(;;) { - auto folderDelimiter = repo_list.find(':', last_delimiter); - if (folderDelimiter == std::string::npos) { - break; - } - auto nextDelimiter = repo_list.find(';', last_delimiter); - folders.push_back(repo_list.substr(last_delimiter, folderDelimiter - last_delimiter)); - - if (nextDelimiter == std::string::npos) { - break; - } - - last_delimiter = nextDelimiter + 1; - } - return folders; -} - -ModelsPath::operator std::string() const { - std::vector absModelsPath; - for (auto & path : getModelsDirs()) { - std::string b = get_models_path(); - const auto absPath = get_models_path() + kPathSeparator + path + _rel_path.str(); - absModelsPath.push_back(absPath); - if (exist(absPath)) { - return absPath; - } - //checking models for precision encoded in folder name - auto dirname = getDirname(absPath); - std::vector> stdprecisions = { - {"_fp32", "FP32"}, - {"_q78", "_Q78"}, - {"_fp16", "FP16"}, - {"_i16", "I16"} - }; - - if (dirname.size() == absPath.size()) - continue; - - auto filename = absPath.substr(dirname.size() + 1); - - for (auto &precision : stdprecisions) { - auto havePrecision = filename.find(precision.first); - if (havePrecision == std::string::npos) continue; - - auto newName = filename.replace(havePrecision, precision.first.size(), ""); - newName = dirname + kPathSeparator + precision.second + kPathSeparator + newName; - - if (exist(newName)) { - return newName; - } - } - } - - // checking dirname - auto getModelsDirname = [](std::string path) -> std::string { - std::string dir = getDirname(path); - - struct stat sb; - if (stat(dir.c_str(), &sb) != 0 || !S_ISDIR(sb.st_mode)) { - return ""; - } - return dir; - }; - - for (auto & path : absModelsPath) { - std::string publicDir = getModelsDirname(path); - - if (!publicDir.empty()) { - return path; - } - } - std::stringstream errorMsg; - errorMsg<< "path to model invalid, models found at: \n"; - - for (auto & path : absModelsPath) { - errorMsg << path <<"\n"; - } - errorMsg << "also searched by parent directory names: \n"; - for (auto & path : absModelsPath) { - errorMsg << getDirname(path) << "\n"; - } - - std::cout << errorMsg.str(); - ::testing::AssertionFailure() << errorMsg.str(); - - // doesn't matter what to return here - return ""; -} diff --git a/src/plugins/intel_gna/tests/deprecated/helpers/test_model_path.hpp b/src/plugins/intel_gna/tests/deprecated/helpers/test_model_path.hpp deleted file mode 100644 index 08eac4268447c3..00000000000000 --- a/src/plugins/intel_gna/tests/deprecated/helpers/test_model_path.hpp +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -static const char kPathSeparator = -#if defined _WIN32 || defined __CYGWIN__ - '\\'; -#else - '/'; -#endif - -class ModelsPath { - std::stringstream _rel_path; - mutable std::string _abs_path; - - public: - - ModelsPath() = default; - - ModelsPath(const ModelsPath & that) { - _rel_path << that._rel_path.str(); - } - - template - ModelsPath operator + (const T & relative_path) const { - ModelsPath newPath(*this); - newPath += relative_path; - return newPath; - } - - template - ModelsPath & operator += (const T & relative_path) { - _rel_path << relative_path; - return *this; - } - - template - ModelsPath & operator << (const T & serializable) { - _rel_path << serializable; - return *this; - } - - std::string str() const { - return this->operator std::string(); - } - - const char * c_str() const { - _abs_path = this->operator std::string (); - return _abs_path.c_str(); - } - - operator std::string() const; -}; - -inline std::ostream & operator << (std::ostream &os, const ModelsPath & path) { - os << path.str(); - return os; -} diff --git a/src/plugins/intel_gna/tests/deprecated/helpers/tests_common.hpp b/src/plugins/intel_gna/tests/deprecated/helpers/tests_common.hpp index 7f06a55f0d49cb..f164fc1e8ca361 100644 --- a/src/plugins/intel_gna/tests/deprecated/helpers/tests_common.hpp +++ b/src/plugins/intel_gna/tests/deprecated/helpers/tests_common.hpp @@ -22,7 +22,6 @@ #include #include "helpers/test_model_repo.hpp" -#include "test_model_path.hpp" #include #include diff --git a/src/plugins/intel_gna/tests/functional/CMakeLists.txt b/src/plugins/intel_gna/tests/functional/CMakeLists.txt index bef471c7b9b1e4..92a86566b869d4 100644 --- a/src/plugins/intel_gna/tests/functional/CMakeLists.txt +++ b/src/plugins/intel_gna/tests/functional/CMakeLists.txt @@ -25,7 +25,3 @@ addIeTargetTest( target_compile_definitions(${TARGET_NAME} PUBLIC ${ARGV} GNA_DATA_PATH=\"${CMAKE_CURRENT_SOURCE_DIR}/data\") - -if (ENABLE_DATA) - add_dependencies(${TARGET_NAME} data) -endif() \ No newline at end of file diff --git a/src/plugins/intel_gna/tests/functional/helpers/test_model_repo.cpp b/src/plugins/intel_gna/tests/functional/helpers/test_model_repo.cpp index 0fbe800dc11c26..8f133d806fe495 100644 --- a/src/plugins/intel_gna/tests/functional/helpers/test_model_repo.cpp +++ b/src/plugins/intel_gna/tests/functional/helpers/test_model_repo.cpp @@ -4,14 +4,14 @@ #include "test_model_repo.hpp" -std::string get_model_repo() { - return "models:"; -}; - -const char* TestDataHelpers::get_model_path_non_fatal() noexcept { - return TestDataHelpers::get_model_path_non_fatal_default(); -} - std::string TestDataHelpers::get_data_path() { - return TestDataHelpers::get_data_path_default(); + if (const auto env_var = std::getenv("GNA_DATA_PATH")) { + return env_var; + } + +#ifdef GNA_DATA_PATH + return GNA_DATA_PATH; +#else + return nullptr; +#endif } diff --git a/src/plugins/intel_gna/tests/functional/helpers/test_model_repo.hpp b/src/plugins/intel_gna/tests/functional/helpers/test_model_repo.hpp index 76994e4b6efa0d..fcbfb6e1c7b7e0 100644 --- a/src/plugins/intel_gna/tests/functional/helpers/test_model_repo.hpp +++ b/src/plugins/intel_gna/tests/functional/helpers/test_model_repo.hpp @@ -5,35 +5,8 @@ #pragma once #include -std::string get_model_repo(); - namespace TestDataHelpers { -const char* get_model_path_non_fatal() noexcept; - std::string get_data_path(); -inline const char* get_model_path_non_fatal_default() noexcept { - if (const auto env_var = std::getenv("MODELS_PATH")) { - return env_var; - } - -#ifdef MODELS_PATH - return MODELS_PATH; -#else - return nullptr; -#endif -}; - -inline std::string get_data_path_default() { - if (const auto env_var = std::getenv("GNA_DATA_PATH")) { - return env_var; - } - -#ifdef GNA_DATA_PATH - return GNA_DATA_PATH; -#else - return nullptr; -#endif -} } // namespace TestDataHelpers