Skip to content

Commit

Permalink
Separate executable for paddlepaddle unit tests (openvinotoolkit#6862)
Browse files Browse the repository at this point in the history
* Separate executable for paddlepaddle unit tests

* Fix CI

* Move PaddlePaddle-specific python requirements to paddlepaddle test folder
Also produce build time warning when paddle test models generation is disabled

* Renamed back PADDLE_TEST_MODELS_DIRNAME to TEST_PADDLE_MODELS_DIRNAME

* Add dependency on CPU plugin (PaddlePaddle fuzzy tests use CPU plugin for inference)

* Fix code style

* Fix review comments #2

* Code style fix

* Add dependency of 'paddlepaddle_test_models' to 'test_model_zoo'
  • Loading branch information
nosovmik authored and rnugmanx committed Aug 26, 2021
1 parent 5e43e94 commit a77ac63
Show file tree
Hide file tree
Showing 34 changed files with 238 additions and 154 deletions.
13 changes: 8 additions & 5 deletions .ci/azure/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,9 @@ jobs:
python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/wheel/requirements-dev.txt
# For running Python API tests
python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/src/requirements-dev.txt
# For running nGraph unit tests dependent on Python frameworks
python3 -m pip install -r $(REPO_DIR)/ngraph/test/requirements_test.txt
# For running PaddlePaddle frontend unit tests
python3 -m pip install -r $(REPO_DIR)/ngraph/test/frontend/paddlepaddle/requirements_dev.txt
# For running ONNX frontend unit tests
python3 -m pip install -r $(REPO_DIR)/ngraph/test/requirements_test_onnx.txt
# For MO unit tests
python3 -m pip install -r $(REPO_DIR)/model-optimizer/requirements.txt
Expand Down Expand Up @@ -171,13 +172,15 @@ jobs:
displayName: 'Model Optimizer UT'
continueOnError: false
- script: |
export FE_TEST_MODELS=$(INSTALL_DIR)/tests
. $(SETUPVARS) && $(INSTALL_TEST_DIR)/unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
- script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
workingDirectory: $(INSTALL_TEST_DIR)
displayName: 'nGraph UT'
continueOnError: false

- script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/paddlepaddle_tests --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-PaddlePaddle.xml
displayName: 'PaddlePaddle Frontend UT'
continueOnError: false

# . $(SETUPVARS) && python3 $(WORK_DIR)/gtest-parallel/gtest_parallel.py $(INSTALL_TEST_DIR)/InferenceEngineUnitTests --workers=16 --dump_json_test_results=InferenceEngineUnitTests.json --gtest_filter=*smoke* -- --gtest_print_time=1
- script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/InferenceEngineUnitTests --gtest_print_time=1 --gtest_output=xml:TEST-InferenceEngineUnitTests.xml
displayName: 'IE UT old'
Expand Down
13 changes: 8 additions & 5 deletions .ci/azure/windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,9 @@ jobs:
python -m pip install --upgrade pip
rem For running Python API tests
python -m pip install -r $(REPO_DIR)\inference-engine\ie_bridges\python\src\requirements-dev.txt
rem For running nGraph unit tests dependent on Python frameworks
python -m pip install -r $(REPO_DIR)\ngraph\test\requirements_test.txt
rem For running PaddlePaddle frontend unit tests
python -m pip install -r $(REPO_DIR)\ngraph\test\frontend\paddlepaddle\requirements_dev.txt
rem For running ONNX frontend unit tests
python -m pip install -r $(REPO_DIR)\ngraph\test\requirements_test_onnx.txt
rem For MO unit tests
python -m pip install -r $(REPO_DIR)\model-optimizer\requirements.txt
Expand Down Expand Up @@ -143,13 +144,15 @@ jobs:
displayName: 'Clean build dir'
continueOnError: false

- script: |
set FE_TEST_MODELS=$(INSTALL_DIR)\tests
call $(SETUPVARS) && $(INSTALL_TEST_DIR)\unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
- script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
workingDirectory: $(INSTALL_TEST_DIR)
displayName: 'nGraph UT'
continueOnError: false

- script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\paddlepaddle_tests --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-PaddlePaddle.xml
displayName: 'PaddlePaddle Frontend UT'
continueOnError: false

- script: |
set PATH=$(IB_DIR);%PATH%
call $(SETUPVARS) && "$(IB_TESTCONSOLE)" $(INSTALL_TEST_DIR)\InferenceEngineUnitTests.exe --gtest_output=xml:TEST-InferenceEngineUnitTests-IB.xml
Expand Down
4 changes: 4 additions & 0 deletions cmake/test_model_zoo.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,10 @@ if(ENABLE_TESTS)
add_dependencies(test_model_zoo test_pip_prerequsites)
endif()

if (NGRAPH_PDPD_FRONTEND_ENABLE)
add_dependencies(test_model_zoo paddlepaddle_test_models)
endif()

install(DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/test_model_zoo"
DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL)

Expand Down
59 changes: 3 additions & 56 deletions ngraph/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -558,31 +558,10 @@ if (NGRAPH_ONNX_IMPORT_ENABLE)
endif()

# SOURCE FOR FRONTEND TESTING

file(GLOB FRONTEND_TESTS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/frontend/frontend_manager.cpp)
set(SRC ${FRONTEND_TESTS_SRC} ${SRC})

file(GLOB FRONTEND_SHARED_TESTS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/frontend/shared/src/*.cpp)
file(GLOB FRONTEND_SHARED_TESTS_HDR ${CMAKE_CURRENT_SOURCE_DIR}/frontend/shared/include/*.hpp)
set(SRC ${FRONTEND_SHARED_TESTS_SRC} ${SRC})

add_definitions("-DTEST_MODEL_BUILD_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\"")

# ---- PaddlePaddle FrontEnd testing ------
if (NGRAPH_PDPD_FRONTEND_ENABLE)
ie_check_pip_package(paddlepaddle WARNING)

if(paddlepaddle_FOUND)
file(GLOB FRONTEND_PDPD_TESTS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/frontend/paddlepaddle/*.cpp)
set(SRC ${FRONTEND_PDPD_TESTS_SRC} ${SRC})
set(PADDLE_TEST_MODELS_DIRNAME paddle_test_models)
set(TEST_PADDLE_MODELS ${CMAKE_CURRENT_BINARY_DIR}/${PADDLE_TEST_MODELS_DIRNAME}/)
add_definitions("-DTEST_PADDLE_MODELS_DIRNAME=\"${PADDLE_TEST_MODELS_DIRNAME}/\"")
endif()
endif()
# ---- End PaddlePaddle FrontEnd testing ------

add_clang_format_target(unit-test_clang FOR_SOURCES ${SRC} ${MULTI_TEST_SRC} ${FRONTEND_SHARED_TESTS_HDR})
add_clang_format_target(unit-test_clang FOR_SOURCES ${SRC} ${MULTI_TEST_SRC})

foreach(BACKEND_NAME ${ACTIVE_BACKEND_LIST})
string(TOLOWER ${BACKEND_NAME} BACKEND_DIR)
Expand All @@ -603,7 +582,6 @@ add_executable(unit-test ${SRC})

target_include_directories(unit-test PRIVATE ".")
target_include_directories(unit-test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/runtime)
target_include_directories(unit-test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/frontend/shared/include)

add_definitions("-DCURDIR=\"${CMAKE_CURRENT_SOURCE_DIR}\"")
add_definitions("-DJSON_INCLUDES=\"${JSON_INCLUDE_DIR}\"")
Expand All @@ -618,7 +596,8 @@ target_link_libraries(unit-test PRIVATE ngraph_test_util
ie_backend
interpreter_backend
Threads::Threads
openvino::conditional_compilation)
openvino::conditional_compilation
frontend_manager)

# Protobuf-lite does not support parsing files from prototxt format
# Since most of the onnx models are stored in this format it have to be disabled
Expand Down Expand Up @@ -655,39 +634,7 @@ install(TARGETS unit-test
COMPONENT tests
EXCLUDE_FROM_ALL)

############ FRONTEND ############
target_include_directories(unit-test PRIVATE ${FRONTEND_INCLUDE_PATH} frontend/shared/include)
target_link_libraries(unit-test PRIVATE frontend_manager cnpy commonTestUtils)

add_subdirectory(frontend)
### END FRONTEND ###

#PaddlePaddle - test models generator
if (NGRAPH_PDPD_FRONTEND_ENABLE AND paddlepaddle_FOUND)
file(GLOB_RECURSE PDPD_GEN_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/files/paddlepaddle/gen_scripts/generate_*.py)
set(OUT_FILES "")
foreach(GEN_SCRIPT ${PDPD_GEN_SCRIPTS})
get_filename_component(FILE_WE ${GEN_SCRIPT} NAME_WE)
set(OUT_DONE_FILE ${TEST_PADDLE_MODELS}/${FILE_WE}_done.txt)
set(OUT_FILES ${OUT_DONE_FILE} ${OUT_FILES})
add_custom_command(OUTPUT ${OUT_DONE_FILE}
COMMAND ${PYTHON_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/files/paddlepaddle/gen_wrapper.py
${GEN_SCRIPT}
${TEST_PADDLE_MODELS}
${OUT_DONE_FILE}
DEPENDS ${GEN_SCRIPT} ${CMAKE_CURRENT_SOURCE_DIR}/files/paddlepaddle/gen_wrapper.py
)
endforeach()
add_custom_target(paddlepaddle_test_models DEPENDS ${OUT_FILES})
add_dependencies(unit-test paddlepaddle_test_models)
add_dependencies(unit-test paddlepaddle_ngraph_frontend)

install(DIRECTORY ${TEST_PADDLE_MODELS}
DESTINATION tests/${PADDLE_TEST_MODELS_DIRNAME}
COMPONENT tests
EXCLUDE_FROM_ALL)
endif()

# process models
add_dependencies(unit-test test_model_zoo)
5 changes: 5 additions & 0 deletions ngraph/test/frontend/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
add_subdirectory(shared)

if (NGRAPH_PDPD_FRONTEND_ENABLE)
add_subdirectory(paddlepaddle)
endif()

set(SRC ${CMAKE_CURRENT_SOURCE_DIR}/mock_frontend.cpp)
add_library(mock1_ngraph_frontend SHARED ${SRC})
Expand Down
70 changes: 70 additions & 0 deletions ngraph/test/frontend/paddlepaddle/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#

set(TARGET_NAME "paddlepaddle_tests")

file(GLOB_RECURSE SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)

add_executable(${TARGET_NAME} ${SRC})

target_link_libraries(${TARGET_NAME} PRIVATE frontend_shared_test_classes)

add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})

install(TARGETS ${TARGET_NAME}
RUNTIME DESTINATION tests
COMPONENT tests
EXCLUDE_FROM_ALL)

# Test model generating
ie_check_pip_package(paddlepaddle WARNING)

set(TEST_PADDLE_MODELS_DIRNAME test_model_zoo/paddle_test_models)
target_compile_definitions(${TARGET_NAME} PRIVATE -D TEST_PADDLE_MODELS_DIRNAME=\"${TEST_PADDLE_MODELS_DIRNAME}/\")

# If 'paddlepaddle' is not found, code will still be compiled
# but models will not be generated and tests will fail
# This is done this way for 'code style' and check cases - cmake shall pass, but CI machine doesn't need to have
# 'paddlepaddle' installed to check code style
if (paddlepaddle_FOUND)
set(TEST_PADDLE_MODELS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_PADDLE_MODELS_DIRNAME}/)

file(GLOB_RECURSE PADDLE_GEN_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/test_models/gen_scripts/generate_*.py)
file(GLOB_RECURSE PADDLE_ALL_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/*.py)
set(OUT_FILES "")
foreach(GEN_SCRIPT ${PADDLE_GEN_SCRIPTS})
get_filename_component(FILE_WE ${GEN_SCRIPT} NAME_WE)
set(OUT_DONE_FILE ${TEST_PADDLE_MODELS}/${FILE_WE}_done.txt)
set(OUT_FILES ${OUT_DONE_FILE} ${OUT_FILES})
add_custom_command(OUTPUT ${OUT_DONE_FILE}
COMMAND ${PYTHON_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/test_models/gen_wrapper.py
${GEN_SCRIPT}
${TEST_PADDLE_MODELS}
${OUT_DONE_FILE}
DEPENDS ${PADDLE_ALL_SCRIPTS}
)
endforeach()
add_custom_target(paddlepaddle_test_models DEPENDS ${OUT_FILES})

install(DIRECTORY ${TEST_PADDLE_MODELS}
DESTINATION tests/${TEST_PADDLE_MODELS_DIRNAME}
COMPONENT tests
EXCLUDE_FROM_ALL)
else()
# Produce warning message at build time as well
add_custom_command(OUTPUT unable_build_paddle_models.txt
COMMAND ${CMAKE_COMMAND}
-E cmake_echo_color --red "Warning: Unable to generate PaddlePaddle test models. Running '${TARGET_NAME}' will likely fail"
)
add_custom_target(paddlepaddle_test_models DEPENDS unable_build_paddle_models.txt)
endif()

add_dependencies(${TARGET_NAME} paddlepaddle_test_models)
add_dependencies(${TARGET_NAME} paddlepaddle_ngraph_frontend)

# Fuzzy tests for PaddlePaddle use IE_CPU engine
if (ENABLE_MKL_DNN)
add_dependencies(${TARGET_NAME} MKLDNNPlugin)
endif()
3 changes: 1 addition & 2 deletions ngraph/test/frontend/paddlepaddle/incorrect_cut_model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ TEST(FrontEndIncorrectCutModelTest, test_incorrect_cut)
ASSERT_NO_THROW(frontEnd = fem.load_by_framework(PADDLE_FE));
ASSERT_NE(frontEnd, nullptr);
auto model_filename = FrontEndTestUtils::make_model_path(
std::string(TEST_PADDLE_MODELS_DIRNAME) +
std::string("2in_2out/2in_2out.pdmodel"));
std::string(TEST_PADDLE_MODELS_DIRNAME) + std::string("2in_2out/2in_2out.pdmodel"));
ASSERT_NO_THROW(inputModel = frontEnd->load(model_filename));
ASSERT_NE(inputModel, nullptr);

Expand Down
10 changes: 10 additions & 0 deletions ngraph/test/frontend/paddlepaddle/main.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include "utils.hpp"

int main(int argc, char** argv)
{
return FrontEndTestUtils::run_tests(argc, argv);
}
1 change: 1 addition & 0 deletions ngraph/test/frontend/paddlepaddle/paddle_utils.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@
#include <string>

static const std::string PADDLE_FE = "paddle";

Loading

0 comments on commit a77ac63

Please sign in to comment.