From 50e6ba9d6c0c9db07d429dee106a0cf15bb6786e Mon Sep 17 00:00:00 2001 From: jiangjiajun Date: Tue, 23 Aug 2022 12:19:55 +0000 Subject: [PATCH 1/3] Add OpenVINO backend support --- CMakeLists.txt | 11 +- FastDeploy.cmake.in | 11 ++ VERSION_NUMBER | 2 +- csrc/fastdeploy/backends/backend.h | 18 +- .../backends/openvino/ov_backend.cc | 158 ++++++++++++++++++ .../fastdeploy/backends/openvino/ov_backend.h | 62 +++++++ csrc/fastdeploy/backends/ort/ort_backend.cc | 4 +- .../backends/paddle/paddle_backend.cc | 6 +- csrc/fastdeploy/core/config.h.in | 4 + csrc/fastdeploy/fastdeploy_model.cc | 27 +-- csrc/fastdeploy/fastdeploy_runtime.cc | 51 +++++- csrc/fastdeploy/fastdeploy_runtime.h | 10 +- .../vision/classification/ppcls/model.cc | 2 +- external/openvino.cmake | 91 ++++++++++ external/summary.cmake | 4 + fastdeploy/c_lib_wrap.py.in | 3 + 16 files changed, 427 insertions(+), 37 deletions(-) create mode 100644 csrc/fastdeploy/backends/openvino/ov_backend.cc create mode 100644 csrc/fastdeploy/backends/openvino/ov_backend.h create mode 100644 external/openvino.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index eb02982d75..0b06869214 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ option(WITH_GPU "Whether WITH_GPU=ON, will enable onnxruntime-gpu/paddle-infernc option(ENABLE_ORT_BACKEND "Whether to enable onnxruntime backend." OFF) option(ENABLE_TRT_BACKEND "Whether to enable tensorrt backend." OFF) option(ENABLE_PADDLE_BACKEND "Whether to enable paddle backend." OFF) +option(ENABLE_OPENVINO_BACKEND "Whether to enable paddle backend." OFF) option(CUDA_DIRECTORY "If build tensorrt backend, need to define path of cuda library.") option(TRT_DIRECTORY "If build tensorrt backend, need to define path of tensorrt library.") option(ENABLE_VISION "Whether to enable vision models usage." OFF) @@ -104,10 +105,11 @@ file(GLOB_RECURSE FDTENSOR_FUNC_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fas file(GLOB_RECURSE DEPLOY_ORT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/ort/*.cc) file(GLOB_RECURSE DEPLOY_PADDLE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/paddle/*.cc) file(GLOB_RECURSE DEPLOY_TRT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/*.cpp) +file(GLOB_RECURSE DEPLOY_OPENVINO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/openvino/*.cc) file(GLOB_RECURSE DEPLOY_VISION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cc) file(GLOB_RECURSE DEPLOY_TEXT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/text/*.cc) file(GLOB_RECURSE DEPLOY_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pybind/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/*_pybind.cc) -list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS} ${FDTENSOR_FUNC_SRCS}) +list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS} ${FDTENSOR_FUNC_SRCS}) set(DEPEND_LIBS "") @@ -144,6 +146,13 @@ if(ENABLE_PADDLE_BACKEND) endif() endif() +if(ENABLE_OPENVINO_BACKEND) + add_definitions(-DENABLE_OPENVINO_BACKEND) + list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_OPENVINO_SRCS}) + include(external/openvino.cmake) + list(APPEND DEPEND_LIBS external_openvino) +endif() + if(WITH_GPU) if(APPLE) message(FATAL_ERROR "Cannot enable GPU while compling in Mac OSX.") diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in index 6ee4f490ee..2f45574705 100644 --- a/FastDeploy.cmake.in +++ b/FastDeploy.cmake.in @@ -3,7 +3,9 @@ CMAKE_MINIMUM_REQUIRED (VERSION 3.12) set(WITH_GPU @WITH_GPU@) set(ENABLE_ORT_BACKEND @ENABLE_ORT_BACKEND@) set(ENABLE_PADDLE_BACKEND @ENABLE_PADDLE_BACKEND@) +set(ENABLE_OPENVINO_BACKEND @ENABLE_OPENVINO_BACKEND@) set(PADDLEINFERENCE_VERSION @PADDLEINFERENCE_VERSION@) +set(OPENVINO_VERSION @OPENVINO_VERSION@) set(ENABLE_TRT_BACKEND @ENABLE_TRT_BACKEND@) set(ENABLE_PADDLE_FRONTEND @ENABLE_PADDLE_FRONTEND@) set(ENABLE_VISION @ENABLE_VISION@) @@ -45,6 +47,11 @@ if(ENABLE_PADDLE_BACKEND) endif() endif() +if(ENABLE_OPENVINO_BACKEND) + find_library(OPENVINO_LIB openvino ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/openvino/lib/intel64 NO_DEFAULT_PATH) + list(APPEND FASTDEPLOY_LIBS ${OPENVINO_LIB}) +endif() + if(WITH_GPU) if (NOT CUDA_DIRECTORY) set(CUDA_DIRECTORY "/usr/local/cuda") @@ -124,6 +131,10 @@ message(STATUS " ENABLE_PADDLE_BACKEND : ${ENABLE_PADDLE_BACKEND}") if(ENABLE_PADDLE_BACKEND) message(STATUS " Paddle Inference version : ${PADDLEINFERENCE_VERSION}") endif() +message(STATUS " ENABLE_OPENVINO_BACKEND : ${ENABLE_OPENVINO_BACKEND}") +if(ENABLE_OPENVINO_BACKEND) + message(STATUS " OpenVINO version : ${OPENVINO_VERSION}") +endif() message(STATUS " ENABLE_TRT_BACKEND : ${ENABLE_TRT_BACKEND}") message(STATUS " ENABLE_VISION : ${ENABLE_VISION}") message(STATUS " ENABLE_TEXT : ${ENABLE_TEXT}") diff --git a/VERSION_NUMBER b/VERSION_NUMBER index 0ea3a944b3..0c62199f16 100644 --- a/VERSION_NUMBER +++ b/VERSION_NUMBER @@ -1 +1 @@ -0.2.0 +0.2.1 diff --git a/csrc/fastdeploy/backends/backend.h b/csrc/fastdeploy/backends/backend.h index de7b5a575f..6d2e11f662 100644 --- a/csrc/fastdeploy/backends/backend.h +++ b/csrc/fastdeploy/backends/backend.h @@ -14,12 +14,12 @@ #pragma once +#include "fastdeploy/backends/common/multiclass_nms.h" +#include "fastdeploy/core/fd_tensor.h" #include #include #include #include -#include "fastdeploy/backends/common/multiclass_nms.h" -#include "fastdeploy/core/fd_tensor.h" namespace fastdeploy { @@ -27,6 +27,20 @@ struct TensorInfo { std::string name; std::vector shape; FDDataType dtype; + + friend std::ostream& operator<<(std::ostream& output, + const TensorInfo& info) { + output << "TensorInfo(name: " << info.name << ", shape: ["; + for (size_t i = 0; i < info.shape.size(); ++i) { + if (i == info.shape.size() - 1) { + output << info.shape[i]; + } else { + output << info.shape[i] << ", "; + } + } + output << "], dtype: " << Str(info.dtype) << ")"; + return output; + } }; class BaseBackend { diff --git a/csrc/fastdeploy/backends/openvino/ov_backend.cc b/csrc/fastdeploy/backends/openvino/ov_backend.cc new file mode 100644 index 0000000000..2b71b9a6fc --- /dev/null +++ b/csrc/fastdeploy/backends/openvino/ov_backend.cc @@ -0,0 +1,158 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/backends/openvino/ov_backend.h" + +namespace fastdeploy { + +std::vector PartialShapeToVec(const ov::PartialShape& shape) { + std::vector res; + for (int i = 0; i < shape.size(); ++i) { + auto dim = shape[i]; + if (dim.is_dynamic()) { + res.push_back(-1); + } else { + res.push_back(dim.get_length()); + } + } + return res; +} + +FDDataType OpenVINODataTypeToFD(const ov::element::Type& type) { + if (type == ov::element::f32) { + return FDDataType::FP32; + } else if (type == ov::element::f64) { + return FDDataType::FP64; + } else if (type == ov::element::i8) { + return FDDataType::INT8; + } else if (type == ov::element::i32) { + return FDDataType::INT32; + } else if (type == ov::element::i64) { + return FDDataType::INT64; + } else { + FDASSERT(false, "Only support float/double/int8/int32/int64 now."); + } + return FDDataType::FP32; +} + +ov::element::Type FDDataTypeToOV(const FDDataType& type) { + if (type == FDDataType::FP32) { + return ov::element::f32; + } else if (type == FDDataType::FP64) { + return ov::element::f64; + } else if (type == FDDataType::INT8) { + return ov::element::i8; + } else if (type == FDDataType::INT32) { + return ov::element::i32; + } else if (type == FDDataType::INT64) { + return ov::element::i64; + } + FDASSERT(false, "Only support float/double/int8/int32/int64 now."); + return ov::element::f32; +} + +bool OpenVINOBackend::InitFromPaddle(const std::string& model_file, + const std::string& params_file, + const OpenVINOBackendOption& option) { + if (initialized_) { + FDERROR << "OpenVINOBackend is already initlized, cannot initialize again." + << std::endl; + return false; + } + option_ = option; + ov::AnyMap properties; + if (option_.cpu_thread_num > 0) { + properties["INFERENCE_NUM_THREADS"] = option_.cpu_thread_num; + } + + std::shared_ptr model = core_.read_model(model_file, params_file); + + // Get inputs/outputs information from loaded model + const std::vector> inputs = model->inputs(); + for (size_t i = 0; i < inputs.size(); ++i) { + TensorInfo info; + auto partial_shape = PartialShapeToVec(inputs[i].get_partial_shape()); + info.shape.assign(partial_shape.begin(), partial_shape.end()); + info.name = inputs[i].get_any_name(); + info.dtype = OpenVINODataTypeToFD(inputs[i].get_element_type()); + input_infos_.emplace_back(info); + } + const std::vector> outputs = model->outputs(); + for (size_t i = 0; i < outputs.size(); ++i) { + TensorInfo info; + auto partial_shape = PartialShapeToVec(outputs[i].get_partial_shape()); + info.shape.assign(partial_shape.begin(), partial_shape.end()); + info.name = outputs[i].get_any_name(); + info.dtype = OpenVINODataTypeToFD(outputs[i].get_element_type()); + output_infos_.emplace_back(info); + } + + compiled_model_ = core_.compile_model(model, "CPU", properties); + request_ = compiled_model_.create_infer_request(); + initialized_ = true; + return true; +} + +TensorInfo OpenVINOBackend::GetInputInfo(int index) { + FDASSERT(index < NumInputs(), + "The index: %d should less than the number of outputs: %d.", index, + NumOutputs()); + return input_infos_[index]; +} + +TensorInfo OpenVINOBackend::GetOutputInfo(int index) { + FDASSERT(index < NumOutputs(), + "The index: %d should less than the number of outputs: %d.", index, + NumOutputs()); + return output_infos_[index]; +} + +int OpenVINOBackend::NumInputs() const { return input_infos_.size(); } + +int OpenVINOBackend::NumOutputs() const { return output_infos_.size(); } + +bool OpenVINOBackend::Infer(std::vector& inputs, + std::vector* outputs) { + if (inputs.size() != input_infos_.size()) { + FDERROR << "[OpenVINOBackend] Size of the inputs(" << inputs.size() + << ") should keep same with the inputs of this model(" + << input_infos_.size() << ")." << std::endl; + return false; + } + + for (size_t i = 0; i < inputs.size(); ++i) { + ov::Shape shape(inputs[i].shape.begin(), inputs[i].shape.end()); + ov::Tensor ov_tensor(FDDataTypeToOV(inputs[i].dtype), shape, + inputs[i].Data()); + request_.set_tensor(inputs[i].name, ov_tensor); + } + + request_.infer(); + + outputs->resize(output_infos_.size()); + for (size_t i = 0; i < output_infos_.size(); ++i) { + auto out_tensor = request_.get_output_tensor(i); + auto out_tensor_shape = out_tensor.get_shape(); + std::vector shape(out_tensor_shape.begin(), + out_tensor_shape.end()); + (*outputs)[i].Allocate(shape, + OpenVINODataTypeToFD(out_tensor.get_element_type()), + output_infos_[i].name); + memcpy((*outputs)[i].MutableData(), out_tensor.data(), + (*outputs)[i].Nbytes()); + } + return true; +} + +} // namespace fastdeploy diff --git a/csrc/fastdeploy/backends/openvino/ov_backend.h b/csrc/fastdeploy/backends/openvino/ov_backend.h new file mode 100644 index 0000000000..69c6104f70 --- /dev/null +++ b/csrc/fastdeploy/backends/openvino/ov_backend.h @@ -0,0 +1,62 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "fastdeploy/backends/backend.h" +#include "openvino/openvino.hpp" + +namespace fastdeploy { + +struct OpenVINOBackendOption { + int cpu_thread_num = 8; + std::map> shape_infos; +}; + +class OpenVINOBackend : public BaseBackend { + public: + OpenVINOBackend() {} + virtual ~OpenVINOBackend() = default; + + bool + InitFromPaddle(const std::string& model_file, const std::string& params_file, + const OpenVINOBackendOption& option = OpenVINOBackendOption()); + + bool + InitFromOnnx(const std::string& model_file, + const OpenVINOBackendOption& option = OpenVINOBackendOption()); + + bool Infer(std::vector& inputs, std::vector* outputs); + + int NumInputs() const; + + int NumOutputs() const; + + TensorInfo GetInputInfo(int index); + TensorInfo GetOutputInfo(int index); + + private: + ov::Core core_; + ov::CompiledModel compiled_model_; + ov::InferRequest request_; + OpenVINOBackendOption option_; + std::vector input_infos_; + std::vector output_infos_; +}; +} // namespace fastdeploy diff --git a/csrc/fastdeploy/backends/ort/ort_backend.cc b/csrc/fastdeploy/backends/ort/ort_backend.cc index a296c6b2b8..1b49928a99 100644 --- a/csrc/fastdeploy/backends/ort/ort_backend.cc +++ b/csrc/fastdeploy/backends/ort/ort_backend.cc @@ -32,10 +32,10 @@ void OrtBackend::BuildOption(const OrtBackendOption& option) { session_options_.SetGraphOptimizationLevel( GraphOptimizationLevel(option.graph_optimization_level)); } - if (option.intra_op_num_threads >= 0) { + if (option.intra_op_num_threads > 0) { session_options_.SetIntraOpNumThreads(option.intra_op_num_threads); } - if (option.inter_op_num_threads >= 0) { + if (option.inter_op_num_threads > 0) { session_options_.SetInterOpNumThreads(option.inter_op_num_threads); } if (option.execution_mode >= 0) { diff --git a/csrc/fastdeploy/backends/paddle/paddle_backend.cc b/csrc/fastdeploy/backends/paddle/paddle_backend.cc index 1b08f90c4c..e5a9d8e789 100644 --- a/csrc/fastdeploy/backends/paddle/paddle_backend.cc +++ b/csrc/fastdeploy/backends/paddle/paddle_backend.cc @@ -29,7 +29,11 @@ void PaddleBackend::BuildOption(const PaddleBackendOption& option) { if (!option.enable_log_info) { config_.DisableGlogInfo(); } - config_.SetCpuMathLibraryNumThreads(option.cpu_thread_num); + if (option.cpu_thread_num <= 0) { + config_.SetCpuMathLibraryNumThreads(8); + } else { + config_.SetCpuMathLibraryNumThreads(option.cpu_thread_num); + } } bool PaddleBackend::InitFromPaddle(const std::string& model_file, diff --git a/csrc/fastdeploy/core/config.h.in b/csrc/fastdeploy/core/config.h.in index b29113f1fd..573439f600 100644 --- a/csrc/fastdeploy/core/config.h.in +++ b/csrc/fastdeploy/core/config.h.in @@ -33,6 +33,10 @@ #cmakedefine ENABLE_PADDLE_BACKEND #endif +#ifndef ENABLE_OPENVINO_BACKEND +#cmakedefine ENABLE_OPENVINO_BACKEND +#endif + #ifndef WITH_GPU #cmakedefine WITH_GPU #endif diff --git a/csrc/fastdeploy/fastdeploy_model.cc b/csrc/fastdeploy/fastdeploy_model.cc index d1d296c6cd..41f1315c1c 100644 --- a/csrc/fastdeploy/fastdeploy_model.cc +++ b/csrc/fastdeploy/fastdeploy_model.cc @@ -26,31 +26,8 @@ bool FastDeployModel::InitRuntime() { return false; } if (runtime_option.backend != Backend::UNKNOWN) { - if (runtime_option.backend == Backend::ORT) { - if (!IsBackendAvailable(Backend::ORT)) { - FDERROR - << "Backend::ORT is not complied with current FastDeploy library." - << std::endl; - return false; - } - } else if (runtime_option.backend == Backend::TRT) { - if (!IsBackendAvailable(Backend::TRT)) { - FDERROR - << "Backend::TRT is not complied with current FastDeploy library." - << std::endl; - return false; - } - } else if (runtime_option.backend == Backend::PDINFER) { - if (!IsBackendAvailable(Backend::PDINFER)) { - FDERROR << "Backend::PDINFER is not compiled with current FastDeploy " - "library." - << std::endl; - return false; - } - } else { - FDERROR - << "Only support Backend::ORT / Backend::TRT / Backend::PDINFER now." - << std::endl; + if (!IsBackendAvailable(runtime_option.backend)) { + FDERROR << Str(runtime_option.backend) << " is not compiled with current FastDeploy library." << std::endl; return false; } diff --git a/csrc/fastdeploy/fastdeploy_runtime.cc b/csrc/fastdeploy/fastdeploy_runtime.cc index c2a16b9032..fcd4d10904 100644 --- a/csrc/fastdeploy/fastdeploy_runtime.cc +++ b/csrc/fastdeploy/fastdeploy_runtime.cc @@ -28,6 +28,10 @@ #include "fastdeploy/backends/paddle/paddle_backend.h" #endif +#ifdef ENABLE_OPENVINO_BACKEND +#include "fastdeploy/backends/openvino/ov_backend.h" +#endif + namespace fastdeploy { std::vector GetAvailableBackends() { @@ -40,6 +44,9 @@ std::vector GetAvailableBackends() { #endif #ifdef ENABLE_PADDLE_BACKEND backends.push_back(Backend::PDINFER); +#endif +#ifdef ENABLE_OPENVINO_BACKEND + backends.push_back(Backend::OPENVINO); #endif return backends; } @@ -61,6 +68,8 @@ std::string Str(const Backend& b) { return "Backend::TRT"; } else if (b == Backend::PDINFER) { return "Backend::PDINFER"; + } else if (b == Backend::OPENVINO) { + return "Backend::OPENVINO"; } return "UNKNOWN-Backend"; } @@ -177,6 +186,13 @@ void RuntimeOption::UseTrtBackend() { #endif } +void RuntimeOption::UseOpenVINOBackend() { +#ifdef ENABLE_OPENVINO_BACKEND + backend = Backend::OPENVINO; +#else + FDASSERT(false, "The FastDeploy didn't compile with OpenVINO."); +#endif +} void RuntimeOption::EnablePaddleMKLDNN() { pd_enable_mkldnn = true; } void RuntimeOption::DisablePaddleMKLDNN() { pd_enable_mkldnn = false; } @@ -228,21 +244,26 @@ bool Runtime::Init(const RuntimeOption& _option) { option.backend = Backend::ORT; } else if (IsBackendAvailable(Backend::PDINFER)) { option.backend = Backend::PDINFER; - } else { + } else if (IsBackendAvailable(Backend::OPENVINO)) { + option.backend = Backend::OPENVINO; + } { FDERROR << "Please define backend in RuntimeOption, current it's " "Backend::UNKNOWN." << std::endl; return false; } } + if (option.backend == Backend::ORT) { FDASSERT(option.device == Device::CPU || option.device == Device::GPU, - "Backend::TRT only supports Device::CPU/Device::GPU."); + "Backend::ORT only supports Device::CPU/Device::GPU."); CreateOrtBackend(); + FDINFO << "Runtime initialized with Backend::ORT." << std::endl; } else if (option.backend == Backend::TRT) { FDASSERT(option.device == Device::GPU, "Backend::TRT only supports Device::GPU."); CreateTrtBackend(); + FDINFO << "Runtime initialized with Backend::TRT." << std::endl; } else if (option.backend == Backend::PDINFER) { FDASSERT(option.device == Device::CPU || option.device == Device::GPU, "Backend::TRT only supports Device::CPU/Device::GPU."); @@ -250,6 +271,11 @@ bool Runtime::Init(const RuntimeOption& _option) { option.model_format == Frontend::PADDLE, "Backend::PDINFER only supports model format of Frontend::PADDLE."); CreatePaddleBackend(); + FDINFO << "Runtime initialized with Backend::PDINFER." << std::endl; + } else if (option.backend == Backend::OPENVINO) { + FDASSERT(option.device == Device::CPU, "Backend::OPENVINO only supports Device::CPU"); + CreateOpenVINOBackend(); + FDINFO << "Runtime initialized with Backend::OPENVINO." << std::endl; } else { FDERROR << "Runtime only support " "Backend::ORT/Backend::TRT/Backend::PDINFER as backend now." @@ -295,6 +321,27 @@ void Runtime::CreatePaddleBackend() { #endif } +void Runtime::CreateOpenVINOBackend() { +#ifdef ENABLE_OPENVINO_BACKEND + auto ov_option = OpenVINOBackendOption(); + ov_option.cpu_thread_num = option.cpu_thread_num; + FDASSERT(option.model_format == Frontend::PADDLE || + option.model_format == Frontend::ONNX, + "OpenVINOBackend only support model format of Frontend::PADDLE / " + "Frontend::ONNX."); + backend_ = utils::make_unique(); + auto casted_backend = dynamic_cast(backend_.get()); + + FDASSERT(casted_backend->InitFromPaddle(option.model_file, option.params_file, + ov_option), + "Load model from Paddle failed while initliazing PaddleBackend."); +#else + FDASSERT(false, + "OpenVINOBackend is not available, please compiled with " + "ENABLE_OPENVINO_BACKEND=ON."); +#endif +} + void Runtime::CreateOrtBackend() { #ifdef ENABLE_ORT_BACKEND auto ort_option = OrtBackendOption(); diff --git a/csrc/fastdeploy/fastdeploy_runtime.h b/csrc/fastdeploy/fastdeploy_runtime.h index ab6b4a188a..4abb1a02f4 100644 --- a/csrc/fastdeploy/fastdeploy_runtime.h +++ b/csrc/fastdeploy/fastdeploy_runtime.h @@ -21,7 +21,7 @@ namespace fastdeploy { -enum FASTDEPLOY_DECL Backend { UNKNOWN, ORT, TRT, PDINFER }; +enum FASTDEPLOY_DECL Backend { UNKNOWN, ORT, TRT, PDINFER, OPENVINO }; // AUTOREC will according to the name of model file // to decide which Frontend is enum FASTDEPLOY_DECL Frontend { AUTOREC, PADDLE, ONNX }; @@ -63,6 +63,9 @@ struct FASTDEPLOY_DECL RuntimeOption { // use tensorrt backend void UseTrtBackend(); + // use openvino backend + void UseOpenVINOBackend(); + // enable mkldnn while use paddle inference in CPU void EnablePaddleMKLDNN(); // disable mkldnn while use paddle inference in CPU @@ -97,7 +100,8 @@ struct FASTDEPLOY_DECL RuntimeOption { Backend backend = Backend::UNKNOWN; // for cpu inference and preprocess - int cpu_thread_num = 8; + // default will let the backend choose their own default value + int cpu_thread_num = -1; int device_id = 0; Device device = Device::CPU; @@ -152,6 +156,8 @@ struct FASTDEPLOY_DECL Runtime { void CreateTrtBackend(); + void CreateOpenVINOBackend(); + int NumInputs() { return backend_->NumInputs(); } int NumOutputs() { return backend_->NumOutputs(); } TensorInfo GetInputInfo(int index); diff --git a/csrc/fastdeploy/vision/classification/ppcls/model.cc b/csrc/fastdeploy/vision/classification/ppcls/model.cc index 2ea6d846f6..60d2f6dc31 100644 --- a/csrc/fastdeploy/vision/classification/ppcls/model.cc +++ b/csrc/fastdeploy/vision/classification/ppcls/model.cc @@ -26,7 +26,7 @@ PaddleClasModel::PaddleClasModel(const std::string& model_file, const RuntimeOption& custom_option, const Frontend& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::ORT, Backend::PDINFER}; + valid_cpu_backends = {Backend::ORT, Backend::OPENVINO, Backend::PDINFER}; valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/external/openvino.cmake b/external/openvino.cmake new file mode 100644 index 0000000000..242e5c2b8b --- /dev/null +++ b/external/openvino.cmake @@ -0,0 +1,91 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(ExternalProject) + +set(OPENVINO_PROJECT "extern_openvino") +set(OPENVINO_PREFIX_DIR ${THIRD_PARTY_PATH}/openvino) +set(OPENVINO_SOURCE_DIR + ${THIRD_PARTY_PATH}/openvino/src/${OPENVINO_PROJECT}) +set(OPENVINO_INSTALL_DIR ${THIRD_PARTY_PATH}/install/openvino) +set(OPENVINO_INC_DIR + "${OPENVINO_INSTALL_DIR}/include" + CACHE PATH "openvino include directory." FORCE) +set(OPENVINO_LIB_DIR + "${OPENVINO_INSTALL_DIR}/lib" + CACHE PATH "openvino lib directory." FORCE) +set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${OPENVINO_LIB_DIR}") + +set(OPENVINO_VERSION "2022.3.0") +set(OPENVINO_URL_PREFIX "https://bj.bcebos.com/fastdeploy/third_libs/") + +if(WIN32) + message(FATAL_ERROR "FastDeploy cannot ENABLE_OPENVINO_BACKEND in windows now.") + set(OPENVINO_FILENAME "openvino-win-x64-${OPENVINO_VERSION}.zip") + if(NOT CMAKE_CL_64) + message(FATAL_ERROR "FastDeploy cannot ENABLE_OPENVINO_BACKEND in win32 now.") + endif() +elseif(APPLE) + message(FATAL_ERROR "FastDeploy cannot ENABLE_OPENVINO_BACKEND in Mac OSX now.") + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64") + set(OPENVINO_FILENAME "openvino-osx-arm64-${OPENVINO_VERSION}.tgz") + else() + set(OPENVINO_FILENAME "openvino-osx-x86_64-${OPENVINO_VERSION}.tgz") + endif() +else() + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") + message("Cannot compile with openvino while in linux-aarch64 platform") + else() + set(OPENVINO_FILENAME "openvino-linux-x64-${OPENVINO_VERSION}.tgz") + endif() +endif() +set(OPENVINO_URL "${OPENVINO_URL_PREFIX}${OPENVINO_FILENAME}") + +include_directories(${OPENVINO_INC_DIR} +)# For OPENVINO code to include internal headers. + +if(WIN32) + set(OPENVINO_LIB + "${OPENVINO_INSTALL_DIR}/lib/openvino.lib" + CACHE FILEPATH "OPENVINO static library." FORCE) +elseif(APPLE) + set(OPENVINO_LIB + "${OPENVINO_INSTALL_DIR}/lib/libopenvino.dylib" + CACHE FILEPATH "OPENVINO static library." FORCE) +else() + set(OPENVINO_LIB + "${OPENVINO_INSTALL_DIR}/lib/intel64/libopenvino.so" + CACHE FILEPATH "OPENVINO static library." FORCE) +endif() + +ExternalProject_Add( + ${OPENVINO_PROJECT} + ${EXTERNAL_PROJECT_LOG_ARGS} + URL ${OPENVINO_URL} + PREFIX ${OPENVINO_PREFIX_DIR} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} -E remove_directory ${OPENVINO_INSTALL_DIR} && + ${CMAKE_COMMAND} -E make_directory ${OPENVINO_INSTALL_DIR} && + ${CMAKE_COMMAND} -E rename ${OPENVINO_SOURCE_DIR}/lib/ ${OPENVINO_INSTALL_DIR}/lib && + ${CMAKE_COMMAND} -E copy_directory ${OPENVINO_SOURCE_DIR}/include + ${OPENVINO_INC_DIR} + BUILD_BYPRODUCTS ${OPENVINO_LIB}) + +add_library(external_openvino STATIC IMPORTED GLOBAL) +set_property(TARGET external_openvino PROPERTY IMPORTED_LOCATION ${OPENVINO_LIB}) +add_dependencies(external_openvino ${OPENVINO_PROJECT}) diff --git a/external/summary.cmake b/external/summary.cmake index 754af9c3ec..86785d6c97 100644 --- a/external/summary.cmake +++ b/external/summary.cmake @@ -33,12 +33,16 @@ function(fastdeploy_summary) message(STATUS " ENABLE_ORT_BACKEND : ${ENABLE_ORT_BACKEND}") message(STATUS " ENABLE_PADDLE_BACKEND : ${ENABLE_PADDLE_BACKEND}") message(STATUS " ENABLE_TRT_BACKEND : ${ENABLE_TRT_BACKEND}") + message(STATUS " ENABLE_OPENVINO_BACKEND : ${ENABLE_OPENVINO_BACKEND}") if(ENABLE_ORT_BACKEND) message(STATUS " ONNXRuntime version : ${ONNXRUNTIME_VERSION}") endif() if(ENABLE_PADDLE_BACKEND) message(STATUS " Paddle Inference version : ${PADDLEINFERENCE_VERSION}") endif() + if(ENABLE_OPENVINO_BACKEND) + message(STATUS " OpenVINO version : ${OPENVINO_VERSION}") + endif() if(WITH_GPU) message(STATUS " WITH_GPU : ${WITH_GPU}") message(STATUS " CUDA_DIRECTORY : ${CUDA_DIRECTORY}") diff --git a/fastdeploy/c_lib_wrap.py.in b/fastdeploy/c_lib_wrap.py.in index 15b37b5c9b..d91e9c8f8e 100644 --- a/fastdeploy/c_lib_wrap.py.in +++ b/fastdeploy/c_lib_wrap.py.in @@ -32,6 +32,9 @@ def is_built_with_trt() -> bool: def is_built_with_paddle() -> bool: return True if "@ENABLE_PADDLE_BACKEND@" == "ON" else False +def is_built_with_openvino() ->bool: + return True if "@ENABLE_OPENVINO_BACKEND@" == "ON" else False + def get_default_cuda_directory() -> str: if not is_built_with_gpu(): From f95018043eb10c098c9e83f9dc12c15578f1eec0 Mon Sep 17 00:00:00 2001 From: jiangjiajun Date: Tue, 23 Aug 2022 12:28:01 +0000 Subject: [PATCH 2/3] fix pybind --- csrc/fastdeploy/pybind/fastdeploy_runtime.cc | 1 + fastdeploy/runtime.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/csrc/fastdeploy/pybind/fastdeploy_runtime.cc b/csrc/fastdeploy/pybind/fastdeploy_runtime.cc index 86e5b69c75..92f14bc6c5 100644 --- a/csrc/fastdeploy/pybind/fastdeploy_runtime.cc +++ b/csrc/fastdeploy/pybind/fastdeploy_runtime.cc @@ -26,6 +26,7 @@ void BindRuntime(pybind11::module& m) { .def("use_paddle_backend", &RuntimeOption::UsePaddleBackend) .def("use_ort_backend", &RuntimeOption::UseOrtBackend) .def("use_trt_backend", &RuntimeOption::UseTrtBackend) + .def("use_openvino_backend", &RuntimeOption::UseOpenVINOBackend) .def("enable_paddle_mkldnn", &RuntimeOption::EnablePaddleMKLDNN) .def("disable_paddle_mkldnn", &RuntimeOption::DisablePaddleMKLDNN) .def("enable_paddle_log_info", &RuntimeOption::EnablePaddleLogInfo) diff --git a/fastdeploy/runtime.py b/fastdeploy/runtime.py index 9613a344bc..b22c34935b 100644 --- a/fastdeploy/runtime.py +++ b/fastdeploy/runtime.py @@ -76,6 +76,9 @@ def use_ort_backend(self): def use_trt_backend(self): return self._option.use_trt_backend() + def use_openvino_backend(self): + return self._option.use_openvino_backend() + def enable_paddle_mkldnn(self): return self._option.enable_paddle_mkldnn() From adba608d424bc58ac238ff13b323c87705e3e096 Mon Sep 17 00:00:00 2001 From: jiangjiajun Date: Tue, 23 Aug 2022 13:34:31 +0000 Subject: [PATCH 3/3] fix python library path --- FastDeploy.cmake.in | 2 +- .../backends/openvino/ov_backend.cc | 41 +++++++++++++++++++ csrc/fastdeploy/fastdeploy_runtime.cc | 11 +++-- .../vision/detection/ppdet/ppyoloe.cc | 2 +- external/openvino.cmake | 6 +-- setup.py | 1 + 6 files changed, 55 insertions(+), 8 deletions(-) diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in index 2f45574705..ecf7497b70 100644 --- a/FastDeploy.cmake.in +++ b/FastDeploy.cmake.in @@ -48,7 +48,7 @@ if(ENABLE_PADDLE_BACKEND) endif() if(ENABLE_OPENVINO_BACKEND) - find_library(OPENVINO_LIB openvino ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/openvino/lib/intel64 NO_DEFAULT_PATH) + find_library(OPENVINO_LIB openvino ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/openvino/lib/ NO_DEFAULT_PATH) list(APPEND FASTDEPLOY_LIBS ${OPENVINO_LIB}) endif() diff --git a/csrc/fastdeploy/backends/openvino/ov_backend.cc b/csrc/fastdeploy/backends/openvino/ov_backend.cc index 2b71b9a6fc..406dfa1138 100644 --- a/csrc/fastdeploy/backends/openvino/ov_backend.cc +++ b/csrc/fastdeploy/backends/openvino/ov_backend.cc @@ -118,6 +118,47 @@ TensorInfo OpenVINOBackend::GetOutputInfo(int index) { return output_infos_[index]; } +bool OpenVINOBackend::InitFromOnnx(const std::string& model_file, + const OpenVINOBackendOption& option) { + if (initialized_) { + FDERROR << "OpenVINOBackend is already initlized, cannot initialize again." + << std::endl; + return false; + } + option_ = option; + ov::AnyMap properties; + if (option_.cpu_thread_num > 0) { + properties["INFERENCE_NUM_THREADS"] = option_.cpu_thread_num; + } + + std::shared_ptr model = core_.read_model(model_file); + + // Get inputs/outputs information from loaded model + const std::vector> inputs = model->inputs(); + for (size_t i = 0; i < inputs.size(); ++i) { + TensorInfo info; + auto partial_shape = PartialShapeToVec(inputs[i].get_partial_shape()); + info.shape.assign(partial_shape.begin(), partial_shape.end()); + info.name = inputs[i].get_any_name(); + info.dtype = OpenVINODataTypeToFD(inputs[i].get_element_type()); + input_infos_.emplace_back(info); + } + const std::vector> outputs = model->outputs(); + for (size_t i = 0; i < outputs.size(); ++i) { + TensorInfo info; + auto partial_shape = PartialShapeToVec(outputs[i].get_partial_shape()); + info.shape.assign(partial_shape.begin(), partial_shape.end()); + info.name = outputs[i].get_any_name(); + info.dtype = OpenVINODataTypeToFD(outputs[i].get_element_type()); + output_infos_.emplace_back(info); + } + + compiled_model_ = core_.compile_model(model, "CPU", properties); + request_ = compiled_model_.create_infer_request(); + initialized_ = true; + return true; +} + int OpenVINOBackend::NumInputs() const { return input_infos_.size(); } int OpenVINOBackend::NumOutputs() const { return output_infos_.size(); } diff --git a/csrc/fastdeploy/fastdeploy_runtime.cc b/csrc/fastdeploy/fastdeploy_runtime.cc index fcd4d10904..2e73af38c6 100644 --- a/csrc/fastdeploy/fastdeploy_runtime.cc +++ b/csrc/fastdeploy/fastdeploy_runtime.cc @@ -332,9 +332,14 @@ void Runtime::CreateOpenVINOBackend() { backend_ = utils::make_unique(); auto casted_backend = dynamic_cast(backend_.get()); - FDASSERT(casted_backend->InitFromPaddle(option.model_file, option.params_file, - ov_option), - "Load model from Paddle failed while initliazing PaddleBackend."); + if (option.model_format == Frontend::ONNX) { + FDASSERT(casted_backend->InitFromOnnx(option.model_file, ov_option), + "Load model from ONNX failed while initliazing OrtBackend."); + } else { + FDASSERT(casted_backend->InitFromPaddle(option.model_file, + option.params_file, ov_option), + "Load model from Paddle failed while initliazing OrtBackend."); + } #else FDASSERT(false, "OpenVINOBackend is not available, please compiled with " diff --git a/csrc/fastdeploy/vision/detection/ppdet/ppyoloe.cc b/csrc/fastdeploy/vision/detection/ppdet/ppyoloe.cc index 0b675ebc6a..e2d501b76d 100644 --- a/csrc/fastdeploy/vision/detection/ppdet/ppyoloe.cc +++ b/csrc/fastdeploy/vision/detection/ppdet/ppyoloe.cc @@ -14,7 +14,7 @@ PPYOLOE::PPYOLOE(const std::string& model_file, const std::string& params_file, const RuntimeOption& custom_option, const Frontend& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::ORT, Backend::PDINFER}; + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, Backend::PDINFER}; valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/external/openvino.cmake b/external/openvino.cmake index 242e5c2b8b..db9b0b105b 100644 --- a/external/openvino.cmake +++ b/external/openvino.cmake @@ -23,7 +23,7 @@ set(OPENVINO_INC_DIR "${OPENVINO_INSTALL_DIR}/include" CACHE PATH "openvino include directory." FORCE) set(OPENVINO_LIB_DIR - "${OPENVINO_INSTALL_DIR}/lib" + "${OPENVINO_INSTALL_DIR}/lib/" CACHE PATH "openvino lib directory." FORCE) set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${OPENVINO_LIB_DIR}") @@ -65,7 +65,7 @@ elseif(APPLE) CACHE FILEPATH "OPENVINO static library." FORCE) else() set(OPENVINO_LIB - "${OPENVINO_INSTALL_DIR}/lib/intel64/libopenvino.so" + "${OPENVINO_INSTALL_DIR}/lib/libopenvino.so" CACHE FILEPATH "OPENVINO static library." FORCE) endif() @@ -81,7 +81,7 @@ ExternalProject_Add( INSTALL_COMMAND ${CMAKE_COMMAND} -E remove_directory ${OPENVINO_INSTALL_DIR} && ${CMAKE_COMMAND} -E make_directory ${OPENVINO_INSTALL_DIR} && - ${CMAKE_COMMAND} -E rename ${OPENVINO_SOURCE_DIR}/lib/ ${OPENVINO_INSTALL_DIR}/lib && + ${CMAKE_COMMAND} -E rename ${OPENVINO_SOURCE_DIR}/lib/intel64 ${OPENVINO_INSTALL_DIR}/lib && ${CMAKE_COMMAND} -E copy_directory ${OPENVINO_SOURCE_DIR}/include ${OPENVINO_INC_DIR} BUILD_BYPRODUCTS ${OPENVINO_LIB}) diff --git a/setup.py b/setup.py index 4d5cd70911..8700997311 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ setup_configs["ENABLE_PADDLE_FRONTEND"] = os.getenv("ENABLE_PADDLE_FRONTEND", "ON") setup_configs["ENABLE_ORT_BACKEND"] = os.getenv("ENABLE_ORT_BACKEND", "ON") +setup_configs["ENABLE_OPENVINO_BACKEND"] = os.getenv("ENABLE_OPENVINO_BACKEND", "OFF") setup_configs["ENABLE_PADDLE_BACKEND"] = os.getenv("ENABLE_PADDLE_BACKEND", "OFF") setup_configs["BUILD_DEMO"] = os.getenv("BUILD_DEMO", "ON")