Merge pull request #17 from jiwaszki/mk/ov_pybind_poc_async

Async infer and Blob dispatch

ngraph/cmake/external_protobuf.cmake

@@ -99,7 +99,7 @@ else()
             CXX_VISIBILITY_PRESET default
             C_VISIBILITY_PRESET default
             VISIBILITY_INLINES_HIDDEN OFF)
-        set_target_properties(libprotobuf libprotobuf-lite PROPERTIES
+        set_target_properties(${_proto_libs} libprotobuf-lite PROPERTIES
             COMPILE_FLAGS "-Wno-unused-variable -Wno-inconsistent-missing-override")
     endif()
 

ngraph/python/setup.py

@@ -158,7 +158,7 @@ def build_cmake(self, extension: Extension):
         ext_args = self.cmake_args.split() if self.cmake_args else []
         self.spawn(["cmake", "-H" + root_dir, "-B" + self.build_temp,
                     "-DCMAKE_BUILD_TYPE={}".format(self.config),
-                    "-DPYTHON_EXECUTABLE={}".format(sys.executable),
+                    "-DENABLE_PYTHON=ON",
                     "-DNGRAPH_PYTHON_BUILD_ENABLE=ON",
                     "-DNGRAPH_ONNX_IMPORT_ENABLE=ON"] + ext_args)
 

ngraph/python/src/openvino/inference_engine/__init__.py

@@ -12,6 +12,7 @@
 from openvino.pyopenvino import TensorDesc
 from openvino.pyopenvino import get_version
 from openvino.pyopenvino import StatusCode
+from openvino.pyopenvino import InferQueue
 from openvino.pyopenvino import Blob
 from openvino.pyopenvino import PreProcessInfo
 from openvino.pyopenvino import MeanVariant

ngraph/python/src/openvino/inference_engine/ie_api.py

@@ -3,15 +3,20 @@
 
 from openvino.pyopenvino import TBlobFloat32
 from openvino.pyopenvino import TBlobFloat64
+from openvino.pyopenvino import TBlobFloat16
 from openvino.pyopenvino import TBlobInt64
 from openvino.pyopenvino import TBlobUint64
 from openvino.pyopenvino import TBlobInt32
 from openvino.pyopenvino import TBlobUint32
 from openvino.pyopenvino import TBlobInt16
 from openvino.pyopenvino import TBlobUint16
 from openvino.pyopenvino import TBlobInt8
+from openvino.pyopenvino import TBlobInt16
+from openvino.pyopenvino import TBlobInt32
+from openvino.pyopenvino import TBlobInt64
 from openvino.pyopenvino import TBlobUint8
 from openvino.pyopenvino import TensorDesc
+from openvino.pyopenvino import TensorDesc
 
 import numpy as np
 
@@ -36,27 +41,26 @@ class BlobWrapper:
     def __new__(cls, tensor_desc : TensorDesc, arr : np.ndarray = None):
         arr_size = 0
         precision = ""
-        if tensor_desc is not None:
-            tensor_desc_size = int(np.prod(tensor_desc.dims))
+        if arr is not None and tensor_desc is not None:
+            arr = np.array(arr) # Keeping array as numpy array
+            arr_size = np.prod(arr.shape)
+            tensor_desc_size = np.prod(tensor_desc.dims)
             precision = tensor_desc.precision
-            if arr is not None:
-                arr = np.array(arr) # Keeping array as numpy array
-                arr_size = int(np.prod(arr.shape))
-                if np.isfortran(arr):
-                    arr = arr.ravel(order="F")
-                else:
-                    arr = arr.ravel(order="C")
-                if arr_size != tensor_desc_size:
-                    raise AttributeError(f'Number of elements in provided numpy array '
-                                         f'{arr_size} and required by TensorDesc '
-                                         f'{tensor_desc_size} are not equal')
-                if arr.dtype != precision_map[precision]:
-                    raise ValueError(f"Data type {arr.dtype} of provided numpy array "
-                                     f"doesn't match to TensorDesc precision {precision}")
-                if not arr.flags['C_CONTIGUOUS']:
-                    arr = np.ascontiguousarray(arr)
-            elif arr is None:
-                arr = np.empty(0, dtype=precision_map[precision])
+            if np.isfortran(arr):
+                arr = arr.ravel(order="F")
+            else:
+                arr = arr.ravel(order="C")
+            if arr_size != tensor_desc_size:
+                raise AttributeError(f'Number of elements in provided numpy array '
+                                     f'{arr_size} and required by TensorDesc '
+                                     f'{tensor_desc_size} are not equal')
+            if arr.dtype != precision_map[precision]:
+                raise ValueError(f"Data type {arr.dtype} of provided numpy array "
+                                 f"doesn't match to TensorDesc precision {precision}")
+            if not arr.flags['C_CONTIGUOUS']:
+                arr = np.ascontiguousarray(arr)
+        elif arr is None and tensor_desc is not None:
+            arr = np.empty(0, dtype=precision_map[precision])
         else:
             raise AttributeError("TensorDesc can't be None")
 
@@ -65,7 +69,7 @@ def __new__(cls, tensor_desc : TensorDesc, arr : np.ndarray = None):
         elif precision in ["FP64"]:
             return TBlobFloat64(tensor_desc, arr, arr_size)
         elif precision in ["FP16", "BF16"]:
-            return TBlobInt16(tensor_desc, arr.view(dtype=np.int16), arr_size)
+            return TBlobFloat16(tensor_desc, arr.view(dtype=np.int16), arr_size)
         elif precision in ["I64"]:
             return TBlobInt64(tensor_desc, arr, arr_size)
         elif precision in ["U64"]:

ngraph/python/src/pyopenvino/inference_engine/common.cpp

@@ -6,73 +6,81 @@
 
 #include "common.hpp"
 
-namespace Common {
-
-    namespace {
-        const std::unordered_map<int, std::string> layout_int_to_str_map = {{0,   "ANY"},
-                                                                            {1,   "NCHW"},
-                                                                            {2,   "NHWC"},
-                                                                            {3,   "NCDHW"},
-                                                                            {4,   "NDHWC"},
-                                                                            {64,  "OIHW"},
-                                                                            {95,  "SCALAR"},
-                                                                            {96,  "C"},
+namespace Common
+{
+    namespace
+    {
+        const std::unordered_map<int, std::string> layout_int_to_str_map = {{0, "ANY"},
+                                                                            {1, "NCHW"},
+                                                                            {2, "NHWC"},
+                                                                            {3, "NCDHW"},
+                                                                            {4, "NDHWC"},
+                                                                            {64, "OIHW"},
+                                                                            {95, "SCALAR"},
+                                                                            {96, "C"},
                                                                             {128, "CHW"},
                                                                             {192, "HW"},
                                                                             {193, "NC"},
                                                                             {194, "CN"},
                                                                             {200, "BLOCKED"}};
 
-        const std::unordered_map <std::string, InferenceEngine::Layout> layout_str_to_enum = {
-                {"ANY",     InferenceEngine::Layout::ANY},
-                {"NHWC",    InferenceEngine::Layout::NHWC},
-                {"NCHW",    InferenceEngine::Layout::NCHW},
-                {"NCDHW",   InferenceEngine::Layout::NCDHW},
-                {"NDHWC",   InferenceEngine::Layout::NDHWC},
-                {"OIHW",    InferenceEngine::Layout::OIHW},
-                {"GOIHW",   InferenceEngine::Layout::GOIHW},
-                {"OIDHW",   InferenceEngine::Layout::OIDHW},
-                {"GOIDHW",  InferenceEngine::Layout::GOIDHW},
-                {"SCALAR",  InferenceEngine::Layout::SCALAR},
-                {"C",       InferenceEngine::Layout::C},
-                {"CHW",     InferenceEngine::Layout::CHW},
-                {"HW",      InferenceEngine::Layout::HW},
-                {"NC",      InferenceEngine::Layout::NC},
-                {"CN",      InferenceEngine::Layout::CN},
-                {"BLOCKED", InferenceEngine::Layout::BLOCKED}
-        };
-    }
+        const std::unordered_map<std::string, InferenceEngine::Layout> layout_str_to_enum = {
+            {"ANY", InferenceEngine::Layout::ANY},
+            {"NHWC", InferenceEngine::Layout::NHWC},
+            {"NCHW", InferenceEngine::Layout::NCHW},
+            {"NCDHW", InferenceEngine::Layout::NCDHW},
+            {"NDHWC", InferenceEngine::Layout::NDHWC},
+            {"OIHW", InferenceEngine::Layout::OIHW},
+            {"GOIHW", InferenceEngine::Layout::GOIHW},
+            {"OIDHW", InferenceEngine::Layout::OIDHW},
+            {"GOIDHW", InferenceEngine::Layout::GOIDHW},
+            {"SCALAR", InferenceEngine::Layout::SCALAR},
+            {"C", InferenceEngine::Layout::C},
+            {"CHW", InferenceEngine::Layout::CHW},
+            {"HW", InferenceEngine::Layout::HW},
+            {"NC", InferenceEngine::Layout::NC},
+            {"CN", InferenceEngine::Layout::CN},
+            {"BLOCKED", InferenceEngine::Layout::BLOCKED}};
+    } // namespace
 
-    InferenceEngine::Layout get_layout_from_string(const std::string &layout) {
+    InferenceEngine::Layout get_layout_from_string(const std::string& layout)
+    {
         return layout_str_to_enum.at(layout);
     }
 
-    const std::string &get_layout_from_enum(const InferenceEngine::Layout &layout) {
+    const std::string& get_layout_from_enum(const InferenceEngine::Layout& layout)
+    {
         return layout_int_to_str_map.at(layout);
     }
 
-    PyObject *parse_parameter(const InferenceEngine::Parameter &param) {
+    PyObject* parse_parameter(const InferenceEngine::Parameter& param)
+    {
         // Check for std::string
-        if (param.is<std::string>()) {
+        if (param.is<std::string>())
+        {
             return PyUnicode_FromString(param.as<std::string>().c_str());
         }
-            // Check for int
-        else if (param.is<int>()) {
+        // Check for int
+        else if (param.is<int>())
+        {
             auto val = param.as<int>();
-            return PyLong_FromLong((long) val);
+            return PyLong_FromLong((long)val);
         }
-            // Check for unsinged int
-        else if (param.is<unsigned int>()) {
+        // Check for unsinged int
+        else if (param.is<unsigned int>())
+        {
             auto val = param.as<unsigned int>();
-            return PyLong_FromLong((unsigned long) val);
+            return PyLong_FromLong((unsigned long)val);
         }
-            // Check for float
-        else if (param.is<float>()) {
+        // Check for float
+        else if (param.is<float>())
+        {
             auto val = param.as<float>();
-            return PyFloat_FromDouble((double) val);
+            return PyFloat_FromDouble((double)val);
         }
-            // Check for bool
-        else if (param.is<bool>()) {
+        // Check for bool
+        else if (param.is<bool>())
+        {
             auto val = param.as<bool>();
             return val ? Py_True : Py_False;
         }
@@ -95,11 +103,13 @@ namespace Common {
             }
             return list;
         }
-            // Check for std::vector<unsigned int>
-        else if (param.is < std::vector < unsigned int >> ()) {
-            auto val = param.as < std::vector < unsigned int >> ();
-            PyObject *list = PyList_New(0);
-            for (const auto &it : val) {
+        // Check for std::vector<unsigned int>
+        else if (param.is<std::vector<unsigned int>>())
+        {
+            auto val = param.as<std::vector<unsigned int>>();
+            PyObject* list = PyList_New(0);
+            for (const auto& it : val)
+            {
                 PyList_Append(list, PyLong_FromLong(it));
             }
             return list;
@@ -147,13 +157,64 @@ namespace Common {
                 PyDict_SetItemString(dict, it.first.c_str(), PyLong_FromLong((long) it.second));
             }
             return dict;
-        } else {
-            PyErr_SetString(PyExc_TypeError, "Failed to convert parameter to Python representation!");
-            return (PyObject *) NULL;
+        }
+        else
+        {
+            PyErr_SetString(PyExc_TypeError,
+                            "Failed to convert parameter to Python representation!");
+            return (PyObject*)NULL;
         }
     }
 
-    const std::shared_ptr<InferenceEngine::Blob> cast_to_blob(const py::handle& blob) {
+    const std::shared_ptr<InferenceEngine::Blob> convert_to_blob(const py::handle& blob)
+    {
+        if (py::isinstance<InferenceEngine::TBlob<float>>(blob))
+        {
+            return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<float>>&>();
+        }
+        else if (py::isinstance<InferenceEngine::TBlob<double>>(blob))
+        {
+            return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<double>>&>();
+        }
+        else if (py::isinstance<InferenceEngine::TBlob<int8_t>>(blob))
+        {
+            return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<int8_t>>&>();
+        }
+        else if (py::isinstance<InferenceEngine::TBlob<int16_t>>(blob))
+        {
+            return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<int16_t>>&>();
+        }
+        else if (py::isinstance<InferenceEngine::TBlob<int32_t>>(blob))
+        {
+            return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<int32_t>>&>();
+        }
+        else if (py::isinstance<InferenceEngine::TBlob<int64_t>>(blob))
+        {
+            return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<int64_t>>&>();
+        }
+        else if (py::isinstance<InferenceEngine::TBlob<uint8_t>>(blob))
+        {
+            return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<uint8_t>>&>();
+        }
+        else if (py::isinstance<InferenceEngine::TBlob<uint16_t>>(blob))
+        {
+            return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<uint16_t>>&>();
+        }
+        else if (py::isinstance<InferenceEngine::TBlob<uint32_t>>(blob))
+        {
+            return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<uint32_t>>&>();
+        }
+        else if (py::isinstance<InferenceEngine::TBlob<uint64_t>>(blob))
+        {
+            return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<uint64_t>>&>();
+        }
+        else
+        {
+            // Throw error
+        }
+    }
+
+    const std::shared_ptr<InferenceEngine::Blob> convert_to_blob(const py::handle& blob) {
         if (py::isinstance<InferenceEngine::TBlob<float>>(blob)) {
             return blob.cast<const std::shared_ptr<InferenceEngine::TBlob<float>> &>();
         } else if (py::isinstance<InferenceEngine::TBlob<double>>(blob)) {
@@ -178,5 +239,4 @@ namespace Common {
             // Throw error
         }
     }
-
 };

ngraph/python/src/pyopenvino/inference_engine/common.hpp

@@ -7,18 +7,21 @@
 #include <pybind11/pybind11.h>
 #include <string>
 #include "Python.h"
-#include <ie_parameter.hpp>
+#include "ie_blob.h"
 #include "ie_common.h"
 #include <ie_blob.h>
 
 namespace py = pybind11;
 
-namespace Common {
-    InferenceEngine::Layout get_layout_from_string(const std::string &layout);
+namespace Common
+{
+    InferenceEngine::Layout get_layout_from_string(const std::string& layout);
 
-    const std::string& get_layout_from_enum(const InferenceEngine::Layout &layout);
+    const std::string& get_layout_from_enum(const InferenceEngine::Layout& layout);
+
+    PyObject* parse_parameter(const InferenceEngine::Parameter& param);
 
     PyObject *parse_parameter(const InferenceEngine::Parameter &param);
 
-    const std::shared_ptr<InferenceEngine::Blob> cast_to_blob(const py::handle& blob);
+    const std::shared_ptr<InferenceEngine::Blob> convert_to_blob(const py::handle& blob);
 }; // namespace Common

ngraph/python/src/pyopenvino/inference_engine/ie_blob.hpp

@@ -20,7 +20,8 @@ namespace py = pybind11;
 void regclass_Blob(py::module m);
 
 template <typename T>
-void regclass_TBlob(py::module m, std::string typestring) {
+void regclass_TBlob(py::module m, std::string typestring)
+{
     auto pyclass_name = py::detail::c_str((std::string("TBlob") + typestring));
 
     py::class_<InferenceEngine::TBlob<T>, std::shared_ptr<InferenceEngine::TBlob<T>>> cls(
@@ -42,7 +43,6 @@ void regclass_TBlob(py::module m, std::string typestring) {
         return py::array_t<T>(shape, &blob_ptr[0], py::cast(self));
     });
 
-    cls.def_property_readonly("tensor_desc",  [](InferenceEngine::TBlob<T>& self) {
-        return self.getTensorDesc();
-    });
+    cls.def_property_readonly("tensor_desc",
+                              [](InferenceEngine::TBlob<T>& self) { return self.getTensorDesc(); });
 }

ngraph/python/src/pyopenvino/inference_engine/ie_executable_network.cpp

@@ -26,7 +26,9 @@ void regclass_ExecutableNetwork(py::module m)
                std::shared_ptr<InferenceEngine::ExecutableNetwork>>
         cls(m, "ExecutableNetwork");
 
-    cls.def("create_infer_request", &InferenceEngine::ExecutableNetwork::CreateInferRequest);
+    cls.def("create_infer_request", [](InferenceEngine::ExecutableNetwork& self) {
+        return static_cast<InferenceEngine::InferRequest>(self.CreateInferRequest());
+    });
 
     cls.def("get_exec_graph_info", &InferenceEngine::ExecutableNetwork::GetExecGraphInfo);