add-mmpose-sdk

configs/mmpose/pose-detection_sdk_static.py

@@ -0,0 +1,14 @@
+_base_ = ['./pose-detection_static.py', '../_base_/backends/sdk.py']
+
+codebase_config = dict(model_type='sdk')
+
+backend_config = dict(pipeline=[
+    dict(type='LoadImageFromFile', channel_order='bgr'),
+    dict(
+        type='Collect',
+        keys=['img'],
+        meta_keys=[
+            'image_file', 'center', 'scale', 'rotation', 'bbox_score',
+            'flip_pairs'
+        ])
+])

csrc/apis/c/CMakeLists.txt

@@ -5,7 +5,7 @@ project(capis)
 include(${CMAKE_SOURCE_DIR}/cmake/MMDeploy.cmake)
 
 if ("all" IN_LIST MMDEPLOY_CODEBASES)
-    set(TASK_LIST "classifier;detector;segmentor;text_detector;text_recognizer;restorer;model")
+    set(TASK_LIST "classifier;detector;segmentor;text_detector;text_recognizer;pose_detector;restorer;model")
 else ()
     set(TASK_LIST "model")
     if ("mmcls" IN_LIST MMDEPLOY_CODEBASES)
@@ -24,6 +24,9 @@ else ()
         list(APPEND TASK_LIST "text_detector")
         list(APPEND TASK_LIST "text_recognizer")
     endif ()
+    if ("mmpose" IN_LIST MMDEPLOY_CODEBASES)
+        list(APPEND TASK_LIST "pose_detector")
+    endif ()
 endif ()
 
 foreach (TASK ${TASK_LIST})

csrc/apis/c/pose_detector.cpp

@@ -0,0 +1,190 @@
+// Copyright (c) OpenMMLab. All rights reserved.
+
+#include "pose_detector.h"
+
+#include <numeric>
+
+#include "codebase/mmpose/mmpose.h"
+#include "core/device.h"
+#include "core/graph.h"
+#include "core/mat.h"
+#include "core/tensor.h"
+#include "core/utils/formatter.h"
+#include "handle.h"
+
+using namespace std;
+using namespace mmdeploy;
+
+namespace {
+
+const Value& config_template() {
+  // clang-format off
+  static Value v {
+    {
+      "pipeline", {
+        {"input", {"img_with_boxes"}},
+        {"output", {"key_points_unflat"}},
+        {
+          "tasks", {
+            {
+              {"name", "flatten"},
+              {"type", "Flatten"},
+              {"input", {"img_with_boxes"}},
+              {"output", {"patch_flat", "patch_index"}},
+            },
+            {
+              {"name", "pose-detector"},
+              {"type", "Inference"},
+              {"params", {{"model", "TBD"},{"batch_size", 1}}},
+              {"input", {"patch_flat"}},
+              {"output", {"key_points"}}
+            },
+            {
+              {"name", "unflatten"},
+              {"type", "Unflatten"},
+              {"input", {"key_points", "patch_index"}},
+              {"output", {"key_points_unflat"}},
+            }
+          }
+        }
+      }
+    }
+  };
+  // clang-format on
+  return v;
+}
+
+template <class ModelType>
+int mmdeploy_pose_detector_create_impl(ModelType&& m, const char* device_name, int device_id,
+                                       mm_handle_t* handle) {
+  try {
+    auto value = config_template();
+    value["pipeline"]["tasks"][1]["params"]["model"] = std::forward<ModelType>(m);
+
+    auto pose_estimator = std::make_unique<Handle>(device_name, device_id, std::move(value));
+
+    *handle = pose_estimator.release();
+    return MM_SUCCESS;
+
+  } catch (const std::exception& e) {
+    MMDEPLOY_ERROR("exception caught: {}", e.what());
+  } catch (...) {
+    MMDEPLOY_ERROR("unknown exception caught");
+  }
+  return MM_E_FAIL;
+}
+
+}  // namespace
+
+int mmdeploy_pose_detector_create(mm_model_t model, const char* device_name, int device_id,
+                                  mm_handle_t* handle) {
+  return mmdeploy_pose_detector_create_impl(*static_cast<Model*>(model), device_name, device_id,
+                                            handle);
+}
+
+int mmdeploy_pose_detector_create_by_path(const char* model_path, const char* device_name,
+                                          int device_id, mm_handle_t* handle) {
+  return mmdeploy_pose_detector_create_impl(model_path, device_name, device_id, handle);
+}
+
+int mmdeploy_pose_detector_apply(mm_handle_t handle, const mm_mat_t* mats, int mat_count,
+                                 mm_pose_detect_t** results, int** result_count) {
+  return mmdeploy_pose_detector_apply_bbox(handle, mats, mat_count, nullptr, nullptr, results);
+}
+
+int mmdeploy_pose_detector_apply_bbox(mm_handle_t handle, const mm_mat_t* mats, int mat_count,
+                                      const mm_rect_t* bboxes, const int* bbox_count,
+                                      mm_pose_detect_t** results) {
+  if (handle == nullptr || mats == nullptr || mat_count == 0 || results == nullptr) {
+    return MM_E_INVALID_ARG;
+  }
+
+  try {
+    auto pose_detector = static_cast<Handle*>(handle);
+    Value input{Value::kArray};
+    auto result_count = 0;
+    for (int i = 0; i < mat_count; ++i) {
+      mmdeploy::Mat _mat{mats[i].height,       mats[i].width, PixelFormat(mats[i].format),
+                         DataType(mats->type), mats[i].data,  Device{"cpu"}};
+
+      Value img_with_boxes;
+      if (bboxes && bbox_count) {
+        for (int j = 0; j < bbox_count[i]; ++j) {
+          Value obj;
+          obj["ori_img"] = _mat;
+          float width = bboxes[j].right - bboxes[j].left + 1;
+          float height = bboxes[j].bottom - bboxes[j].top + 1;
+          obj["box"] = {bboxes[j].left, bboxes[j].top, width, height, 1.0};
+          obj["rotation"] = 0.f;
+          img_with_boxes.push_back(obj);
+        }
+        bboxes += bbox_count[i];
+        result_count += bbox_count[i];
+      } else {
+        // inference whole image
+        Value obj;
+        obj["ori_img"] = _mat;
+        obj["box"] = {0, 0, _mat.width(), _mat.height(), 1.0};
+        obj["rotation"] = 0.f;
+        img_with_boxes.push_back(obj);
+        result_count += 1;
+      }
+      input.front().push_back(img_with_boxes);
+    }
+
+    auto output = pose_detector->Run(std::move(input)).value().front();
+
+    auto pose_outputs = from_value<vector<vector<mmpose::PoseDetectorOutput>>>(output);
+
+    std::vector<int> counts;
+    if (bboxes && bbox_count) {
+      counts = std::vector<int>(bbox_count, bbox_count + mat_count);
+    } else {
+      counts.resize(mat_count, 1);
+    }
+    std::vector<int> offsets{0};
+    std::partial_sum(begin(counts), end(counts), back_inserter(offsets));
+
+    auto deleter = [&](mm_pose_detect_t* p) {
+      mmdeploy_pose_detector_release_result(p, offsets.back());
+    };
+
+    std::unique_ptr<mm_pose_detect_t[], decltype(deleter)> _results(
+        new mm_pose_detect_t[result_count]{}, deleter);
+
+    for (int i = 0; i < mat_count; ++i) {
+      auto& pose_output = pose_outputs[i];
+      for (int j = 0; j < pose_output.size(); ++j) {
+        auto& res = _results[offsets[i] + j];
+        auto& box_result = pose_output[j];
+        int sz = box_result.key_points.size();
+
+        res.point = new mm_pointf_t[sz];
+        res.score = new float[sz];
+        res.length = sz;
+        for (int k = 0; k < sz; k++) {
+          res.point[k].x = box_result.key_points[k].bbox[0];
+          res.point[k].y = box_result.key_points[k].bbox[1];
+          res.score[k] = box_result.key_points[k].score;
+        }
+      }
+    }
+    *results = _results.release();
+    return MM_SUCCESS;
+
+  } catch (const std::exception& e) {
+    MMDEPLOY_ERROR("exception caught: {}", e.what());
+  } catch (...) {
+    MMDEPLOY_ERROR("unknown exception caught");
+  }
+  return MM_E_FAIL;
+}
+
+void mmdeploy_pose_detector_release_result(mm_pose_detect_t* results, int count) {
+  for (int i = 0; i < count; ++i) {
+    delete[] results[i].point;
+    delete[] results[i].score;
+  }
+  delete[] results;
+}
+void mmdeploy_pose_detector_destroy(mm_handle_t handle) { delete static_cast<Handle*>(handle); }

csrc/apis/c/pose_detector.h

@@ -0,0 +1,98 @@
+// Copyright (c) OpenMMLab. All rights reserved.
+
+/**
+ * @file pose_detector.h
+ * @brief Interface to MMPose task
+ */
+
+#ifndef MMDEPLOY_SRC_APIS_C_POSE_DETECTOR_H_
+#define MMDEPLOY_SRC_APIS_C_POSE_DETECTOR_H_
+
+#include "common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct mm_pose_detect_t {
+  mm_pointf_t* point;
+  float* score;
+  int length;
+} mm_pose_detect_t;
+
+/**
+ * @brief Create a pose detector instance
+ * @param[in] model an instance of mmpose model created by
+ * \ref mmdeploy_model_create_by_path or \ref mmdeploy_model_create in \ref model.h
+ * @param[in] device_name name of device, such as "cpu", "cuda", etc.
+ * @param[in] device_id id of device.
+ * @param[out] handle handle of the created pose detector, which must be destroyed
+ * by \ref mmdeploy_pose_detector_destroy
+ * @return status code of the operation
+ */
+MMDEPLOY_API int mmdeploy_pose_detector_create(mm_model_t model, const char* device_name,
+                                               int device_id, mm_handle_t* handle);
+
+/**
+ * @brief Create a pose detector instance
+ * @param[in] model_path path to pose detection model
+ * @param[in] device_name name of device, such as "cpu", "cuda", etc.
+ * @param[in] device_id id of device.
+ * @param[out] handle handle of the created pose detector, which must be destroyed
+ * by \ref mmdeploy_pose_detector_destroy
+ * @return status code of the operation
+ */
+MMDEPLOY_API int mmdeploy_pose_detector_create_by_path(const char* model_path,
+                                                       const char* device_name, int device_id,
+                                                       mm_handle_t* handle);
+
+/**
+ * @brief Apply pose detector to a batch of images with full image roi
+ * @param[in] handle pose detector's handle created by \ref
+ * mmdeploy_pose_detector_create_by_path
+ * @param[in] images a batch of images
+ * @param[in] count number of images in the batch
+ * @param[out] results a linear buffer contains the pose result, must be release
+ * by \ref mmdeploy_pose_detector_release_result
+ * @return status code of the operation
+ */
+MMDEPLOY_API int mmdeploy_pose_detector_apply(mm_handle_t handle, const mm_mat_t* mats,
+                                              int mat_count, mm_pose_detect_t** results,
+                                              int** result_count);
+
+/**
+ * @brief Apply pose detector to a batch of images supplied with bboxes(roi)
+ * @param[in] handle pose detector's handle created by \ref
+ * mmdeploy_pose_detector_create_by_path
+ * @param[in] images a batch of images
+ * @param[in] image_count number of images in the batch
+ * @param[in] bboxes bounding boxes(roi) detected by mmdet
+ * @param[in] bbox_count number of bboxes of each \p images, must be same length as \p images
+ * @param[out] results a linear buffer contains the pose result, which has the same length as \p
+ * bboxes, must be release by \ref mmdeploy_pose_detector_release_result
+ * @return status code of the operation
+ */
+MMDEPLOY_API int mmdeploy_pose_detector_apply_bbox(mm_handle_t handle, const mm_mat_t* mats,
+                                                   int mat_count, const mm_rect_t* bboxes,
+                                                   const int* bbox_count,
+                                                   mm_pose_detect_t** results);
+
+/** @brief Release result buffer returned by \ref mmdeploy_pose_detector_apply or \ref
+ * mmdeploy_pose_detector_apply_bbox
+ * @param[in] results result buffer by pose detector
+ * @param[in] count length of \p result
+ */
+MMDEPLOY_API void mmdeploy_pose_detector_release_result(mm_pose_detect_t* results, int count);
+
+/**
+ * @brief destroy pose_detector
+ * @param[in] handle handle of pose_detector created by \ref
+ * mmdeploy_pose_detector_create_by_path or \ref mmdeploy_pose_detector_create
+ */
+MMDEPLOY_API void mmdeploy_pose_detector_destroy(mm_handle_t handle);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // MMDEPLOY_SRC_APIS_C_POSE_DETECTOR_H_

csrc/apis/python/CMakeLists.txt

@@ -24,6 +24,7 @@ mmdeploy_python_add_module(segmentor)
 mmdeploy_python_add_module(text_detector)
 mmdeploy_python_add_module(text_recognizer)
 mmdeploy_python_add_module(restorer)
+mmdeploy_python_add_module(pose_detector)
 
 pybind11_add_module(${PROJECT_NAME} ${MMDEPLOY_PYTHON_SRCS})