[MOT] add FairMot c++ deploy (PaddlePaddle#4322)

* add fairmot deploy

* separate main_jde

* update copyright & add PrintBenchmark

* refine lap

configs/mot/fairmot/_base_/fairmot_reader_1088x608.yml

@@ -25,7 +25,7 @@ EvalMOTReader:
   sample_transforms:
     - Decode: {}
     - LetterBoxResize: {target_size: [608, 1088]}
-    - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1]}
+    - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1], is_scale: True}
     - Permute: {}
   batch_size: 1
 
@@ -36,6 +36,6 @@ TestMOTReader:
   sample_transforms:
     - Decode: {}
     - LetterBoxResize: {target_size: [608, 1088]}
-    - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1]}
+    - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1], is_scale: True}
     - Permute: {}
   batch_size: 1

configs/mot/fairmot/_base_/fairmot_reader_576x320.yml

@@ -36,6 +36,6 @@ TestMOTReader:
   sample_transforms:
     - Decode: {}
     - LetterBoxResize: {target_size: [320, 576]}
-    - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1]}
+    - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1], is_scale: True}
     - Permute: {}
   batch_size: 1

configs/mot/fairmot/_base_/fairmot_reader_864x480.yml

@@ -36,6 +36,6 @@ TestMOTReader:
   sample_transforms:
     - Decode: {}
     - LetterBoxResize: {target_size: [480, 864]}
-    - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1]}
+    - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1], is_scale: True}
     - Permute: {}
   batch_size: 1

deploy/cpp/CMakeLists.txt

@@ -6,6 +6,7 @@ option(WITH_GPU        "Compile demo with GPU/CPU, default use CPU."
 option(WITH_TENSORRT   "Compile demo with TensorRT."                                    OFF)
 
 option(WITH_KEYPOINT        "Whether to Compile KeyPoint detector"                    OFF)
+option(WITH_MOT       "Whether to Compile MOT detector" OFF)
 
 SET(PADDLE_DIR "" CACHE PATH "Location of libraries")
 SET(PADDLE_LIB_NAME "" CACHE STRING "libpaddle_inference")
@@ -23,6 +24,8 @@ link_directories("${CMAKE_CURRENT_BINARY_DIR}/ext/yaml-cpp/lib")
 
 if (WITH_KEYPOINT)
     set(SRCS src/main_keypoint.cc src/preprocess_op.cc src/object_detector.cc src/picodet_postprocess.cc src/utils.cc src/keypoint_detector.cc src/keypoint_postprocess.cc)
+elseif (WITH_MOT)
+    set(SRCS src/main_jde.cc src/preprocess_op.cc src/object_detector.cc src/jde_detector.cc src/tracker.cc src/trajectory.cc src/lapjv.cpp src/picodet_postprocess.cc src/utils.cc)
 else ()
     set(SRCS src/main.cc src/preprocess_op.cc src/object_detector.cc src/picodet_postprocess.cc src/utils.cc)
 endif()

deploy/cpp/include/config_parser.h

@@ -99,6 +99,16 @@ class ConfigPaser {
       return false;
     }
 
+    // Get conf_thresh for tracker
+    if (config["tracker"].IsDefined()) {
+      if (config["tracker"]["conf_thres"].IsDefined()) {
+        conf_thresh_ = config["tracker"]["conf_thres"].as<float>();
+      } else {
+        std::cerr << "Please set conf_thres in tracker." << std::endl;
+        return false;
+      }
+    }
+
     // Get NMS for postprocess
     if (config["NMS"].IsDefined()) {
       nms_info_ = config["NMS"];
@@ -122,6 +132,7 @@ class ConfigPaser {
   std::vector<std::string> label_list_;
   std::vector<int> fpn_stride_;
   bool use_dynamic_shape_;
+  float conf_thresh_;
 };
 
 }  // namespace PaddleDetection

deploy/cpp/include/jde_detector.h

@@ -0,0 +1,139 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <utility>
+#include <ctime>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+#include <opencv2/highgui/highgui.hpp>
+
+#include "paddle_inference_api.h" // NOLINT
+
+#include "include/preprocess_op.h"
+#include "include/config_parser.h"
+#include "include/tracker.h"
+
+using namespace paddle_infer;
+
+namespace PaddleDetection {
+// JDE Detection Result
+struct MOT_Rect
+{
+    float left;
+    float top;
+    float right;
+    float bottom;
+};
+
+struct MOT_Track
+{
+    int ids;
+    float score;
+    MOT_Rect rects;
+};
+
+typedef std::vector<MOT_Track> MOT_Result;
+
+// Generate visualization color
+cv::Scalar GetColor(int idx);
+
+// Visualiztion Detection Result
+cv::Mat VisualizeTrackResult(const cv::Mat& img,
+                     const MOT_Result& results,
+                     const float fps, const int frame_id);
+
+
+class JDEDetector {
+ public:
+  explicit JDEDetector(const std::string& model_dir, 
+                          const std::string& device="CPU",
+                          bool use_mkldnn=false,
+                          int cpu_threads=1,
+                          const std::string& run_mode="fluid",
+                          const int batch_size=1,
+                          const int gpu_id=0,
+                          const int trt_min_shape=1,
+                          const int trt_max_shape=1280,
+                          const int trt_opt_shape=640,
+                          bool trt_calib_mode=false,
+                          const int min_box_area=200) {
+    this->device_ = device;
+    this->gpu_id_ = gpu_id;
+    this->cpu_math_library_num_threads_ = cpu_threads;
+    this->use_mkldnn_ = use_mkldnn;
+
+    this->trt_min_shape_ = trt_min_shape;
+    this->trt_max_shape_ = trt_max_shape;
+    this->trt_opt_shape_ = trt_opt_shape;
+    this->trt_calib_mode_ = trt_calib_mode;
+    config_.load_config(model_dir);
+    this->use_dynamic_shape_ = config_.use_dynamic_shape_;
+    this->min_subgraph_size_ = config_.min_subgraph_size_;
+    threshold_ = config_.draw_threshold_;
+    preprocessor_.Init(config_.preprocess_info_);
+    LoadModel(model_dir, batch_size, run_mode);
+    this->min_box_area_ = min_box_area;
+    this->conf_thresh_ = config_.conf_thresh_;
+  }
+
+  // Load Paddle inference model
+  void LoadModel(
+    const std::string& model_dir,
+    const int batch_size = 1,
+    const std::string& run_mode = "fluid");
+
+  // Run predictor
+  void Predict(const std::vector<cv::Mat> imgs,
+      const double threshold = 0.5,
+      const int warmup = 0,
+      const int repeats = 1,
+      MOT_Result* result = nullptr,
+      std::vector<double>* times = nullptr);
+
+ private:
+  std::string device_ = "CPU";
+  int gpu_id_ = 0;
+  int cpu_math_library_num_threads_ = 1;
+  bool use_mkldnn_ = false;
+  int min_subgraph_size_ = 3;
+  bool use_dynamic_shape_ = false;
+  int trt_min_shape_ = 1;
+  int trt_max_shape_ = 1280;
+  int trt_opt_shape_ = 640;
+  bool trt_calib_mode_ = false;
+  // Preprocess image and copy data to input buffer
+  void Preprocess(const cv::Mat& image_mat);
+  // Postprocess result
+  void Postprocess(
+      const cv::Mat dets, const cv::Mat emb,
+      MOT_Result* result);
+
+  std::shared_ptr<Predictor> predictor_;
+  Preprocessor preprocessor_;
+  ImageBlob inputs_;
+  std::vector<float> bbox_data_;
+  std::vector<float> emb_data_;
+  float threshold_;
+  ConfigPaser config_;
+  float min_box_area_;
+  float conf_thresh_;
+};
+
+}  // namespace PaddleDetection

deploy/cpp/include/lapjv.h

@@ -0,0 +1,48 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#ifndef LAPJV_H
+#define LAPJV_H
+
+#define LARGE 1000000
+
+#if !defined TRUE
+#define TRUE 1
+#endif
+#if !defined FALSE
+#define FALSE 0
+#endif
+
+#define NEW(x, t, n) if ((x = (t *)malloc(sizeof(t) * (n))) == 0) {return -1;}
+#define FREE(x) if (x != 0) { free(x); x = 0; }
+#define SWAP_INDICES(a, b) { int_t _temp_index = a; a = b; b = _temp_index; }
+#include <opencv2/opencv.hpp>
+
+namespace PaddleDetection {
+
+typedef signed int int_t;
+typedef unsigned int uint_t;
+typedef double cost_t;
+typedef char boolean;
+typedef enum fp_t { FP_1 = 1, FP_2 = 2, FP_DYNAMIC = 3 } fp_t;
+
+int lapjv_internal(
+    const cv::Mat &cost, const bool extend_cost, const float cost_limit,
+    int *x, int *y);
+
+} // namespace PaddleDetection
+
+#endif // LAPJV_H
+

deploy/cpp/include/preprocess_op.h

@@ -102,6 +102,20 @@ class Resize : public PreprocessOp {
   std::vector<int> in_net_shape_;
 };
 
+class LetterBoxResize : public PreprocessOp {
+ public:
+  virtual void Init(const YAML::Node& item) {
+    target_size_ = item["target_size"].as<std::vector<int>>();
+ }
+
+  float GenerateScale(const cv::Mat& im);
+
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+
+ private:
+  std::vector<int> target_size_;
+  std::vector<int> in_net_shape_;
+};
 // Models with FPN need input shape % stride == 0
 class PadStride : public PreprocessOp {
  public:
@@ -146,6 +160,8 @@ class Preprocessor {
   std::shared_ptr<PreprocessOp> CreateOp(const std::string& name) {
     if (name == "Resize") {
       return std::make_shared<Resize>();
+    } else if (name == "LetterBoxResize") {
+      return std::make_shared<LetterBoxResize>();
     } else if (name == "Permute") {
       return std::make_shared<Permute>();
     } else if (name == "NormalizeImage") {

deploy/cpp/include/tracker.h

@@ -0,0 +1,58 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <map>
+#include <vector>
+#include <opencv2/opencv.hpp>
+
+#include "trajectory.h"
+
+namespace PaddleDetection {
+
+typedef std::map<int, int> Match;
+typedef std::map<int, int>::iterator MatchIterator;
+
+struct Track
+{
+    int id;
+    float score;
+    cv::Vec4f ltrb;
+};
+
+class JDETracker
+{
+public:
+    static JDETracker *instance(void);
+    virtual bool update(const cv::Mat &dets, const cv::Mat &emb, std::vector<Track> &tracks);
+private:
+    JDETracker(void);
+    virtual ~JDETracker(void) {}
+    cv::Mat motion_distance(const TrajectoryPtrPool &a, const TrajectoryPool &b);
+    void linear_assignment(const cv::Mat &cost, float cost_limit, Match &matches,
+        std::vector<int> &mismatch_row, std::vector<int> &mismatch_col);
+    void remove_duplicate_trajectory(TrajectoryPool &a, TrajectoryPool &b, float iou_thresh=0.15f);
+private:
+    static JDETracker *me;
+    int timestamp;
+    TrajectoryPool tracked_trajectories;
+    TrajectoryPool lost_trajectories;
+    TrajectoryPool removed_trajectories;
+    int max_lost_time;
+    float lambda;
+    float det_thresh;
+};
+
+} // namespace PaddleDetection