diff --git a/.github/workflows/dev-ov_2020-3.yml b/.github/workflows/dev-ov_2020-3.yml new file mode 100644 index 00000000..a8fbd5f6 --- /dev/null +++ b/.github/workflows/dev-ov_2020-3.yml @@ -0,0 +1,72 @@ +name: dev-ov.2020.3-CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the dev-ov2020.3 branch +on: + push: + branches: [ dev-ov2020.3 ] + pull_request: + branches: [ dev-ov2020.3 ] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-18.04 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # install ros melodic + - uses: ros-tooling/setup-ros@0.1.1 + with: + required-ros-distributions: melodic + - run: "source /opt/ros/melodic/setup.bash" + + # install openvino 2020.3 + - name: install openvino 2020.3 + run: | + sudo apt update && sudo apt install curl gnupg2 lsb-release + curl -s https://apt.repos.intel.com/openvino/2020/GPG-PUB-KEY-INTEL-OPENVINO-2020 |sudo apt-key add - + echo "deb https://apt.repos.intel.com/openvino/2020 all main" | sudo tee /etc/apt/sources.list.d/intel-openvino-2020.list + sudo apt update + sudo apt-cache search openvino + sudo apt-get install -y \ + intel-openvino-runtime-ubuntu18-2020.3.341 \ + intel-openvino-ie-samples-2020.3.341 \ + intel-openvino-omz-dev-2020.3.341 \ + intel-openvino-omz-tools-2020.3.341 \ + intel-openvino-gstreamer-rt-ubuntu-bionic-2020.3.341 \ + intel-openvino-gva-dev-ubuntu-bionic-2020.3.341 \ + intel-openvino-gva-rt-ubuntu-bionic-2020.3.341 \ + intel-openvino-ie-bin-python-tools-ubuntu-bionic-2020.3.341 \ + intel-openvino-ie-rt-core-ubuntu-bionic-2020.3.341 \ + intel-openvino-ie-rt-cpu-ubuntu-bionic-2020.3.341 \ + intel-openvino-ie-rt-gna-ubuntu-bionic-2020.3.341 \ + intel-openvino-ie-rt-gpu-ubuntu-bionic-2020.3.341 \ + intel-openvino-ie-rt-hddl-ubuntu-bionic-2020.3.341 \ + intel-openvino-ie-rt-vpu-ubuntu-bionic-2020.3.341 \ + intel-openvino-ie-sdk-ubuntu-bionic-2020.3.341 \ + intel-openvino-opencv-lib-ubuntu-bionic-2020.3.341 + sudo apt-get install -y libgflags-dev + ls -lh /opt/intel/openvino + source /opt/intel/openvino/bin/setupvars.sh + + # build ros openvino toolkit + - name: build ros openvino toolkit + run: | + mkdir -p ~/catkin_ws/src + cp -rf ${GITHUB_WORKSPACE} ~/catkin_ws/src + cd ~/catkin_ws/src + git clone https://github.com/intel/object_msgs.git + cd ~/catkin_ws/ + source /opt/ros/melodic/setup.bash + source /opt/intel/openvino/bin/setupvars.sh + export CPU_EXTENSION_LIB+=/opt/intel/openvino_2020.3.341/deployment_tools/inference_engine/lib/intel64/libinference_engine.so + export GFLAGS_LIB+=/usr/lib/x86_64-linux-gnu/libgflags_nothreads.a + env + catkin_make + diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..4dd351b2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +dynamic_vino_lib/html/* +dynamic_vino_lib/latex/* diff --git a/README.md b/README.md index d77adfcf..e0056607 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Introduction + The OpenVINO™ (Open visual inference and neural network optimization) toolkit provides a ROS-adaptered runtime framework of neural network which quickly deploys applications and solutions for vision inference. By leveraging Intel® OpenVINO™ toolkit and corresponding libraries, this runtime framework extends workloads across Intel® hardware (including accelerators) and maximizes performance. * Enables CNN-based deep learning inference at the edge * Supports heterogeneous execution across computer vision accelerators—CPU, GPU, Intel® Movidius™ Neural Compute Stick, and FPGA—using a common API @@ -8,7 +9,7 @@ The OpenVINO™ (Open visual inference and neural network optimization) toolkit ## Design Architecture From the view of hirarchical architecture design, the package is divided into different functional components, as shown in below picture. -![OpenVINO_Architecture](https://github.com/intel/ros_openvino_toolkit/blob/master/data/images/design_arch.PNG "OpenVINO RunTime Architecture") +![OpenVINO_Architecture](https://github.com/intel/ros_openvino_toolkit/blob/devel/data/images/design_arch.PNG "OpenVINO RunTime Architecture") - **Intel® OpenVINO™ toolkit** is leveraged to provide deep learning basic implementation for data inference. is free software that helps developers and data scientists speed up computer vision workloads, streamline deep learning inference and deployments, and enable easy, heterogeneous execution across Intel® platforms from edge to cloud. It helps to: @@ -23,11 +24,11 @@ and enable easy, heterogeneous execution across Intel® platforms from edge to c ## Logic Flow From the view of logic implementation, the package introduces the definitions of parameter manager, pipeline and pipeline manager. The below picture depicts how these entities co-work together when the corresponding program is launched. -![Logic_Flow](https://github.com/intel/ros_openvino_toolkit/blob/master/data/images/impletation_logic.PNG "OpenVINO RunTime Logic Flow") +![Logic_Flow](https://github.com/intel/ros_openvino_toolkit/blob/devel/data/images/impletation_logic.PNG "OpenVINO RunTime Logic Flow") Once a corresponding program is launched with a specified .yaml config file passed in the .launch file or via commandline, _**parameter manager**_ analyzes the configurations about pipeline and the whole framework, then shares the parsed configuration information with pipeline procedure. A _**pipeline instance**_ is created by following the configuration info and is added into _**pipeline manager**_ for lifecycle control and inference action triggering. -The contents in **.yaml config file** should be well structured and follow the supported rules and entity names. Please see [the configuration guidance](https://github.com/intel/ros_openvino_toolkit/blob/master/doc/YAML_CONFIGURATION_GUIDE.md) for how to create or edit the config files. +The contents in **.yaml config file** should be well structured and follow the supported rules and entity names. Please see [the configuration guidance](https://github.com/intel/ros_openvino_toolkit/blob/devel/doc/YAML_CONFIGURATION_GUIDE.md) for how to create or edit the config files. **Pipeline** fulfills the whole data handling process: initiliazing Input Component for image data gathering and formating; building up the structured inference network and passing the formatted data through the inference network; transfering the inference results and handling output, etc. @@ -79,6 +80,10 @@ Currently, the inference feature list is supported: ```/ros_openvino_toolkit/segmented_obejcts```([people_msgs::ObjectsInMasks](https://github.com/intel/ros_openvino_toolkit/blob/devel/people_msgs/msg/ObjectsInMasks.msg)) - Person Reidentification: ```/ros_openvino_toolkit/reidentified_persons```([people_msgs::ReidentificationStamped](https://github.com/intel/ros_openvino_toolkit/blob/devel/people_msgs/msg/ReidentificationStamped.msg)) +- Vehicle Detection: +```/ros_openvino_toolkit/detected_license_plates```([people_msgs::msg::VehicleAttribsStamped](https://github.com/intel/ros2_openvino_toolkit/blob/devel/people_msgs/msg/VehicleAttribsStamped.msg) +- Vehicle License Detection: +```/ros_openvino_toolkit/detected_license_plates```([people_msgs::msg::LicensePlateStamped](https://github.com/intel/ros2_openvino_toolkit/blob/devel/people_msgs/msg/LicensePlateStamped.msg) - Rviz Output: ```/ros_openvino_toolkit/image_rviz```([sensor_msgs::Image](http://docs.ros.org/melodic/api/sensor_msgs/html/msg/Image.html)) @@ -102,75 +107,45 @@ To show in RViz tool, add an image marker with the composited topic: ### Image Window OpenCV based image window is natively supported by the package. -To enable window, Image Window output should be added into the output choices in .yaml config file. see [the config file guidance](https://github.com/intel/ros_openvino_toolkit/blob/master/doc/YAML_CONFIGURATION_GUIDE.md) for checking/adding this feature in your launching. +To enable window, Image Window output should be added into the output choices in .yaml config file. see [the config file guidance](https://github.com/intel/ros_openvino_toolkit/blob/devel/doc/YAML_CONFIGURATION_GUIDE.md) for checking/adding this feature in your launching. ## Demo Result Snapshots See below pictures for the demo result snapshots. * face detection input from standard camera -![face_detection_demo_image](https://github.com/intel/ros_openvino_toolkit/blob/master/data/images/face_detection.png "face detection demo image") +![face_detection_demo_image](https://github.com/intel/ros_openvino_toolkit/blob/devel/data/images/face_detection.png "face detection demo image") * object detection input from realsense camera -![object_detection_demo_realsense](https://github.com/intel/ros_openvino_toolkit/blob/master/data/images/object_detection.gif "object detection demo realsense") +![object_detection_demo_realsense](https://github.com/intel/ros_openvino_toolkit/blob/devel/data/images/object_detection.gif "object detection demo realsense") * object segmentation input from video -![object_segmentation_demo_video](https://github.com/intel/ros_openvino_toolkit/blob/master/data/images/object_segmentation.gif "object segmentation demo video") +![object_segmentation_demo_video](https://github.com/intel/ros_openvino_toolkit/blob/devel/data/images/object_segmentation.gif "object segmentation demo video") * Person Reidentification input from standard camera -![person_reidentification_demo_video](https://github.com/intel/ros2_openvino_toolkit/blob/master/data/images/person-reidentification.gif "person reidentification demo video") +![person_reidentification_demo_video](https://github.com/intel/ros2_openvino_toolkit/blob/devel/data/images/person-reidentification.gif "person reidentification demo video") # Installation & Launching -**NOTE:** Intel releases 2 different series of OpenVINO Toolkit, we call them as [OpenSource Version](https://github.com/opencv/dldt/) and [Tarball Version](https://software.intel.com/en-us/openvino-toolkit). This guidelie uses OpenSource Version as the installation and launching example. **If you want to use Tarball version, please follow [the guide for Tarball Version](https://github.com/intel/ros_openvino_toolkit/blob/master/doc/BINARY_VERSION_README.md).** - -## Enable Intel® Neural Compute Stick 2 (Intel® NCS 2) under the OpenVINO Open Source version (Optional)
-1. Intel Distribution of OpenVINO toolkit
- * Download OpenVINO toolkit by following the [guide](https://software.intel.com/en-us/openvino-toolkit/choose-download)
- ```bash - cd ~/Downloads - wget -c http://registrationcenter-download.intel.com/akdlm/irc_nas/15078/l_openvino_toolkit_p_2018.5.455.tgz - ``` - * Install OpenVINO toolkit by following the [guide](https://software.intel.com/en-us/articles/OpenVINO-Install-Linux)
- ```bash - cd ~/Downloads - tar -xvf l_openvino_toolkit_p_2018.5.455.tgz - cd l_openvino_toolkit_p_2018.5.455 - # root is required instead of sudo - sudo -E ./install_cv_sdk_dependencies.sh - sudo ./install_GUI.sh - # build sample code under OpenVINO toolkit - source /opt/intel/computer_vision_sdk/bin/setupvars.sh - cd /opt/intel/computer_vision_sdk/deployment_tools/inference_engine/samples/ - mkdir build - cd build - cmake .. - make - ``` - * Configure the Neural Compute Stick USB Driver - ```bash - cd ~/Downloads - cat < 97-usbboot.rules - SUBSYSTEM=="usb", ATTRS{idProduct}=="2150", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" - SUBSYSTEM=="usb", ATTRS{idProduct}=="2485", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" - SUBSYSTEM=="usb", ATTRS{idProduct}=="f63b", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" - EOF - sudo cp 97-usbboot.rules /etc/udev/rules.d/ - sudo udevadm control --reload-rules - sudo udevadm trigger - sudo ldconfig - rm 97-usbboot.rules - ``` - -2. Configure the environment (you can write the configuration to your ~/.basrch file)
- **Note**: If you used root privileges to install the OpenVINO binary package, it installs the Intel Distribution of OpenVINO toolkit in this directory: */opt/intel/openvino_/* - ```bash - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/computer_vision_sdk/deployment_tools/inference_engine/samples/build/intel64/Release/lib - source /opt/intel/computer_vision_sdk/bin/setupvars.sh - ``` +**NOTE:** Intel releases 2 different series of OpenVINO Toolkit, we call them as [OpenSource Version](https://github.com/opencv/dldt/) and [Tarball Version](https://software.intel.com/en-us/openvino-toolkit). This guidelie uses OpenSource Version as the installation and launching example. **If you want to use Tarball version, please follow [the guide for Tarball Version](https://github.com/intel/ros_openvino_toolkit/blob/devel/doc/BINARY_VERSION_README.md).** ## Dependencies Installation -One-step installation scripts are provided for the dependencies' installation. Please see [the guide](https://github.com/intel/ros_openvino_toolkit/blob/master/doc/OPEN_SOURCE_CODE_README.md) for details. +One-step installation scripts are provided for the dependencies' installation. Please see [the guide](https://github.com/intel/ros_openvino_toolkit/blob/devel/doc/OPEN_SOURCE_CODE_README.md) for details. ## Launching * Preparation + * Configure the Neural Compute Stick USB Driver + ```bash + cd ~/Downloads + cat < 97-usbboot.rules + SUBSYSTEM=="usb", ATTRS{idProduct}=="2150", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" + SUBSYSTEM=="usb", ATTRS{idProduct}=="2485", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" + SUBSYSTEM=="usb", ATTRS{idProduct}=="f63b", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" + EOF + sudo cp 97-usbboot.rules /etc/udev/rules.d/ + sudo udevadm control --reload-rules + sudo udevadm trigger + sudo ldconfig + rm 97-usbboot.rules + ``` + * download [Object Detection model](https://github.com/intel/ros_openvino_toolkit/tree/devel/doc/OBJECT_DETECTION.md) * download and convert a trained model to produce an optimized Intermediate Representation (IR) of the model ```bash #object segmentation model @@ -181,33 +156,37 @@ One-step installation scripts are provided for the dependencies' installation. P wget http://download.tensorflow.org/models/object_detection/mask_rcnn_inception_v2_coco_2018_01_28.tar.gz tar -zxvf mask_rcnn_inception_v2_coco_2018_01_28.tar.gz cd mask_rcnn_inception_v2_coco_2018_01_28 - python3 /opt/openvino_toolkit/dldt/model-optimizer/mo_tf.py --input_model frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/openvino_toolkit/dldt/model-optimizer/extensions/front/tf/mask_rcnn_support.json --tensorflow_object_detection_api_pipeline_config pipeline.config --reverse_input_channels --output_dir ./output/ - sudo mkdir -p /opt/models - sudo ln -s ~/Downloads/models/mask_rcnn_inception_v2_coco_2018_01_28 /opt/models/ - #object detection model - cd /opt/openvino_toolkit/open_model_zoo/model_downloader - python3 ./downloader.py --name mobilenet-ssd - #FP32 precision model - sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo.py --input_model /opt/openvino_toolkit/open_model_zoo/model_downloader/object_detection/common/mobilenet-ssd/caffe/mobilenet-ssd.caffemodel --output_dir /opt/openvino_toolkit/open_model_zoo/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP32 --mean_values [127.5,127.5,127.5] --scale_values [127.5] - #FP16 precision model - sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo.py --input_model /opt/openvino_toolkit/open_model_zoo/model_downloader/object_detection/common/mobilenet-ssd/caffe/mobilenet-ssd.caffemodel --output_dir /opt/openvino_toolkit/open_model_zoo/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP16 --data_type=FP16 --mean_values [127.5,127.5,127.5] --scale_values [127.5] + #FP32 + sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo_tf.py --input_model frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/openvino_toolkit/dldt/model-optimizer/extensions/front/tf/mask_rcnn_support.json --tensorflow_object_detection_api_pipeline_config pipeline.config --reverse_input_channels --output_dir /opt/openvino_toolkit/models/segmentation/output/FP32 + #FP16 + sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo_tf.py --input_model frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/openvino_toolkit/dldt/model-optimizer/extensions/front/tf/mask_rcnn_support.json --tensorflow_object_detection_api_pipeline_config pipeline.config --reverse_input_channels --data_type=FP16 --output_dir /opt/openvino_toolkit/models/segmentation/output/FP16 + ``` * download the optimized Intermediate Representation (IR) of model (excute _once_)
```bash - cd /opt/openvino_toolkit/open_model_zoo/model_downloader - python3 downloader.py --name face-detection-adas-0001 - python3 downloader.py --name age-gender-recognition-retail-0013 - python3 downloader.py --name emotions-recognition-retail-0003 - python3 downloader.py --name head-pose-estimation-adas-0001 - python3 downloader.py --name person-detection-retail-0013 - python3 downloader.py --name person-reidentification-retail-0076 + cd /opt/openvino_toolkit/open_model_zoo/tools/downloader + sudo python3 downloader.py --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output + sudo python3 downloader.py --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output + sudo python3 downloader.py --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output + sudo python3 downloader.py --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output + sudo python3 downloader.py --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output + sudo python3 downloader.py --name person-reidentification-retail-0076 --output_dir /opt/openvino_toolkit/models/person-reidentification/output + sudo python3 downloader.py --name vehicle-license-plate-detection-barrier-0106 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection/output + sudo python3 downloader.py --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recongnition/output + sudo python3 downloader.py --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output + sudo python3 downloader.py --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output + sudo python3 downloader.py --name face-reidentification-retail-0095 --output_dir /opt/openvino_toolkit/models/face-reidentification/output ``` * copy label files (excute _once_)
```bash - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_attributes/emotions_recognition/0003/dldt - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/open_model_zoo/model_downloader/Transportation/object_detection/face/pruned_mobilenet_reduced_ssd_shared_weights/dldt - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/models/mask_rcnn_inception_v2_coco_2018_01_28/output - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/open_model_zoo/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP32 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/open_model_zoo/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP16 + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/segmentation/output/FP32/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/segmentation/output/FP16/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + ``` * set ENV LD_LIBRARY_PATH
```bash @@ -215,20 +194,11 @@ One-step installation scripts are provided for the dependencies' installation. P ``` * run face detection sample code input from StandardCamera. ```bash - roslaunch vino_launch pipeline_people_oss.launch + roslaunch vino_launch pipeline_people.launch ``` - * run face detection sample code input from Image. ```bash - roslaunch vino_launch pipeline_image_oss.launch - ``` -* run object detection sample code input from RealsensCamera. - ```bash - roslaunch vino_launch pipeline_object_oss.launch - ``` -* run object detection sample code input from RealsensCameraTopic. - ```bash - roslaunch vino_launch pipeline_object_oss_topic.launch + roslaunch vino_launch pipeline_image.launch ``` * run object segmentation sample code input from RealSenseCameraTopic. ```bash @@ -240,12 +210,20 @@ One-step installation scripts are provided for the dependencies' installation. P ``` * run person reidentification sample code input from StandardCamera. ```bash - roslaunch vino_launch pipeline_reidentification_oss.launch + roslaunch vino_launch pipeline_reidentification.launch + ``` +* run face re-identification with facial landmarks from realsense camera + ```bash + roslaunch vino_launch pipeline_face_reidentification.launch + ``` +* run vehicle detection sample code input from StandardCamera. + ```bash + roslaunch vino_launch pipeline_vehicle_detection.launch ``` * run object detection service sample code input from Image Run image processing service: ```bash - roslaunch vino_launch image_object_server_oss.launch + roslaunch vino_launch image_object_server.launch ``` Run example application with an absolute path of an image on another console: ```bash @@ -254,7 +232,7 @@ One-step installation scripts are provided for the dependencies' installation. P * run face detection service sample code input from Image Run image processing service: ```bash - roslaunch vino_launch image_people_server_oss.launch + roslaunch vino_launch image_people_server.launch ``` Run example application with an absolute path of an image on another console: ```bash @@ -268,4 +246,4 @@ One-step installation scripts are provided for the dependencies' installation. P # More Information * ros OpenVINO discription writen in Chinese: https://mp.weixin.qq.com/s/BgG3RGauv5pmHzV_hkVAdw -###### *Any security issue should be reported using process at https://01.org/security* + diff --git a/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels b/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels new file mode 100644 index 00000000..23d4cd9a --- /dev/null +++ b/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels @@ -0,0 +1,2 @@ +vehicle +license diff --git a/data/labels/object_detection/yolov2-voc.labels b/data/labels/object_detection/yolov2-voc.labels new file mode 100644 index 00000000..6d218467 --- /dev/null +++ b/data/labels/object_detection/yolov2-voc.labels @@ -0,0 +1,20 @@ +Aeroplane +Bicycle +Bird +Boat +Bottle +Bus +Car +Cat +Chair +Cow +DiningTable +Dog +Horse +Motorbike +Person +PottedPlant +Sheep +Sofa +Train +TV/Monitor diff --git a/doc/BINARY_VERSION_README.md b/doc/BINARY_VERSION_README.md index f4d731c8..a30ba6b6 100644 --- a/doc/BINARY_VERSION_README.md +++ b/doc/BINARY_VERSION_README.md @@ -13,11 +13,11 @@ This project is a ROS wrapper for CV API of [OpenVINO™](https://software.intel * Demo application to show above detection and recognitions ## 2. Prerequisite -- An x86_64 computer running Ubuntu 16.04. Below processors are supported: +- An x86_64 computer running Ubuntu 18.04. Below processors are supported: * 6th-8th Generation Intel® Core™ * Intel® Xeon® v5 family * Intel® Xeon® v6 family -- ROS Kinetic +- ROS [Dashing](https://github.com/ros2/ros2) - [OpenVINO™ Toolkit](https://software.intel.com/en-us/openvino-toolkit) - RGB Camera, e.g. RealSense D400 Series or standard USB camera or Video/Image File @@ -38,9 +38,9 @@ This project is a ROS wrapper for CV API of [OpenVINO™](https://software.intel ``` **Note**:You can also choose to follow the steps below to build the environment step by step. -- Install ROS Kinetic Desktop-Full ([guide](http://wiki.ros.org/kinetic/Installation/Ubuntu)) +- Install ROS Dashing Desktop-Full ([guide](https://index.ros.org/doc/ros2/Installation/Dashing/)) -- Install [OpenVINO™ Toolkit](https://software.intel.com/en-us/openvino-toolkit) ([guide](https://software.intel.com/en-us/articles/OpenVINO-Install-Linux)). Choose "2018 R4" when download tarball. +- Install [OpenVINO™ Toolkit](https://software.intel.com/en-us/openvino-toolkit) ([guide](https://software.intel.com/en-us/articles/OpenVINO-Install-Linux)). **Note**: Please use *root privileges* to run the installer when installing the core components. - Install OpenCL Driver for GPU @@ -70,9 +70,8 @@ sudo ./install_NEO_OCL_driver.sh sudo apt update sudo apt install libjasper1 libjasper-dev ``` -- Install Intel® RealSense™ SDK 2.0 [(tag v2.17.1)](https://github.com/IntelRealSense/librealsense/tree/v2.17.1) - * [Install from source code](https://github.com/IntelRealSense/librealsense/blob/v2.17.1/doc/installation.md)(Recommended) - * [Install from package](https://github.com/IntelRealSense/librealsense/blob/v2.17.1/doc/distribution_linux.md) +- Install Intel® RealSense™ SDK 2.0 [(tag v2.30.0)](https://github.com/IntelRealSense/librealsense/tree/v2.30.0) + * [Install from package](https://github.com/IntelRealSense/librealsense/blob/v2.30.0/doc/distribution_linux.md) - Other Dependencies ```bash @@ -89,8 +88,8 @@ sudo ln -sf libboost_python-py35.so libboost_python3.so - Build sample code under openvino toolkit ```bash # root is required instead of sudo -source /opt/intel/computer_vision_sdk/bin/setupvars.sh -cd /opt/intel/computer_vision_sdk/deployment_tools/inference_engine/samples/ +source /opt/intel/openvino/bin/setupvars.sh +cd /opt/intel/openvino/deployment_tools/inference_engine/samples/ mkdir build cd build cmake .. @@ -99,8 +98,8 @@ make - Set Environment CPU_EXTENSION_LIB and GFLAGS_LIB ```bash -export CPU_EXTENSION_LIB=/opt/intel/computer_vision_sdk/deployment_tools/inference_engine/samples/build/intel64/Release/lib/libcpu_extension.so -export GFLAGS_LIB=/opt/intel/computer_vision_sdk/deployment_tools/inference_engine/samples/build/intel64/Release/lib/libgflags_nothreads.a +export CPU_EXTENSION_LIB=/opt/intel/openvino/deployment_tools/inference_engine/samples/build/intel64/Release/lib/libcpu_extension.so +export GFLAGS_LIB=/opt/intel/openvino/deployment_tools/inference_engine/samples/build/intel64/Release/lib/libgflags_nothreads.a ``` - Install ROS_OpenVINO packages @@ -122,7 +121,7 @@ source /opt/ros/kinetic/setup.bash # Ubuntu 18.04 source /opt/ros/melodic/setup.bash -source /opt/intel/computer_vision_sdk/bin/setupvars.sh +source /opt/intel/openvino/bin/setupvars.sh export OpenCV_DIR=$HOME/code/opencv/build cd ~/catkin_ws catkin_make @@ -133,59 +132,76 @@ sudo ln -s ~/catkin_ws/src/ros_openvino_toolkit /opt/openvino_toolkit/ros_openvi ## 5. Running the Demo * Preparation + * Configure the Neural Compute Stick USB Driver + ```bash + cd ~/Downloads + cat < 97-usbboot.rules + SUBSYSTEM=="usb", ATTRS{idProduct}=="2150", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" + SUBSYSTEM=="usb", ATTRS{idProduct}=="2485", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" + SUBSYSTEM=="usb", ATTRS{idProduct}=="f63b", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" + EOF + sudo cp 97-usbboot.rules /etc/udev/rules.d/ + sudo udevadm control --reload-rules + sudo udevadm trigger + sudo ldconfig + rm 97-usbboot.rules + ``` + * download [Object Detection model](https://github.com/intel/ros_openvino_toolkit/tree/devel/doc/OBJECT_DETECTION.md) * download and convert a trained model to produce an optimized Intermediate Representation (IR) of the model - ```bash - #object segmentation model - cd /opt/intel/computer_vision_sdk/deployment_tools/model_optimizer/install_prerequisites - sudo ./install_prerequisites.sh - mkdir -p ~/Downloads/models - cd ~/Downloads/models - wget http://download.tensorflow.org/models/object_detection/mask_rcnn_inception_v2_coco_2018_01_28.tar.gz - tar -zxvf mask_rcnn_inception_v2_coco_2018_01_28.tar.gz - cd mask_rcnn_inception_v2_coco_2018_01_28 - python3 /opt/intel/computer_vision_sdk/deployment_tools/model_optimizer/mo_tf.py --input_model frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/intel/computer_vision_sdk/deployment_tools/model_optimizer/extensions/front/tf/mask_rcnn_support.json --tensorflow_object_detection_api_pipeline_config pipeline.config --reverse_input_channels --output_dir ./output/ - sudo mkdir -p /opt/models - sudo ln -sf ~/Downloads/models/mask_rcnn_inception_v2_coco_2018_01_28 /opt/models/ - #object detection model - cd /opt/intel/computer_vision_sdk/deployment_tools/model_downloader - sudo python3 ./downloader.py --name mobilenet-ssd - #FP32 precision model - sudo python3 /opt/intel/computer_vision_sdk/deployment_tools/model_optimizer/mo.py --input_model /opt/intel/computer_vision_sdk/deployment_tools/model_downloader/object_detection/common/mobilenet-ssd/caffe/mobilenet-ssd.caffemodel --output_dir /opt/intel/computer_vision_sdk/deployment_tools/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP32 --mean_values [127.5,127.5,127.5] --scale_values [127.5] - #FP16 precision model - sudo python3 /opt/intel/computer_vision_sdk/deployment_tools/model_optimizer/mo.py --input_model /opt/intel/computer_vision_sdk/deployment_tools/model_downloader/object_detection/common/mobilenet-ssd/caffe/mobilenet-ssd.caffemodel --output_dir /opt/intel/computer_vision_sdk/deployment_tools/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP16 --data_type=FP16 --mean_values [127.5,127.5,127.5] --scale_values [127.5] - ``` + ```bash + #object segmentation model + cd /opt/intel/openvino/deployment_tools/model_optimizer/install_prerequisites + sudo ./install_prerequisites.sh + mkdir -p ~/Downloads/models + cd ~/Downloads/models + wget http://download.tensorflow.org/models/object_detection/mask_rcnn_inception_v2_coco_2018_01_28.tar.gz + tar -zxvf mask_rcnn_inception_v2_coco_2018_01_28.tar.gz + cd mask_rcnn_inception_v2_coco_2018_01_28 + #FP32 + sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo_tf.py --input_model frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/intel/openvino/deployment_tools/model_optimizer/extensions/front/tf/mask_rcnn_support.json --tensorflow_object_detection_api_pipeline_config pipeline.config --reverse_input_channels --output_dir /opt/openvino_toolkit/models/segmentation/output/FP32 + #FP16 + sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo_tf.py --input_model frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/intel/openvino/deployment_tools/model_optimizer/extensions/front/tf/mask_rcnn_support.json --tensorflow_object_detection_api_pipeline_config pipeline.config --reverse_input_channels --data_type=FP16 --output_dir /opt/openvino_toolkit/models/segmentation/output/FP16 + ``` + * download the optimized Intermediate Representation (IR) of model (excute once) + ```bash + cd /opt/intel/openvino/deployment_tools/tools/model_downloader + sudo python3 downloader.py --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output + sudo python3 downloader.py --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output + sudo python3 downloader.py --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output + sudo python3 downloader.py --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output + sudo python3 downloader.py --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output + sudo python3 downloader.py --name person-reidentification-retail-0076 --output_dir /opt/openvino_toolkit/models/person-reidentification/output + sudo python3 downloader.py --name vehicle-license-plate-detection-barrier-0106 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection/output + sudo python3 downloader.py --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recongnition/output + sudo python3 downloader.py --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output + sudo python3 downloader.py --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output + sudo python3 downloader.py --name face-reidentification-retail-0095 --output_dir /opt/openvino_toolkit/models/face-reidentification/output + ``` * copy label files (excute _once_)
- ```bash - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/intel/computer_vision_sdk/deployment_tools/intel_models/emotions-recognition-retail-0003/FP32 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/intel/computer_vision_sdk/deployment_tools/intel_models/emotions-recognition-retail-0003/FP16 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-detection-adas-0001/FP32 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-detection-adas-0001/FP16 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels ~/Downloads/models/mask_rcnn_inception_v2_coco_2018_01_28/output - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/intel/computer_vision_sdk/deployment_tools/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP32 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/intel/computer_vision_sdk/deployment_tools/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP16 - ``` - * set ENV LD_LIBRARY_PATH and environment - ```bash - source /opt/intel/computer_vision_sdk/bin/setupvars.sh - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/computer_vision_sdk/deployment_tools/inference_engine/samples/build/intel64/Release/lib - ``` -* run face detection sample code input from StandardCamera.(connect Intel® Neural Compute Stick 2) ```bash - roslaunch vino_launch pipeline_people_myriad.launch + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/segmentation/output/FP32/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/segmentation/output/FP16/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ ``` -* run face detection sample code input from Image. + * set ENV LD_LIBRARY_PATH and environment ```bash - roslaunch vino_launch pipeline_image.launch + source /opt/intel/openvino/bin/setupvars.sh + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/openvino/deployment_tools/inference_engine/samples/build/intel64/Release/lib ``` -* run object detection sample code input from RealSenseCamera.(connect Intel® Neural Compute Stick 2) +* run face detection sample code input from StandardCamera.(connect Intel® Neural Compute Stick 2) ```bash - roslaunch vino_launch pipeline_object.launch + roslaunch vino_launch pipeline_people.launch ``` -* run object detection sample code input from RealSenseCameraTopic.(connect Intel® Neural Compute Stick 2) +* run face detection sample code input from Image. ```bash - roslaunch vino_launch pipeline_object_topic.launch + roslaunch vino_launch pipeline_image.launch ``` -* run object segmentation sample code input from RealSenseCameraTopic.(connect Intel® Neural Compute Stick 2) +* run object segmentation sample code input from RealSenseCameraTopic. ```bash roslaunch vino_launch pipeline_segmentation.launch ``` @@ -197,6 +213,14 @@ sudo ln -s ~/catkin_ws/src/ros_openvino_toolkit /opt/openvino_toolkit/ros_openvi ```bash roslaunch vino_launch pipeline_reidentification.launch ``` +* run face re-identification with facial landmarks from realsense camera + ```bash + roslaunch vino_launch pipeline_face_reidentification.launch + ``` +* run vehicle detection sample code input from StandardCamera. + ```bash + roslaunch vino_launch pipeline_vehicle_detection.launch + ``` * run object detection service sample code input from Image Run image processing service: ```bash @@ -228,3 +252,4 @@ sudo ln -s ~/catkin_ws/src/ros_openvino_toolkit /opt/openvino_toolkit/ros_openvi ###### *Any security issue should be reported using process at https://01.org/security* + diff --git a/doc/OBJECT_DETECTION.md b/doc/OBJECT_DETECTION.md new file mode 100644 index 00000000..5e23b1ae --- /dev/null +++ b/doc/OBJECT_DETECTION.md @@ -0,0 +1,190 @@ +# Object Detection +## Launching +### OpenSource Version +#### mobilenet-ssd +* download and convert a trained model to produce an optimized Intermediate Representation (IR) of the model + ```bash + cd /opt/openvino_toolkit/open_model_zoo/tools/downloader + python3 ./downloader.py --name mobilenet-ssd + #FP32 precision model + sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo.py --input_model /opt/openvino_toolkit/open_model_zoo/tools/downloader/public/mobilenet-ssd/mobilenet-ssd.caffemodel --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP32 --mean_values [127.5,127.5,127.5] --scale_values [127.5] + #FP16 precision model + sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo.py --input_model /opt/openvino_toolkit/open_model_zoo/tools/downloader/public/mobilenet-ssd/mobilenet-ssd.caffemodel --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP16 --data_type=FP16 --mean_values [127.5,127.5,127.5] --scale_values [127.5] + ``` +* copy label files (excute _once_)
+ ```bash + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP32 + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP16 + ``` +* run object detection sample code input from RealSenseCamera.(connect Intel® Neural Compute Stick 2) + ```bash + roslaunch vino_launch pipeline_object.launch + ``` +* run object detection sample code input from RealSenseCameraTopic.(connect Intel® Neural Compute Stick 2) + ```bash + roslaunch vino_launch pipeline_object_topic.launch + ``` +#### YOLOv2-voc +* Darkflow to protobuf(.pb) + - install [darkflow](https://github.com/thtrieu/darkflow) + - install prerequsites + ```bash + pip3 install tensorflow opencv-python numpy networkx cython + ``` + - Get darkflow and YOLO-OpenVINO + ```bash + mkdir -p ~/code && cd ~/code + git clone https://github.com/thtrieu/darkflow + git clone https://github.com/chaoli2/YOLO-OpenVINO + sudo ln -sf ~/code/darkflow /opt/openvino_toolkit/ + ``` + - modify the line self.offset = 16 in the ./darkflow/utils/loader.py file and replace with self.offset = 20 + - Install darkflow + ```bash + cd ~/code/darkflow + pip3 install . + ``` + - Copy voc.names in YOLO-OpenVINO/common to labels.txt in darkflow. + ```bash + cp ~/code/YOLO-OpenVINO/common/voc.names ~/code/darkflow/labels.txt + ``` + - Get yolov2 weights and cfg + ```bash + cd ~/code/darkflow + mkdir -p models + cd models + wget -c https://pjreddie.com/media/files/yolov2-voc.weights + wget https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2-voc.cfg + ``` + - Run convert script + ```bash + cd ~/code/darkflow + flow --model models/yolov2-voc.cfg --load models/yolov2-voc.weights --savepb + ``` +* Convert YOLOv2-voc TensorFlow Model to the optimized Intermediate Representation (IR) of model + ```bash + cd ~/code/darkflow + # FP32 precision model + sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo_tf.py \ + --input_model built_graph/yolov2-voc.pb \ + --batch 1 \ + --tensorflow_use_custom_operations_config /opt/openvino_toolkit/dldt/model-optimizer/extensions/front/tf/yolo_v2_voc.json \ + --data_type FP32 \ + --output_dir /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP32 + # FP16 precision model + sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo_tf.py \ + --input_model built_graph/yolov2-voc.pb \ + --batch 1 \ + --tensorflow_use_custom_operations_config /opt/openvino_toolkit/dldt/model-optimizer/extensions/front/tf/yolo_v2_voc.json \ + --data_type FP16 \ + --output_dir /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP16 + ``` +* copy label files (excute _once_)
+ ```bash + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/yolov2-voc.labels /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP32 + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/yolov2-voc.labels /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP16 + ``` +* run object detection sample code input from RealSenseCamera.(connect Intel® Neural Compute Stick 2) + ```bash + roslaunch vino_launch pipeline_object_yolo.launch + ``` +* run object detection sample code input from RealSenseCameraTopic.(connect Intel® Neural Compute Stick 2) + ```bash + roslaunch vino_launch pipeline_object_yolo_topic.launch + ``` +### Binary Version +#### mobilenet-ssd +* download and convert a trained model to produce an optimized Intermediate Representation (IR) of the model + ```bash + cd /opt/intel/openvino/deployment_tools/tools/model_downloader + sudo python3 ./downloader.py --name mobilenet-ssd + #FP32 precision model + sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo.py --input_model /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/public/mobilenet-ssd/mobilenet-ssd.caffemodel --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP32 --mean_values [127.5,127.5,127.5] --scale_values [127.5] + #FP16 precision model + sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo.py --input_model /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/public/mobilenet-ssd/mobilenet-ssd.caffemodel --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP16 --data_type=FP16 --mean_values [127.5,127.5,127.5] --scale_values [127.5] + ``` +* copy label files (excute _once_)
+ ```bash + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP32 + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP16 + ``` +* run object detection sample code input from RealSenseCamera.(connect Intel® Neural Compute Stick 2) + ```bash + roslaunch vino_launch pipeline_object.launch + ``` +* run object detection sample code input from RealSenseCameraTopic.(connect Intel® Neural Compute Stick 2) + ```bash + roslaunch vino_launch pipeline_object_topic.launch + ``` + #### YOLOv2-voc +* Darkflow to protobuf(.pb) + - install [darkflow](https://github.com/thtrieu/darkflow) + - install prerequsites + ```bash + pip3 install tensorflow opencv-python numpy networkx cython + ``` + - Get darkflow and YOLO-OpenVINO + ```bash + mkdir -p ~/code && cd ~/code + git clone https://github.com/thtrieu/darkflow + git clone https://github.com/chaoli2/YOLO-OpenVINO + sudo ln -sf ~/code/darkflow /opt/openvino_toolkit/ + ``` + - modify the line self.offset = 16 in the ./darkflow/utils/loader.py file and replace with self.offset = 20 + - Install darkflow + ```bash + cd ~/code/darkflow + pip3 install . + ``` + - Copy voc.names in YOLO-OpenVINO/common to labels.txt in darkflow. + ```bash + cp ~/code/YOLO-OpenVINO/common/voc.names ~/code/darkflow/labels.txt + ``` + - Get yolov2 weights and cfg + ```bash + cd ~/code/darkflow + mkdir -p models + cd models + wget -c https://pjreddie.com/media/files/yolov2-voc.weights + wget https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2-voc.cfg + ``` + - Run convert script + ```bash + cd ~/code/darkflow + flow --model models/yolov2-voc.cfg --load models/yolov2-voc.weights --savepb + ``` +* Convert YOLOv2-voc TensorFlow Model to the optimized Intermediate Representation (IR) of model + ```bash + cd ~/code/darkflow + # FP32 precision model + sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo_tf.py \ + --input_model built_graph/yolov2-voc.pb \ + --batch 1 \ + --tensorflow_use_custom_operations_config /opt/intel/openvino/deployment_tools/model_optimizer/extensions/front/tf/yolo_v2_voc.json \ + --data_type FP32 \ + --output_dir /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP32 + # FP16 precision model + sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo_tf.py \ + --input_model built_graph/yolov2-voc.pb \ + --batch 1 \ + --tensorflow_use_custom_operations_config /opt/intel/openvino/deployment_tools/model_optimizer/extensions/front/tf/yolo_v2_voc.json \ + --data_type FP16 \ + --output_dir /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP16 + ``` +* copy label files (excute _once_)
+ ```bash + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/yolov2-voc.labels /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP32 + sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/yolov2-voc.labels /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP16 + ``` +* run object detection sample code input from RealSenseCamera.(connect Intel® Neural Compute Stick 2) + ```bash + roslaunch vino_launch pipeline_object_yolo.launch + ``` +* run object detection sample code input from RealSenseCameraTopic.(connect Intel® Neural Compute Stick 2) + ```bash + roslaunch vino_launch pipeline_object_yolo_topic.launch + ``` + + + + diff --git a/doc/OPEN_SOURCE_CODE_README.md b/doc/OPEN_SOURCE_CODE_README.md index c03a66a8..48e811ce 100644 --- a/doc/OPEN_SOURCE_CODE_README.md +++ b/doc/OPEN_SOURCE_CODE_README.md @@ -79,7 +79,7 @@ This project is a ROS wrapper for CV API of [OpenVINO™](https://software.intel mkdir ~/code && cd ~/code git clone https://github.com/opencv/dldt.git cd dldt/inference-engine/ - git checkout 2018_R4 + git checkout 2019_R3 ./install_dependencies.sh mkdir build && cd build cmake -DCMAKE_BUILD_TYPE=Release .. @@ -92,7 +92,7 @@ This project is a ROS wrapper for CV API of [OpenVINO™](https://software.intel cd ~/code git clone https://github.com/opencv/open_model_zoo.git cd open_model_zoo/demos/ - git checkout 2018_R4 + git checkout 2019_R3 mkdir build && cd build cmake -DCMAKE_BUILD_TYPE=Release /opt/openvino_toolkit/dldt/inference-engine make -j8 @@ -100,9 +100,8 @@ This project is a ROS wrapper for CV API of [OpenVINO™](https://software.intel sudo ln -s ~/code/open_model_zoo /opt/openvino_toolkit/open_model_zoo ``` -- Install Intel® RealSense™ SDK 2.0 [(tag v2.17.1)](https://github.com/IntelRealSense/librealsense/tree/v2.17.1)
- * [Install from source code](https://github.com/IntelRealSense/librealsense/blob/v2.17.1/doc/installation.md)(Recommended)
- * [Install from package](https://github.com/IntelRealSense/librealsense/blob/v2.17.1/doc/distribution_linux.md)
+- Install Intel® RealSense™ SDK 2.0 [(tag v2.30.0)](https://github.com/IntelRealSense/librealsense/tree/v2.30.0) + * [Install from package](https://github.com/IntelRealSense/librealsense/blob/v2.30.0/doc/distribution_linux.md) - Other Dependencies ```bash @@ -150,3 +149,4 @@ This project is a ROS wrapper for CV API of [OpenVINO™](https://software.intel ``` ###### *Any security issue should be reported using process at https://01.org/security* + diff --git a/dynamic_vino_lib/CMakeLists.txt b/dynamic_vino_lib/CMakeLists.txt index 86c0679f..7ea14b70 100644 --- a/dynamic_vino_lib/CMakeLists.txt +++ b/dynamic_vino_lib/CMakeLists.txt @@ -12,12 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required (VERSION 2.8.3) +cmake_minimum_required (VERSION 3.0.2) project(dynamic_vino_lib) +#################################### +## to use new InferenceEngine API (InferenceEngine::Core), +## then, uncomment below line +#add_definitions(-DUSE_IE_CORE) +#################################### + message(STATUS "Looking for inference engine configuration file at: ${CMAKE_PREFIX_PATH}") -find_package(InferenceEngine 1.1) +find_package(InferenceEngine REQUIRED) if (NOT InferenceEngine_FOUND) message(FATAL_ERROR "") endif() @@ -66,13 +72,13 @@ include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/../vino_param_lib/include ) -if (NOT DEFINED ENV{CPU_EXTENSION_LIB}) - message(FATAL_ERROR "Please set ENV CPU_EXTENSION_LIB with 'export CPU_EXTENSION_LIB='") -endif() -set (CpuExtension_lib $ENV{CPU_EXTENSION_LIB}) -add_library(cpu_extension SHARED IMPORTED) -set_target_properties(cpu_extension PROPERTIES - IMPORTED_LOCATION $ENV{CPU_EXTENSION_LIB}) +#if (NOT DEFINED ENV{CPU_EXTENSION_LIB}) +# message(FATAL_ERROR "Please set ENV CPU_EXTENSION_LIB with 'export CPU_EXTENSION_LIB='") +#endif() +#set (CpuExtension_lib $ENV{CPU_EXTENSION_LIB}) +#add_library(cpu_extension SHARED IMPORTED) +#set_target_properties(cpu_extension PROPERTIES +# IMPORTED_LOCATION $ENV{CPU_EXTENSION_LIB}) # Flags if(UNIX OR APPLE) @@ -107,7 +113,7 @@ if(UNIX OR APPLE) # Generic flags. set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fno-operator-names -Wformat -Wformat-security -Wall") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") # Add OpenMP support set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp") @@ -130,15 +136,18 @@ if (SUPPORT_SSE41 EQUAL 0) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.1") endif() -set(DEPENDENCIES realsense2 ${OpenCV_LIBS} cpu_extension) +set(DEPENDENCIES realsense2 ${OpenCV_LIBS}) add_library(${PROJECT_NAME} SHARED src/services/frame_processing_server.cpp - src/factory.cpp + src/services/pipeline_processing_server.cpp + src/services/test.cpp src/pipeline.cpp src/pipeline_params.cpp src/pipeline_manager.cpp src/engines/engine.cpp + src/engines/engine_manager.cpp + src/inferences/base_filter.cpp src/inferences/base_inference.cpp src/inferences/emotions_detection.cpp src/inferences/age_gender_detection.cpp @@ -147,8 +156,14 @@ add_library(${PROJECT_NAME} SHARED src/inferences/object_detection.cpp src/inferences/object_segmentation.cpp src/inferences/person_reidentification.cpp + src/inferences/face_reidentification.cpp + src/inferences/base_reidentification.cpp + src/inferences/person_attribs_detection.cpp + src/inferences/landmarks_detection.cpp + src/inferences/vehicle_attribs_detection.cpp + src/inferences/license_plate_detection.cpp src/inputs/realsense_camera.cpp - src/inputs/realsense_camera_topic.cpp + src/inputs/image_topic.cpp src/inputs/standard_camera.cpp src/inputs/video_input.cpp src/inputs/image_input.cpp @@ -157,9 +172,15 @@ add_library(${PROJECT_NAME} SHARED src/models/age_gender_detection_model.cpp src/models/face_detection_model.cpp src/models/head_pose_detection_model.cpp - src/models/object_detection_model.cpp + src/models/object_detection_ssd_model.cpp + src/models/object_detection_yolov2voc_model.cpp src/models/object_segmentation_model.cpp src/models/person_reidentification_model.cpp + src/models/face_reidentification_model.cpp + src/models/person_attribs_detection_model.cpp + src/models/landmarks_detection_model.cpp + src/models/vehicle_attribs_detection_model.cpp + src/models/license_plate_detection_model.cpp src/outputs/image_window_output.cpp src/outputs/ros_topic_output.cpp src/outputs/rviz_output.cpp @@ -168,9 +189,9 @@ add_library(${PROJECT_NAME} SHARED ) add_dependencies(${PROJECT_NAME} - ${${PROJECT_NAME}_EXPORTED_TARGETS} - ${catkin_EXPORTED_TARGETS} - "object_msgs" + ${${PROJECT_NAME}_EXPORTED_TARGETS} + ${catkin_EXPORTED_TARGETS} + "object_msgs" ) target_link_libraries(${PROJECT_NAME} diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/args_helper.h b/dynamic_vino_lib/include/dynamic_vino_lib/args_helper.h index cc163f47..b02bc4da 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/args_helper.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/args_helper.h @@ -37,14 +37,12 @@ /** * @brief This function check input args and find images in given folder */ -void readImagesArguments(std::vector& images, - const std::string& arg) +void readImagesArguments(std::vector& images, const std::string& arg) { struct stat sb; if (stat(arg.c_str(), &sb) != 0) { - std::cout << "[ WARNING ] File " << arg << " cannot be opened!" - << std::endl; + std::cout << "[ WARNING ] File " << arg << " cannot be opened!" << std::endl; return; } if (S_ISDIR(sb.st_mode)) @@ -53,8 +51,7 @@ void readImagesArguments(std::vector& images, dp = opendir(arg.c_str()); if (dp == nullptr) { - std::cout << "[ WARNING ] Directory " << arg << " cannot be opened!" - << std::endl; + std::cout << "[ WARNING ] Directory " << arg << " cannot be opened!" << std::endl; return; } @@ -62,9 +59,9 @@ void readImagesArguments(std::vector& images, while (nullptr != (ep = readdir(dp))) { std::string fileName = ep->d_name; - if (fileName == "." || fileName == "..") continue; - std::cout << "[ INFO ] Add file " << ep->d_name << " from directory " - << arg << "." << std::endl; + if (fileName == "." || fileName == "..") + continue; + std::cout << "[ INFO ] Add file " << ep->d_name << " from directory " << arg << "." << std::endl; images.push_back(arg + "/" + ep->d_name); } } diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/common.h b/dynamic_vino_lib/include/dynamic_vino_lib/common.h index f97eb4a2..9f95db9f 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/common.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/common.h @@ -1,18 +1,18 @@ /* -// Copyright (c) 2018 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -*/ + * Copyright (c) 2018 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * @brief a header file with common samples functionality @@ -57,14 +57,10 @@ * @param s - string to trim * @return trimmed string */ -inline std::string &trim(std::string &s) { - s.erase(s.begin(), - std::find_if(s.begin(), s.end(), - std::not1(std::ptr_fun(std::isspace)))); - s.erase(std::find_if(s.rbegin(), s.rend(), - std::not1(std::ptr_fun(std::isspace))) - .base(), - s.end()); +inline std::string& trim(std::string& s) +{ + s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun(std::isspace)))); + s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun(std::isspace))).base(), s.end()); return s; } @@ -74,8 +70,8 @@ inline std::string &trim(std::string &s) { * @return TargetDevice value that corresponds to input string. * eDefault in case no corresponding value was found */ -static InferenceEngine::TargetDevice getDeviceFromStr( - const std::string &deviceName) { +static InferenceEngine::TargetDevice getDeviceFromStr(const std::string& deviceName) +{ return InferenceEngine::TargetDeviceInfo::fromStr(deviceName); } @@ -86,14 +82,18 @@ static InferenceEngine::TargetDevice getDeviceFromStr( * @param device - device to infer on * @return Plugin pointer */ -static InferenceEngine::InferenceEnginePluginPtr selectPlugin( - const std::vector &pluginDirs, const std::string &plugin, - InferenceEngine::TargetDevice device) { +static InferenceEngine::InferenceEnginePluginPtr selectPlugin(const std::vector& pluginDirs, + const std::string& plugin, + InferenceEngine::TargetDevice device) +{ InferenceEngine::PluginDispatcher dispatcher(pluginDirs); - if (!plugin.empty()) { + if (!plugin.empty()) + { return dispatcher.getPluginByName(plugin); - } else { + } + else + { return dispatcher.getSuitablePlugin(device); } } @@ -105,9 +105,10 @@ static InferenceEngine::InferenceEnginePluginPtr selectPlugin( * @param device - string representation of device to infer on * @return Plugin pointer */ -static UNUSED InferenceEngine::InferenceEnginePluginPtr selectPlugin( - const std::vector &pluginDirs, const std::string &plugin, - const std::string &device) { +static UNUSED InferenceEngine::InferenceEnginePluginPtr selectPlugin(const std::vector& pluginDirs, + const std::string& plugin, + const std::string& device) +{ return selectPlugin(pluginDirs, plugin, getDeviceFromStr(device)); } @@ -116,9 +117,11 @@ static UNUSED InferenceEngine::InferenceEnginePluginPtr selectPlugin( * @param filepath - full file name * @return filename without extension */ -static UNUSED std::string fileNameNoExt(const std::string &filepath) { +static UNUSED std::string fileNameNoExt(const std::string& filepath) +{ auto pos = filepath.rfind('.'); - if (pos == std::string::npos) return filepath; + if (pos == std::string::npos) + return filepath; return filepath.substr(0, pos); } @@ -127,24 +130,31 @@ static UNUSED std::string fileNameNoExt(const std::string &filepath) { * @param filename - name of the file which extension should be extracted * @return string with extracted file extension */ -inline std::string fileExt(const std::string &filename) { +inline std::string fileExt(const std::string& filename) +{ auto pos = filename.rfind('.'); - if (pos == std::string::npos) return ""; + if (pos == std::string::npos) + return ""; return filename.substr(pos + 1); } -static UNUSED std::ostream &operator<<( - std::ostream &os, const InferenceEngine::Version *version) { +static UNUSED std::ostream& operator<<(std::ostream& os, const InferenceEngine::Version* version) +{ os << "\n\tAPI version ............ "; - if (nullptr == version) { + if (nullptr == version) + { os << "UNKNOWN"; - } else { + } + else + { os << version->apiVersion.major << "." << version->apiVersion.minor; - if (nullptr != version->buildNumber) { + if (nullptr != version->buildNumber) + { os << "\n\t" << "Build .................. " << version->buildNumber; } - if (nullptr != version->description) { + if (nullptr != version->description) + { os << "\n\t" << "Description ....... " << version->description; } @@ -156,71 +166,91 @@ static UNUSED std::ostream &operator<<( * @class PluginVersion * @brief A PluginVersion class stores plugin version and initialization status */ -struct PluginVersion : public InferenceEngine::Version { +struct PluginVersion : public InferenceEngine::Version +{ bool initialized = false; - explicit PluginVersion(const InferenceEngine::Version *ver) { - if (nullptr == ver) { + explicit PluginVersion(const InferenceEngine::Version* ver) + { + if (nullptr == ver) + { return; } InferenceEngine::Version::operator=(*ver); initialized = true; } - operator bool() const noexcept { return initialized; } + operator bool() const noexcept + { + return initialized; + } }; -static UNUSED std::ostream &operator<<(std::ostream &os, - const PluginVersion &version) { +static UNUSED std::ostream& operator<<(std::ostream& os, const PluginVersion& version) +{ os << "\tPlugin version ......... "; - if (!version) { + if (!version) + { os << "UNKNOWN"; - } else { + } + else + { os << version.apiVersion.major << "." << version.apiVersion.minor; } os << "\n\tPlugin name ............ "; - if (!version || version.description == nullptr) { + if (!version || version.description == nullptr) + { os << "UNKNOWN"; - } else { + } + else + { os << version.description; } os << "\n\tPlugin build ........... "; - if (!version || version.buildNumber == nullptr) { + if (!version || version.buildNumber == nullptr) + { os << "UNKNOWN"; - } else { + } + else + { os << version.buildNumber; } return os; } -inline void printPluginVersion(InferenceEngine::InferenceEnginePluginPtr ptr, - std::ostream &stream) { - const PluginVersion *pluginVersion; - ptr->GetVersion((const InferenceEngine::Version *&)pluginVersion); +inline void printPluginVersion(InferenceEngine::InferenceEnginePluginPtr ptr, std::ostream& stream) +{ + const PluginVersion* pluginVersion; + ptr->GetVersion((const InferenceEngine::Version*&)pluginVersion); stream << pluginVersion << std::endl; } -static UNUSED std::vector> blobToImageOutputArray( - InferenceEngine::TBlob::Ptr output, size_t *pWidth, size_t *pHeight, - size_t *pChannels) { +static UNUSED std::vector> blobToImageOutputArray(InferenceEngine::TBlob::Ptr output, + size_t* pWidth, size_t* pHeight, + size_t* pChannels) +{ std::vector> outArray; size_t W = output->dims().at(0); size_t H = output->dims().at(1); size_t C = output->dims().at(2); // Get classes - const float *outData = output->data(); - for (unsigned h = 0; h < H; h++) { + const float* outData = output->data(); + for (unsigned h = 0; h < H; h++) + { std::vector row; - for (unsigned w = 0; w < W; w++) { + for (unsigned w = 0; w < W; w++) + { float max_value = outData[h * W + w]; size_t index = 0; - for (size_t c = 1; c < C; c++) { + for (size_t c = 1; c < C; c++) + { size_t dataIndex = c * H * W + h * W + w; - if (outData[dataIndex] > max_value) { + if (outData[dataIndex] > max_value) + { index = c; max_value = outData[dataIndex]; } @@ -230,9 +260,12 @@ static UNUSED std::vector> blobToImageOutputArray( outArray.push_back(row); } - if (pWidth != nullptr) *pWidth = W; - if (pHeight != nullptr) *pHeight = H; - if (pChannels != nullptr) *pChannels = C; + if (pWidth != nullptr) + *pWidth = W; + if (pHeight != nullptr) + *pHeight = H; + if (pChannels != nullptr) + *pChannels = C; return outArray; } @@ -241,27 +274,38 @@ static UNUSED std::vector> blobToImageOutputArray( * @class Color * @brief A Color class stores channels of a given color */ -class Color { - private: +class Color +{ +private: unsigned char _r; unsigned char _g; unsigned char _b; - public: +public: /** * A default constructor. * @param r - value for red channel * @param g - value for green channel * @param b - value for blue channel */ - Color(unsigned char r, unsigned char g, unsigned char b) - : _r(r), _g(g), _b(b) {} + Color(unsigned char r, unsigned char g, unsigned char b) : _r(r), _g(g), _b(b) + { + } - inline unsigned char red() { return _r; } + inline unsigned char red() + { + return _r; + } - inline unsigned char blue() { return _b; } + inline unsigned char blue() + { + return _b; + } - inline unsigned char green() { return _g; } + inline unsigned char green() + { + return _g; + } }; // TODO : keep only one version of writeOutputBMP @@ -273,19 +317,19 @@ class Color { * @param classesNum - the number of classes * @return false if error else true */ -static UNUSED void writeOutputBmp(std::vector> data, - size_t classesNum, std::ostream &outFile) { +static UNUSED void writeOutputBmp(std::vector> data, size_t classesNum, std::ostream& outFile) +{ unsigned int seed = (unsigned int)time(NULL); // Known colors for training classes from Cityscape dataset - static std::vector colors = { - {128, 64, 128}, {232, 35, 244}, {70, 70, 70}, {156, 102, 102}, - {153, 153, 190}, {153, 153, 153}, {30, 170, 250}, {0, 220, 220}, - {35, 142, 107}, {152, 251, 152}, {180, 130, 70}, {60, 20, 220}, - {0, 0, 255}, {142, 0, 0}, {70, 0, 0}, {100, 60, 0}, - {90, 0, 0}, {230, 0, 0}, {32, 11, 119}, {0, 74, 111}, - {81, 0, 81}}; - - while (classesNum > colors.size()) { + static std::vector colors = { { 128, 64, 128 }, { 232, 35, 244 }, { 70, 70, 70 }, { 156, 102, 102 }, + { 153, 153, 190 }, { 153, 153, 153 }, { 30, 170, 250 }, { 0, 220, 220 }, + { 35, 142, 107 }, { 152, 251, 152 }, { 180, 130, 70 }, { 60, 20, 220 }, + { 0, 0, 255 }, { 142, 0, 0 }, { 70, 0, 0 }, { 100, 60, 0 }, + { 90, 0, 0 }, { 230, 0, 0 }, { 32, 11, 119 }, { 0, 74, 111 }, + { 81, 0, 81 } }; + + while (classesNum > colors.size()) + { static std::mt19937 rng(seed); std::uniform_int_distribution dist(0, 255); Color color(dist(rng), dist(rng), dist(rng)); @@ -293,31 +337,31 @@ static UNUSED void writeOutputBmp(std::vector> data, } unsigned char file[14] = { - 'B', 'M', // magic - 0, 0, 0, 0, // size in bytes - 0, 0, // app data - 0, 0, // app data - 40 + 14, 0, 0, 0 // start of data offset + 'B', 'M', // magic + 0, 0, 0, 0, // size in bytes + 0, 0, // app data + 0, 0, // app data + 40 + 14, 0, 0, 0 // start of data offset }; unsigned char info[40] = { - 40, 0, 0, 0, // info hd size - 0, 0, 0, 0, // width - 0, 0, 0, 0, // height - 1, 0, // number color planes - 24, 0, // bits per pixel - 0, 0, 0, 0, // compression is none - 0, 0, 0, 0, // image bits size - 0x13, 0x0B, 0, 0, // horz resolution in pixel / m - 0x13, 0x0B, 0, 0, // vert resolution (0x03C3 = 96 dpi, 0x0B13 = 72 dpi) - 0, 0, 0, 0, // #colors in palette - 0, 0, 0, 0, // #important colors + 40, 0, 0, 0, // info hd size + 0, 0, 0, 0, // width + 0, 0, 0, 0, // height + 1, 0, // number color planes + 24, 0, // bits per pixel + 0, 0, 0, 0, // compression is none + 0, 0, 0, 0, // image bits size + 0x13, 0x0B, 0, 0, // horz resolution in pixel / m + 0x13, 0x0B, 0, 0, // vert resolution (0x03C3 = 96 dpi, 0x0B13 = 72 dpi) + 0, 0, 0, 0, // #colors in palette + 0, 0, 0, 0, // #important colors }; auto height = data.size(); auto width = data.at(0).size(); - if (height > (size_t)std::numeric_limits::max || - width > (size_t)std::numeric_limits::max) { + if (height > (size_t)std::numeric_limits::max || width > (size_t)std::numeric_limits::max) + { THROW_IE_EXCEPTION << "File size is too big: " << height << " X " << width; } @@ -346,21 +390,23 @@ static UNUSED void writeOutputBmp(std::vector> data, info[22] = (unsigned char)(sizeData >> 16); info[23] = (unsigned char)(sizeData >> 24); - outFile.write(reinterpret_cast(file), sizeof(file)); - outFile.write(reinterpret_cast(info), sizeof(info)); + outFile.write(reinterpret_cast(file), sizeof(file)); + outFile.write(reinterpret_cast(info), sizeof(info)); - unsigned char pad[3] = {0, 0, 0}; + unsigned char pad[3] = { 0, 0, 0 }; - for (size_t y = 0; y < height; y++) { - for (size_t x = 0; x < width; x++) { + for (size_t y = 0; y < height; y++) + { + for (size_t x = 0; x < width; x++) + { unsigned char pixel[3]; size_t index = data.at(y).at(x); pixel[0] = colors.at(index).red(); pixel[1] = colors.at(index).green(); pixel[2] = colors.at(index).blue(); - outFile.write(reinterpret_cast(pixel), 3); + outFile.write(reinterpret_cast(pixel), 3); } - outFile.write(reinterpret_cast(pad), padSize); + outFile.write(reinterpret_cast(pad), padSize); } } @@ -372,37 +418,38 @@ static UNUSED void writeOutputBmp(std::vector> data, * @param width - width of the target image * @return false if error else true */ -static UNUSED bool writeOutputBmp(std::string name, unsigned char *data, - size_t height, size_t width) { +static UNUSED bool writeOutputBmp(std::string name, unsigned char* data, size_t height, size_t width) +{ std::ofstream outFile; outFile.open(name, std::ofstream::binary); - if (!outFile.is_open()) { + if (!outFile.is_open()) + { return false; } unsigned char file[14] = { - 'B', 'M', // magic - 0, 0, 0, 0, // size in bytes - 0, 0, // app data - 0, 0, // app data - 40 + 14, 0, 0, 0 // start of data offset + 'B', 'M', // magic + 0, 0, 0, 0, // size in bytes + 0, 0, // app data + 0, 0, // app data + 40 + 14, 0, 0, 0 // start of data offset }; unsigned char info[40] = { - 40, 0, 0, 0, // info hd size - 0, 0, 0, 0, // width - 0, 0, 0, 0, // height - 1, 0, // number color planes - 24, 0, // bits per pixel - 0, 0, 0, 0, // compression is none - 0, 0, 0, 0, // image bits size - 0x13, 0x0B, 0, 0, // horz resolution in pixel / m - 0x13, 0x0B, 0, 0, // vert resolution (0x03C3 = 96 dpi, 0x0B13 = 72 dpi) - 0, 0, 0, 0, // #colors in palette - 0, 0, 0, 0, // #important colors + 40, 0, 0, 0, // info hd size + 0, 0, 0, 0, // width + 0, 0, 0, 0, // height + 1, 0, // number color planes + 24, 0, // bits per pixel + 0, 0, 0, 0, // compression is none + 0, 0, 0, 0, // image bits size + 0x13, 0x0B, 0, 0, // horz resolution in pixel / m + 0x13, 0x0B, 0, 0, // vert resolution (0x03C3 = 96 dpi, 0x0B13 = 72 dpi) + 0, 0, 0, 0, // #colors in palette + 0, 0, 0, 0, // #important colors }; - if (height > (size_t)std::numeric_limits::max || - width > (size_t)std::numeric_limits::max) { + if (height > (size_t)std::numeric_limits::max || width > (size_t)std::numeric_limits::max) + { THROW_IE_EXCEPTION << "File size is too big: " << height << " X " << width; } @@ -431,21 +478,23 @@ static UNUSED bool writeOutputBmp(std::string name, unsigned char *data, info[22] = (unsigned char)(sizeData >> 16); info[23] = (unsigned char)(sizeData >> 24); - outFile.write(reinterpret_cast(file), sizeof(file)); - outFile.write(reinterpret_cast(info), sizeof(info)); + outFile.write(reinterpret_cast(file), sizeof(file)); + outFile.write(reinterpret_cast(info), sizeof(info)); - unsigned char pad[3] = {0, 0, 0}; + unsigned char pad[3] = { 0, 0, 0 }; - for (size_t y = 0; y < height; y++) { - for (size_t x = 0; x < width; x++) { + for (size_t y = 0; y < height; y++) + { + for (size_t x = 0; x < width; x++) + { unsigned char pixel[3]; pixel[0] = data[y * width * 3 + x * 3]; pixel[1] = data[y * width * 3 + x * 3 + 1]; pixel[2] = data[y * width * 3 + x * 3 + 2]; - outFile.write(reinterpret_cast(pixel), 3); + outFile.write(reinterpret_cast(pixel), 3); } - outFile.write(reinterpret_cast(pad), padSize); + outFile.write(reinterpret_cast(pad), padSize); } return true; } @@ -458,31 +507,31 @@ static UNUSED bool writeOutputBmp(std::string name, unsigned char *data, * \return false if error else true */ -static UNUSED bool writeOutputBmp(unsigned char *data, size_t height, - size_t width, std::ostream &outFile) { +static UNUSED bool writeOutputBmp(unsigned char* data, size_t height, size_t width, std::ostream& outFile) +{ unsigned char file[14] = { - 'B', 'M', // magic - 0, 0, 0, 0, // size in bytes - 0, 0, // app data - 0, 0, // app data - 40 + 14, 0, 0, 0 // start of data offset + 'B', 'M', // magic + 0, 0, 0, 0, // size in bytes + 0, 0, // app data + 0, 0, // app data + 40 + 14, 0, 0, 0 // start of data offset }; unsigned char info[40] = { - 40, 0, 0, 0, // info hd size - 0, 0, 0, 0, // width - 0, 0, 0, 0, // height - 1, 0, // number color planes - 24, 0, // bits per pixel - 0, 0, 0, 0, // compression is none - 0, 0, 0, 0, // image bits size - 0x13, 0x0B, 0, 0, // horz resolution in pixel / m - 0x13, 0x0B, 0, 0, // vert resolution (0x03C3 = 96 dpi, 0x0B13 = 72 dpi) - 0, 0, 0, 0, // #colors in palette - 0, 0, 0, 0, // #important colors + 40, 0, 0, 0, // info hd size + 0, 0, 0, 0, // width + 0, 0, 0, 0, // height + 1, 0, // number color planes + 24, 0, // bits per pixel + 0, 0, 0, 0, // compression is none + 0, 0, 0, 0, // image bits size + 0x13, 0x0B, 0, 0, // horz resolution in pixel / m + 0x13, 0x0B, 0, 0, // vert resolution (0x03C3 = 96 dpi, 0x0B13 = 72 dpi) + 0, 0, 0, 0, // #colors in palette + 0, 0, 0, 0, // #important colors }; - if (height > (size_t)std::numeric_limits::max || - width > (size_t)std::numeric_limits::max) { + if (height > (size_t)std::numeric_limits::max || width > (size_t)std::numeric_limits::max) + { THROW_IE_EXCEPTION << "File size is too big: " << height << " X " << width; } @@ -511,55 +560,60 @@ static UNUSED bool writeOutputBmp(unsigned char *data, size_t height, info[22] = (unsigned char)(sizeData >> 16); info[23] = (unsigned char)(sizeData >> 24); - outFile.write(reinterpret_cast(file), sizeof(file)); - outFile.write(reinterpret_cast(info), sizeof(info)); + outFile.write(reinterpret_cast(file), sizeof(file)); + outFile.write(reinterpret_cast(info), sizeof(info)); - unsigned char pad[3] = {0, 0, 0}; + unsigned char pad[3] = { 0, 0, 0 }; - for (size_t y = 0; y < height; y++) { - for (size_t x = 0; x < width; x++) { + for (size_t y = 0; y < height; y++) + { + for (size_t x = 0; x < width; x++) + { unsigned char pixel[3]; pixel[0] = data[y * width * 3 + x * 3]; pixel[1] = data[y * width * 3 + x * 3 + 1]; pixel[2] = data[y * width * 3 + x * 3 + 2]; - outFile.write(reinterpret_cast(pixel), 3); + outFile.write(reinterpret_cast(pixel), 3); } - outFile.write(reinterpret_cast(pad), padSize); + outFile.write(reinterpret_cast(pad), padSize); } return true; } -inline double getDurationOf(std::function func) { +inline double getDurationOf(std::function func) +{ auto t0 = std::chrono::high_resolution_clock::now(); func(); auto t1 = std::chrono::high_resolution_clock::now(); std::chrono::duration fs = t1 - t0; - return std::chrono::duration_cast< - std::chrono::duration>>(fs) - .count(); + return std::chrono::duration_cast>>(fs).count(); } -static UNUSED void printPerformanceCounts( - const std::map - &performanceMap, - std::ostream &stream, bool bshowHeader = true) { +static UNUSED void +printPerformanceCounts(const std::map& performanceMap, + std::ostream& stream, bool bshowHeader = true) +{ long long totalTime = 0; // Print performance counts - if (bshowHeader) { + if (bshowHeader) + { stream << std::endl << "performance counts:" << std::endl << std::endl; } - for (const auto &it : performanceMap) { + for (const auto& it : performanceMap) + { std::string toPrint(it.first); const int maxLayerName = 30; - if (it.first.length() >= maxLayerName) { + if (it.first.length() >= maxLayerName) + { toPrint = it.first.substr(0, maxLayerName - 4); toPrint += "..."; } stream << std::setw(maxLayerName) << std::left << toPrint; - switch (it.second.status) { + switch (it.second.status) + { case InferenceEngine::InferenceEngineProfileInfo::EXECUTED: stream << std::setw(15) << std::left << "EXECUTED"; break; @@ -570,24 +624,20 @@ static UNUSED void printPerformanceCounts( stream << std::setw(15) << std::left << "OPTIMIZED_OUT"; break; } - stream << std::setw(30) << std::left - << "layerType: " + std::string(it.second.layer_type) + " "; - stream << std::setw(20) << std::left - << "realTime: " + std::to_string(it.second.realTime_uSec); - stream << std::setw(20) << std::left - << " cpu: " + std::to_string(it.second.cpu_uSec); + stream << std::setw(30) << std::left << "layerType: " + std::string(it.second.layer_type) + " "; + stream << std::setw(20) << std::left << "realTime: " + std::to_string(it.second.realTime_uSec); + stream << std::setw(20) << std::left << " cpu: " + std::to_string(it.second.cpu_uSec); stream << " execType: " << it.second.exec_type << std::endl; - if (it.second.realTime_uSec > 0) { + if (it.second.realTime_uSec > 0) + { totalTime += it.second.realTime_uSec; } } - stream << std::setw(20) << std::left - << "Total time: " + std::to_string(totalTime) << " microseconds" - << std::endl; + stream << std::setw(20) << std::left << "Total time: " + std::to_string(totalTime) << " microseconds" << std::endl; } -static UNUSED void printPerformanceCounts(InferenceEngine::InferRequest request, - std::ostream &stream) { +static UNUSED void printPerformanceCounts(InferenceEngine::InferRequest request, std::ostream& stream) +{ auto perfomanceMap = request.GetPerformanceCounts(); printPerformanceCounts(perfomanceMap, stream); } @@ -595,10 +645,9 @@ static UNUSED void printPerformanceCounts(InferenceEngine::InferRequest request, /** * @deprecated */ -static UNUSED void printPerformanceCountsPlugin( - InferenceEngine::InferenceEnginePluginPtr plugin, std::ostream &stream) { - std::map - perfomanceMap; +static UNUSED void printPerformanceCountsPlugin(InferenceEngine::InferenceEnginePluginPtr plugin, std::ostream& stream) +{ + std::map perfomanceMap; plugin->GetPerformanceCounts(perfomanceMap, nullptr); printPerformanceCounts(perfomanceMap, stream); } @@ -607,47 +656,46 @@ static UNUSED void printPerformanceCountsPlugin( * @brief This class represents an object that is found by an object detection * net */ -class DetectedObject { - public: +class DetectedObject +{ +public: int objectType; float xmin, xmax, ymin, ymax, prob; bool difficult; - DetectedObject(int objectType, float xmin, float ymin, float xmax, float ymax, - float prob, bool difficult = false) - : objectType(objectType), - xmin(xmin), - xmax(xmax), - ymin(ymin), - ymax(ymax), - prob(prob), - difficult(difficult) {} + DetectedObject(int objectType, float xmin, float ymin, float xmax, float ymax, float prob, bool difficult = false) + : objectType(objectType), xmin(xmin), xmax(xmax), ymin(ymin), ymax(ymax), prob(prob), difficult(difficult) + { + } - DetectedObject(const DetectedObject &other) = default; + DetectedObject(const DetectedObject& other) = default; - static float ioU(const DetectedObject &detectedObject1_, - const DetectedObject &detectedObject2_) { + static float ioU(const DetectedObject& detectedObject1_, const DetectedObject& detectedObject2_) + { // Add small space to eliminate empty squares float epsilon = 0; // 1e-5f; - DetectedObject detectedObject1( - detectedObject1_.objectType, (detectedObject1_.xmin - epsilon), - (detectedObject1_.ymin - epsilon), (detectedObject1_.xmax - epsilon), - (detectedObject1_.ymax - epsilon), detectedObject1_.prob); - DetectedObject detectedObject2( - detectedObject2_.objectType, (detectedObject2_.xmin + epsilon), - (detectedObject2_.ymin + epsilon), (detectedObject2_.xmax), - (detectedObject2_.ymax), detectedObject2_.prob); + DetectedObject detectedObject1(detectedObject1_.objectType, (detectedObject1_.xmin - epsilon), + (detectedObject1_.ymin - epsilon), (detectedObject1_.xmax - epsilon), + (detectedObject1_.ymax - epsilon), detectedObject1_.prob); + DetectedObject detectedObject2(detectedObject2_.objectType, (detectedObject2_.xmin + epsilon), + (detectedObject2_.ymin + epsilon), (detectedObject2_.xmax), (detectedObject2_.ymax), + detectedObject2_.prob); - if (detectedObject1.objectType != detectedObject2.objectType) { + if (detectedObject1.objectType != detectedObject2.objectType) + { // objects are different, so the result is 0 return 0.0f; } - if (detectedObject1.xmax < detectedObject1.xmin) return 0.0; - if (detectedObject1.ymax < detectedObject1.ymin) return 0.0; - if (detectedObject2.xmax < detectedObject2.xmin) return 0.0; - if (detectedObject2.ymax < detectedObject2.ymin) return 0.0; + if (detectedObject1.xmax < detectedObject1.xmin) + return 0.0; + if (detectedObject1.ymax < detectedObject1.ymin) + return 0.0; + if (detectedObject2.xmax < detectedObject2.xmin) + return 0.0; + if (detectedObject2.ymax < detectedObject2.ymin) + return 0.0; float xmin = (std::max)(detectedObject1.xmin, detectedObject2.xmin); float ymin = (std::max)(detectedObject1.ymin, detectedObject2.ymin); @@ -663,9 +711,12 @@ class DetectedObject { // intersection float intr; - if ((xmax >= xmin) && (ymax >= ymin)) { + if ((xmax >= xmin) && (ymax >= ymin)) + { intr = (addendum + xmax - xmin) * (addendum + ymax - ymin); - } else { + } + else + { intr = 0.0f; } @@ -680,33 +731,42 @@ class DetectedObject { return static_cast(intr) / unn; } - DetectedObject scale(float scale_x, float scale_y) const { - return DetectedObject(objectType, xmin * scale_x, ymin * scale_y, - xmax * scale_x, ymax * scale_y, prob, difficult); + DetectedObject scale(float scale_x, float scale_y) const + { + return DetectedObject(objectType, xmin * scale_x, ymin * scale_y, xmax * scale_x, ymax * scale_y, prob, difficult); } }; -class ImageDescription { - public: +class ImageDescription +{ +public: const std::list alist; const bool check_probs; - explicit ImageDescription(const std::list &alist, - bool check_probs = false) - : alist(alist), check_probs(check_probs) {} + explicit ImageDescription(const std::list& alist, bool check_probs = false) + : alist(alist), check_probs(check_probs) + { + } - ImageDescription scale(float scale_x, float scale_y) const { + ImageDescription scale(float scale_x, float scale_y) const + { std::list slist; - for (auto &dob : alist) { + for (auto& dob : alist) + { slist.push_back(dob.scale(scale_x, scale_y)); } return ImageDescription(slist, check_probs); } }; -struct AveragePrecisionCalculator { - private: - enum MatchKind { TruePositive, FalsePositive }; +struct AveragePrecisionCalculator +{ +private: + enum MatchKind + { + TruePositive, + FalsePositive + }; /** * Here we count all TP and FP matches for all the classes in all the images. @@ -717,18 +777,20 @@ struct AveragePrecisionCalculator { double threshold; - static bool SortBBoxDescend(const DetectedObject &bbox1, - const DetectedObject &bbox2) { + static bool SortBBoxDescend(const DetectedObject& bbox1, const DetectedObject& bbox2) + { return bbox1.prob > bbox2.prob; } - static bool SortPairDescend(const std::pair &p1, - const std::pair &p2) { + static bool SortPairDescend(const std::pair& p1, const std::pair& p2) + { return p1.first > p2.first; } - public: - AveragePrecisionCalculator(double threshold) : threshold(threshold) {} +public: + AveragePrecisionCalculator(double threshold) : threshold(threshold) + { + } // gt_bboxes -> des // bboxes -> det @@ -741,18 +803,18 @@ struct AveragePrecisionCalculator { * @param width - width of the rectangle * @param detectedObjects - vector of detected objects */ -static UNUSED void addRectangles(unsigned char *data, size_t height, - size_t width, - std::vector detectedObjects) { - std::vector colors = { - {128, 64, 128}, {232, 35, 244}, {70, 70, 70}, {156, 102, 102}, - {153, 153, 190}, {153, 153, 153}, {30, 170, 250}, {0, 220, 220}, - {35, 142, 107}, {152, 251, 152}, {180, 130, 70}, {60, 20, 220}, - {0, 0, 255}, {142, 0, 0}, {70, 0, 0}, {100, 60, 0}, - {90, 0, 0}, {230, 0, 0}, {32, 11, 119}, {0, 74, 111}, - {81, 0, 81}}; - - for (size_t i = 0; i < detectedObjects.size(); i++) { +static UNUSED void addRectangles(unsigned char* data, size_t height, size_t width, + std::vector detectedObjects) +{ + std::vector colors = { { 128, 64, 128 }, { 232, 35, 244 }, { 70, 70, 70 }, { 156, 102, 102 }, + { 153, 153, 190 }, { 153, 153, 153 }, { 30, 170, 250 }, { 0, 220, 220 }, + { 35, 142, 107 }, { 152, 251, 152 }, { 180, 130, 70 }, { 60, 20, 220 }, + { 0, 0, 255 }, { 142, 0, 0 }, { 70, 0, 0 }, { 100, 60, 0 }, + { 90, 0, 0 }, { 230, 0, 0 }, { 32, 11, 119 }, { 0, 74, 111 }, + { 81, 0, 81 } }; + + for (size_t i = 0; i < detectedObjects.size(); i++) + { int cls = detectedObjects[i].objectType % colors.size(); int xmin = detectedObjects[i].xmin * width; @@ -762,7 +824,8 @@ static UNUSED void addRectangles(unsigned char *data, size_t height, size_t shift_first = ymin * width * 3; size_t shift_second = ymax * width * 3; - for (int x = xmin; x < xmax; x++) { + for (int x = xmin; x < xmax; x++) + { data[shift_first + x * 3] = colors.at(cls).red(); data[shift_first + x * 3 + 1] = colors.at(cls).green(); data[shift_first + x * 3 + 2] = colors.at(cls).blue(); @@ -773,7 +836,8 @@ static UNUSED void addRectangles(unsigned char *data, size_t height, shift_first = xmin * 3; shift_second = xmax * 3; - for (int y = ymin; y < ymax; y++) { + for (int y = ymin; y < ymax; y++) + { data[shift_first + y * width * 3] = colors.at(cls).red(); data[shift_first + y * width * 3 + 1] = colors.at(cls).green(); data[shift_first + y * width * 3 + 2] = colors.at(cls).blue(); diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/engines/engine.h b/dynamic_vino_lib/include/dynamic_vino_lib/engines/engine.h index 5eb284a2..a467a02a 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/engines/engine.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/engines/engine.h @@ -35,12 +35,20 @@ namespace Engines { class Engine { - public: +public: +#if (defined(USE_OLD_E_PLUGIN_API)) /** + * DEPRECATED! instead of using Engine(InferenceEngine::InferRequest::Ptr &) * @brief Create an NetworkEngine instance * from a inference plugin and an inference network. */ Engine(InferenceEngine::InferencePlugin, Models::BaseModel::Ptr); +#endif + + /** + * @brief Using an Inference Request to initialize the inference Engine. + */ + Engine(InferenceEngine::InferRequest::Ptr&); /** * @brief Get the inference request this instance holds. * @return The inference request this instance holds. @@ -60,8 +68,8 @@ class Engine request_->SetCompletionCallback(callbackToSet); } - private: - InferenceEngine::InferRequest::Ptr request_; +private: + InferenceEngine::InferRequest::Ptr request_ = nullptr; }; } // namespace Engines diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/engines/engine_manager.h b/dynamic_vino_lib/include/dynamic_vino_lib/engines/engine_manager.h new file mode 100644 index 00000000..cf6a8896 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/engines/engine_manager.h @@ -0,0 +1,59 @@ +// Copyright (c) 2018-2019 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for NetworkEngine class + * @file engine_manager.h + */ +#ifndef DYNAMIC_VINO_LIB__ENGINES__ENGINE_MANAGER_HPP_ +#define DYNAMIC_VINO_LIB__ENGINES__ENGINE_MANAGER_HPP_ + +#pragma once + +#include "dynamic_vino_lib/models/base_model.h" +#include "dynamic_vino_lib/engines/engine.h" +#include "inference_engine.hpp" + +namespace Engines +{ +/** + * @class EngineManager + * @brief This class is used to create and manage Inference engines. + */ +class EngineManager +{ +public: + /** + * @brief Create InferenceEngine instance by given Engine Name and Network. + * @return The shared pointer of created Engine instance. + */ + std::shared_ptr createEngine(const std::string&, const std::shared_ptr&); + +private: +#if (defined(USE_OLD_E_PLUGIN_API)) + std::map plugins_for_devices_; + + std::unique_ptr makePluginByName(const std::string& device_name, + const std::string& custom_cpu_library_message, + const std::string& custom_cldnn_message, + bool performance_message); + + std::shared_ptr createEngine_beforeV2019R2(const std::string&, const std::shared_ptr&); +#endif + + std::shared_ptr createEngine_V2019R2_plus(const std::string&, const std::shared_ptr&); +}; +} // namespace Engines + +#endif // DYNAMIC_VINO_LIB__ENGINES__ENGINE_MANAGER_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/factory.h b/dynamic_vino_lib/include/dynamic_vino_lib/factory.h deleted file mode 100644 index 98210efb..00000000 --- a/dynamic_vino_lib/include/dynamic_vino_lib/factory.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2018 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @brief a header file with declaration of Factory class - * @file factory.h - */ - -#ifndef DYNAMIC_VINO_LIB_FACTORY_H -#define DYNAMIC_VINO_LIB_FACTORY_H - -#include - -#include -#include - -#include "dynamic_vino_lib/common.h" -#include "dynamic_vino_lib/inputs/base_input.h" -#include "extension/ext_list.hpp" - -/** -* @class Factory -* @brief This class is a factory class that produces the derived input device -* class corresponding to -* the input string -*/ -class Factory -{ - public: - /** - * @brief This function produces the derived input device class corresponding - * to the input string - * @param[in] input device name, can be RealSenseCamera, StandardCamera or - * video directory - * @param[in] input file path, file path for a video or image file - * @return the instance of derived input device referenced by a smart pointer - */ - static std::shared_ptr makeInputDeviceByName( - const std::string& input_device_name, const std::string& input_file_path); - /** - * @brief This function produces the derived inference plugin corresponding to - * the input string - * @param[in] device_name The name of target device (CPU, GPU, FPGA, MYRIAD) - * @param[in] custom_cpu_library_message Absolute path to CPU library with user - * layers - * @param[in] custom_cldnn_message clDNN custom kernels path - * @param[in] performance_message Enable per-layer performance report - * @return the instance of derived inference plugin referenced by a smart - * pointer - */ - static std::shared_ptr makePluginByName( - const std::string& device_name, - const std::string& custom_cpu_library_message, - const std::string& custom_cldnn_message, bool performance_message); -}; - -#endif // DYNAMIC_VINO_LIB_FACTORY_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/age_gender_detection.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/age_gender_detection.h index 6c1b2749..dc25b836 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/age_gender_detection.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/age_gender_detection.h @@ -44,7 +44,7 @@ namespace dynamic_vino_lib */ class AgeGenderResult : public Result { - public: +public: explicit AgeGenderResult(const cv::Rect& location); /** * @brief Get the age of the detected person from the result. @@ -75,7 +75,7 @@ class AgeGenderResult : public Result */ class AgeGenderDetection : public BaseInference { - public: +public: using Result = dynamic_vino_lib::AgeGenderResult; AgeGenderDetection(); ~AgeGenderDetection() override; @@ -108,7 +108,7 @@ class AgeGenderDetection : public BaseInference * @brief Get the length of the buffer result array. * @return The length of the buffer result array. */ - const int getResultsLength() const override; + int getResultsLength() const override; /** * @brief Get the location of result with respect * to the frame generated by the input device. @@ -124,10 +124,11 @@ class AgeGenderDetection : public BaseInference * @brief Show the observed detection result either through image window * or ROS topic. */ - const void observeOutput( - const std::shared_ptr& output) override; + void observeOutput(const std::shared_ptr& output) override; - private: + const std::vector getFilteredROIs(const std::string filter_conditions) const override; + +private: std::shared_ptr valid_model_; std::vector results_; }; diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/base_filter.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/base_filter.h new file mode 100644 index 00000000..ded5ebd6 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/base_filter.h @@ -0,0 +1,204 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for BaseFilter Class + * @file base_filter.hpp + */ +#ifndef DYNAMIC_VINO_LIB__INFERENCES__BASE_FILTER_HPP_ +#define DYNAMIC_VINO_LIB__INFERENCES__BASE_FILTER_HPP_ + +#include +#include +#include +#include +#include "dynamic_vino_lib/inferences/base_inference.h" + +namespace dynamic_vino_lib +{ +/** + * @class BaseFilter + * @brief Base class for result filter. + */ +class BaseFilter +{ +public: + BaseFilter(); + /** + * @brief Initiate a result filter. + */ + virtual void init() = 0; + + /** + * @brief Get the filtered results' ROIs. + * @return The filtered ROIs. + */ + virtual std::vector getFilteredLocations() = 0; + + /** + * @brief Check if the filter conditions is valid for filtering. + * @param[in] Filter conditions. + * @return true if some of the conditions are valid, otherwise false. + */ + bool isValidFilterConditions(const std::string&); + + /** + * @brief Accept the filter conditions for filtering. + * @param[in] Filter conditions. + */ + void acceptFilterConditions(const std::string&); + + /** + * @brief Decide whether the input string is a relational operator or not. + * @param[in] A string to be decided. + * @return True if the input string is a relational operator, false if not. + */ + bool isRelationOperator(const std::string&); + + /** + * @brief Decide whether the input string is a logic operator or not. + * @param[in] A string to be decided. + * @return True if the input string is a logic operator, false if not. + */ + bool isLogicOperator(const std::string&); + + /** + * @brief Decide whether the an operator has a higher priority than anthor. + * @param[in] The two operators. + * @return True if the first operator has higher priority, false if not. + */ + bool isPriorTo(const std::string&, const std::string&); + + /** + * @brief Convert the input bool variable to a string type. + * @param[in] A bool type to be converted. + * @return A converted string result. + */ + std::string boolToStr(bool); + + /** + * @brief Convert the input string variable to a bool type. + * @param[in] A string type to be converted. + * @return A converted bool result. + */ + bool strToBool(const std::string&); + + /** + * @brief Get the filter conditions in the suffix order. + * @return A vector with suffix-order filter conditions. + */ + const std::vector& getSuffixConditions() const; + + /** + * @brief Do logic operation with the given bool values and the operator. + * @param[in] A bool string, an logic operator, the other bool string. + * @return The logic operation result. + */ + bool logicOperation(const std::string&, const std::string&, const std::string&); + + /** + * @brief Compare two strings with a given relational operator. + * @param[in] A string, an relational operator, the other string. + * @return True if valid, false if not. + */ + static bool stringCompare(const std::string&, const std::string&, const std::string&); + + /** + * @brief Compare two floats with a given relational operator. + * @param[in] A float number, an relational operator, the other float number. + * @return True if valid, false if not. + */ + static bool floatCompare(float, const std::string&, float); + + /** + * @brief Convert a string into a float number. + * @param[in] A string to be converted. + * @return The converted float number, 0 if string is invalid. + */ + static float stringToFloat(const std::string&); + +/** + * @brief A macro to decide whether a given result satisfies the filter condition. + * @param[in] A key to function mapping, a given result. + * @return True if valid, false if not. + */ +#define ISVALIDRESULT(key_to_function, result) \ + { \ + std::vector suffix_conditons = getSuffixConditions(); \ + std::stack result_stack; \ + for (auto elem : suffix_conditons) \ + { \ + if (!isRelationOperator(elem) && !isLogicOperator(elem)) \ + { \ + result_stack.push(elem); \ + } \ + else \ + { \ + try \ + { \ + std::string str1 = result_stack.top(); \ + result_stack.pop(); \ + std::string str2 = result_stack.top(); \ + result_stack.pop(); \ + if (key_to_function.count(str2)) \ + { \ + result_stack.push(boolToStr(key_to_function[str2](result, elem, str1))); \ + } \ + else \ + { \ + result_stack.push(boolToStr(logicOperation(str1, elem, str2))); \ + } \ + } \ + catch (...) \ + { \ + slog::err << "Invalid filter conditions format!" << slog::endl; \ + } \ + } \ + } \ + if (result_stack.empty()) \ + { \ + return true; \ + } \ + return strToBool(result_stack.top()); \ + } + +private: + /** + * @brief Parse the filter conditions and stores it into a vector. + * @param[in] A string form filter conditions. + * @return The vector form filter conditions. + */ + std::vector split(const std::string& filter_conditions); + + /** + * @brief Convert the infix expression into suffix expression. + * @param[in] The infix form filter conditions. + */ + void infixToSuffix(std::vector& infix_conditions); + + /** + * @brief Strip the extra space in a string. + * @param[in] A string to be striped. + * @return The striped string. + */ + std::string strip(const std::string& str); + + std::string striped_conditions_ = ""; + std::vector suffix_conditons_; + std::vector relation_operators_ = { "==", "!=", "<=", ">=", "<", ">" }; + std::vector logic_operators_ = { "&&", "||" }; +}; +} // namespace dynamic_vino_lib + +#endif // DYNAMIC_VINO_LIB__INFERENCES__BASE_FILTER_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/base_inference.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/base_inference.h index e2b569d9..0fa21adf 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/base_inference.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/base_inference.h @@ -23,8 +23,10 @@ #include #include +#include #include "dynamic_vino_lib/engines/engine.h" +#include "dynamic_vino_lib/models/base_model.h" #include "dynamic_vino_lib/slog.h" #include "inference_engine.hpp" #include "opencv2/opencv.hpp" @@ -41,30 +43,27 @@ class BaseOutput; * @param[in] batch_index Indicates the batch index for the frame. */ template -void matU8ToBlob(const cv::Mat& orig_image, InferenceEngine::Blob::Ptr& blob, - float scale_factor = 1.0, int batch_index = 0) +void matU8ToBlob(const cv::Mat& orig_image, InferenceEngine::Blob::Ptr& blob, float scale_factor = 1.0, + int batch_index = 0) { InferenceEngine::SizeVector blob_size = blob->getTensorDesc().getDims(); - // const size_t width = blob_size[3]; - const int width = blob_size[3]; - // const size_t height = blob_size[2]; - const int height = blob_size[2]; - // const size_t channels = blob_size[1]; - const int channels = blob_size[1]; + const size_t width = blob_size[3]; + const size_t height = blob_size[2]; + const size_t channels = blob_size[1]; T* blob_data = blob->buffer().as(); cv::Mat resized_image(orig_image); - if (width != orig_image.size().width || height != orig_image.size().height) + if (width != (size_t)orig_image.size().width || height != (size_t)orig_image.size().height) { cv::resize(orig_image, resized_image, cv::Size(width, height)); } int batchOffset = batch_index * width * height * channels; - for (int c = 0; c < channels; c++) + for (size_t c = 0; c < channels; c++) { - for (int h = 0; h < height; h++) + for (size_t h = 0; h < height; h++) { - for (int w = 0; w < width; w++) + for (size_t w = 0; w < width; w++) { blob_data[batchOffset + c * width * height + h * width + w] = resized_image.at(h, w)[c] * scale_factor; @@ -81,12 +80,15 @@ namespace dynamic_vino_lib */ class Result { - public: +public: friend class BaseInference; explicit Result(const cv::Rect& location); - inline const cv::Rect getLocation() const { return location_; } + inline const cv::Rect getLocation() const + { + return location_; + } - private: +private: cv::Rect location_; }; @@ -96,7 +98,7 @@ class Result */ class BaseInference { - public: +public: BaseInference(); virtual ~BaseInference(); /** @@ -116,10 +118,25 @@ class BaseInference * @brief Get the number of enqueued frames to be infered. * @return The number of enqueued frames to be infered. */ - inline const int getEnqueuedNum() const + inline int getEnqueuedNum() const + { + return enqueued_frames_; + } + + /** + * @brief Get the size of the maximum of inference batch. + * @return The max batch size. + */ + inline int getMaxBatchSize() const + { + return max_batch_size_; + } + + inline void setMaxBatchSize(int max) { - return enqueued_frames; + max_batch_size_ = max; } + /** * @brief Enqueue a frame to this class. * The frame will be buffered but not infered yet. @@ -128,16 +145,15 @@ class BaseInference * to the frame generated by the input device. * @return Whether this operation is successful. */ - virtual bool enqueue(const cv::Mat& frame, - const cv::Rect& input_frame_loc) = 0; + virtual bool enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) = 0; /** * @brief Start inference for all buffered frames. * @return Whether this operation is successful. */ virtual bool submitRequest(); + virtual bool SynchronousRequest(); - virtual const void observeOutput( - const std::shared_ptr& output) = 0; + virtual void observeOutput(const std::shared_ptr& output) = 0; /** * @brief This function will fetch the results of the previous inference and @@ -149,7 +165,7 @@ class BaseInference /** * @brief Get the length of the buffer result array. */ - virtual const int getResultsLength() const = 0; + virtual int getResultsLength() const = 0; /** * @brief Get the location of result with respect * to the frame generated by the input device. @@ -162,42 +178,41 @@ class BaseInference */ virtual const std::string getName() const = 0; - protected: + virtual const std::vector getFilteredROIs(const std::string filter_conditions) const = 0; + + void addCandidatedModel(std::shared_ptr model); + +protected: /** * @brief Enqueue the fram into the input blob of the target calculation * device. Check OpenVINO document for detailed information. * @return Whether this operation is successful. */ template - bool enqueue(const cv::Mat& frame, const cv::Rect&, float scale_factor, - int batch_index, const std::string& input_name) + bool enqueue(const cv::Mat& frame, const cv::Rect&, float scale_factor, int batch_index, + const std::string& input_name) { - if (enqueued_frames == max_batch_size_) + if (enqueued_frames_ == max_batch_size_) { - slog::warn << "Number of " << getName() << "input more than maximum(" - << max_batch_size_ << ") processed by inference" << slog::endl; + slog::warn << "Number of " << getName() << "input more than maximum(" << max_batch_size_ + << ") processed by inference" << slog::endl; return false; } - InferenceEngine::Blob::Ptr input_blob = - engine_->getRequest()->GetBlob(input_name); + InferenceEngine::Blob::Ptr input_blob = engine_->getRequest()->GetBlob(input_name); matU8ToBlob(frame, input_blob, scale_factor, batch_index); - enqueued_frames += 1; + enqueued_frames_ += 1; return true; } - /** - * @brief Set the max batch size for one inference. - */ - inline void setMaxBatchSize(int max_batch_size) - { - max_batch_size_ = max_batch_size; - } - private: - std::shared_ptr engine_; + std::vector results_; + +protected: + std::shared_ptr engine_ = nullptr; + std::vector > candidated_models_; int max_batch_size_ = 1; - int enqueued_frames = 0; + int enqueued_frames_ = 0; bool results_fetched_ = false; }; } // namespace dynamic_vino_lib -#endif // DYNAMIC_VINO_LIB_INFERENCES_BASE_INFERENCE_H +#endif // DYNAMIC_VINO_LIB__INFERENCES__BASE_INFERENCE_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/base_reidentification.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/base_reidentification.h new file mode 100644 index 00000000..4ebfd732 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/base_reidentification.h @@ -0,0 +1,99 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for BaseReidentification Class + * @file base_reidentification.hpp + */ +#ifndef DYNAMIC_VINO_LIB__INFERENCES__BASE_REIDENTIFICATION_HPP_ +#define DYNAMIC_VINO_LIB__INFERENCES__BASE_REIDENTIFICATION_HPP_ +#include +#include +#include +#include +#include +#include + +// namespace +namespace dynamic_vino_lib +{ +/** + * @class Tracker + * @brief Class for storing and tracking the detected objects. + */ +class Tracker +{ +public: + explicit Tracker(int, double, double); + /** + * @brief Process the new detected track. + * @param[in] feature The new detected track feature. + * @return The detected track ID. + */ + int processNewTrack(const std::vector& feature); + +private: + /** + * @brief Find the most similar track from the recorded tracks. + * @param[in] feature The input track's feature. + * @param[in] most similar track's ID to be recorded. + * @return similarity with the most similar track. + */ + double findMostSimilarTrack(const std::vector& feature, int& most_similar_id); + /** + * @brief Update the matched track's feature by the new track. + * @param[in] track_id The matched track ID. + * @param[in] feature The matched track's feature + */ + void updateMatchTrack(int track_id, const std::vector& feature); + /** + * @brief Remove the earlest track from the recorded tracks. + */ + void removeEarlestTrack(); + /** + * @brief Add a new track to the recorded tracks, remove oldest track if needed. + * @param[in] feature A track's feature. + * @return new added track's ID. + */ + int addNewTrack(const std::vector& feature); + /** + * @brief Calculate the cosine similarity between two features. + * @return The simlarity result. + */ + double calcSimilarity(const std::vector& feature_a, const std::vector& feature_b); + /** + * @brief get the current millisecond count since epoch. + * @return millisecond count since epoch. + */ + int64_t getCurrentTime(); + + bool saveTracksToFile(std::string filepath); + bool loadTracksFromFile(std::string filepath); + + struct Track + { + int64_t lastest_update_time; + std::vector feature; + }; + + int max_record_size_ = 1000; + int max_track_id_ = -1; + double same_track_thresh_ = 0.9; + double new_track_thresh_ = 0.3; + std::mutex tracks_mtx_; + std::unordered_map recorded_tracks_; +}; + +} // namespace dynamic_vino_lib +#endif // DYNAMIC_VINO_LIB__INFERENCES__BASE_REIDENTIFICATION_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/emotions_detection.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/emotions_detection.h index bb4e1f29..7de27d47 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/emotions_detection.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/emotions_detection.h @@ -43,7 +43,7 @@ namespace dynamic_vino_lib */ class EmotionsResult : public Result { - public: +public: friend class EmotionsDetection; explicit EmotionsResult(const cv::Rect& location); /** @@ -55,7 +55,7 @@ class EmotionsResult : public Result return label_; } - private: +private: std::string label_ = ""; float confidence_ = -1; }; @@ -66,7 +66,7 @@ class EmotionsResult : public Result */ class EmotionsDetection : public BaseInference { - public: +public: using Result = dynamic_vino_lib::EmotionsResult; EmotionsDetection(); ~EmotionsDetection() override; @@ -99,7 +99,7 @@ class EmotionsDetection : public BaseInference * @brief Get the length of the buffer result array. * @return The length of the buffer result array. */ - const int getResultsLength() const override; + int getResultsLength() const override; /** * @brief Get the location of result with respect * to the frame generated by the input device. @@ -115,10 +115,15 @@ class EmotionsDetection : public BaseInference * @brief Show the observed detection result either through image window * or ROS topic. */ - const void observeOutput( - const std::shared_ptr& output) override; + void observeOutput(const std::shared_ptr& output) override; - private: + std::vector getResults() + { + return results_; + } + const std::vector getFilteredROIs(const std::string filter_conditions) const override; + +private: std::shared_ptr valid_model_; std::vector results_; }; diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/face_detection.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/face_detection.h index 7addb3a5..c24a64db 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/face_detection.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/face_detection.h @@ -32,6 +32,7 @@ #include "dynamic_vino_lib/engines/engine.h" #include "dynamic_vino_lib/inferences/base_inference.h" +#include "dynamic_vino_lib/inferences/object_detection.h" #include "dynamic_vino_lib/models/face_detection_model.h" #include "inference_engine.hpp" #include "opencv2/opencv.hpp" @@ -43,94 +44,20 @@ namespace dynamic_vino_lib * @class FaceDetectionResult * @brief Class for storing and processing face detection result. */ -class FaceDetectionResult : public Result +class FaceDetectionResult : public ObjectDetectionResult { - public: - friend class FaceDetection; +public: explicit FaceDetectionResult(const cv::Rect& location); - std::string getLabel() const - { - return label_; - } - /** - * @brief Get the confidence that the detected area is a face. - * @return The confidence value. - */ - float getConfidence() const - { - return confidence_; - } - - private: - std::string label_ = ""; - float confidence_ = -1; }; /** * @class FaceDetection * @brief Class to load face detection model and perform face detection. */ -class FaceDetection : public BaseInference +class FaceDetection : public ObjectDetection { - public: - using Result = dynamic_vino_lib::FaceDetectionResult; - explicit FaceDetection(double); - ~FaceDetection() override; - /** - * @brief Load the face detection model. - */ - void loadNetwork(std::shared_ptr); - /** - * @brief Enqueue a frame to this class. - * The frame will be buffered but not infered yet. - * @param[in] frame The frame to be enqueued. - * @param[in] input_frame_loc The location of the enqueued frame with respect - * to the frame generated by the input device. - * @return Whether this operation is successful. - */ - bool enqueue(const cv::Mat&, const cv::Rect&) override; - /** - * @brief Start inference for all buffered frames. - * @return Whether this operation is successful. - */ - bool submitRequest() override; - /** - * @brief This function will fetch the results of the previous inference and - * stores the results in a result buffer array. All buffered frames will be - * cleared. - * @return Whether the Inference object fetches a result this time - */ - bool fetchResults() override; - /** - * @brief Get the length of the buffer result array. - * @return The length of the buffer result array. - */ - const int getResultsLength() const override; - /** - * @brief Get the location of result with respect - * to the frame generated by the input device. - * @param[in] idx The index of the result. - */ - const dynamic_vino_lib::Result* getLocationResult(int idx) const override; - /** - * @brief Show the observed detection result either through image window - or ROS topic. - */ - const void observeOutput(const std::shared_ptr& output); - /** - * @brief Get the name of the Inference instance. - * @return The name of the Inference instance. - */ - const std::string getName() const override; - - private: - std::shared_ptr valid_model_; - std::vector results_; - int width_ = 0; - int height_ = 0; - int max_proposal_count_; - int object_size_; - double show_output_thresh_ = 0; +public: + explicit FaceDetection(bool, double); }; } // namespace dynamic_vino_lib #endif // DYNAMIC_VINO_LIB_INFERENCES_FACE_DETECTION_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/face_reidentification.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/face_reidentification.h new file mode 100644 index 00000000..fe2b9bfc --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/face_reidentification.h @@ -0,0 +1,115 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for FaceReidentification Class + * @file face_reidentification.hpp + */ +#ifndef DYNAMIC_VINO_LIB__INFERENCES__FACE_REIDENTIFICATION_H_ +#define DYNAMIC_VINO_LIB__INFERENCES__FACE_REIDENTIFICATION_H_ +#include +#include +#include +#include "dynamic_vino_lib/models/face_reidentification_model.h" +#include "dynamic_vino_lib/engines/engine.h" +#include "dynamic_vino_lib/inferences/base_inference.h" +#include "dynamic_vino_lib/inferences/base_reidentification.h" +#include "inference_engine.hpp" +#include "opencv2/opencv.hpp" +// namespace +namespace dynamic_vino_lib +{ +/** + * @class FaceReidentificationResult + * @brief Class for storing and processing face reidentification result. + */ +class FaceReidentificationResult : public Result +{ +public: + friend class FaceReidentification; + explicit FaceReidentificationResult(const cv::Rect& location); + std::string getFaceID() const + { + return face_id_; + } + +private: + std::string face_id_ = "No.#"; +}; + +/** + * @class FaceReidentification + * @brief Class to load face reidentification model and perform face reidentification. + */ +class FaceReidentification : public BaseInference +{ +public: + using Result = dynamic_vino_lib::FaceReidentificationResult; + explicit FaceReidentification(double); + ~FaceReidentification() override; + /** + * @brief Load the face reidentification model. + */ + void loadNetwork(std::shared_ptr); + /** + * @brief Enqueue a frame to this class. + * The frame will be buffered but not infered yet. + * @param[in] frame The frame to be enqueued. + * @param[in] input_frame_loc The location of the enqueued frame with respect + * to the frame generated by the input device. + * @return Whether this operation is successful. + */ + bool enqueue(const cv::Mat&, const cv::Rect&) override; + /** + * @brief Start inference for all buffered frames. + * @return Whether this operation is successful. + */ + bool submitRequest() override; + /** + * @brief This function will fetch the results of the previous inference and + * stores the results in a result buffer array. All buffered frames will be + * cleared. + * @return Whether the Inference object fetches a result this time + */ + bool fetchResults() override; + /** + * @brief Get the length of the buffer result array. + * @return The length of the buffer result array. + */ + int getResultsLength() const override; + /** + * @brief Get the location of result with respect + * to the frame generated by the input device. + * @param[in] idx The index of the result. + */ + const dynamic_vino_lib::Result* getLocationResult(int idx) const override; + /** + * @brief Show the observed reidentification result either through image window + or ROS topic. + */ + void observeOutput(const std::shared_ptr& output); + /** + * @brief Get the name of the Inference instance. + * @return The name of the Inference instance. + */ + const std::string getName() const override; + const std::vector getFilteredROIs(const std::string filter_conditions) const override; + +private: + std::shared_ptr valid_model_; + std::vector results_; + std::shared_ptr face_tracker_; +}; +} // namespace dynamic_vino_lib +#endif // DYNAMIC_VINO_LIB__INFERENCES__FACE_REIDENTIFICATION_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/head_pose_detection.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/head_pose_detection.h index 125d6207..f5ef466a 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/head_pose_detection.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/head_pose_detection.h @@ -39,7 +39,7 @@ namespace dynamic_vino_lib */ class HeadPoseResult : public Result { - public: +public: friend class HeadPoseDetection; explicit HeadPoseResult(const cv::Rect& location); /** @@ -67,7 +67,7 @@ class HeadPoseResult : public Result return angle_r_; } - private: +private: float angle_y_ = -1; float angle_p_ = -1; float angle_r_ = -1; @@ -79,7 +79,7 @@ class HeadPoseResult : public Result */ class HeadPoseDetection : public BaseInference { - public: +public: using Result = dynamic_vino_lib::HeadPoseResult; HeadPoseDetection(); ~HeadPoseDetection() override; @@ -112,7 +112,7 @@ class HeadPoseDetection : public BaseInference * @brief Get the length of the buffer result array. * @return The length of the buffer result array. */ - const int getResultsLength() const override; + int getResultsLength() const override; /** * @brief Get the location of result with respect * to the frame generated by the input device. @@ -128,10 +128,15 @@ class HeadPoseDetection : public BaseInference * @brief Show the observed detection result either through image window or ROS topic. */ - const void observeOutput( - const std::shared_ptr& output) override; + void observeOutput(const std::shared_ptr& output) override; - private: + std::vector getResults() + { + return results_; + } + const std::vector getFilteredROIs(const std::string filter_conditions) const override; + +private: std::shared_ptr valid_model_; std::vector results_; }; diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/inference_manager.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/inference_manager.h new file mode 100644 index 00000000..4fd33f8a --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/inference_manager.h @@ -0,0 +1,103 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with declaration of Inference Manager class + * @file inference_manager.hpp + */ +#ifndef DYNAMIC_VINO_LIB__INFERENCES__INFERENCE_MANAGER_HPP_ +#define DYNAMIC_VINO_LIB__INFERENCES__INFERENCE_MANAGER_HPP_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dynamic_vino_lib/pipeline.hpp" + +/** + * @class InferenceManager + * @brief This class manages inference resources. + */ +class InferenceManager +{ +public: + /** + * @brief Get the singleton instance of InferenceManager class. + * The instance will be created when first call. + * @return The reference of InferenceManager instance. + */ + static InferenceManager& getInstance() + { + static InferenceManager manager_; + return manager_; + } + + std::shared_ptr createPipeline(const Params::ParamManager::PipelineRawData& params); + void removePipeline(const std::string& name); + InferenceManager& updatePipeline(const std::string& name, const Params::ParamManager::PipelineRawData& params); + + void runAll(); + void stopAll(); + void joinAll(); + + enum PipelineState + { + PipelineState_ThreadNotCreated, + PipelineState_ThreadStopped, + PipelineState_ThreadRunning, + PipelineState_Error + }; + struct PipelineData + { + Params::ParamManager::PipelineRawData params; + std::shared_ptr pipeline; + std::vector> spin_nodes; + std::shared_ptr thread; + PipelineState state; + }; + +private: + InferenceManager() + { + } + InferenceManager(InferenceManager const&); + void operator=(InferenceManager const&); + void threadPipeline(const char* name); + std::map> + parseInputDevice(const Params::ParamManager::PipelineRawData& params); + std::map> + parseOutput(const Params::ParamManager::PipelineRawData& params); + std::map> + parseInference(const Params::ParamManager::PipelineRawData& params); + std::shared_ptr + createFaceDetection(const Params::ParamManager::InferenceParams& infer); + std::shared_ptr + createAgeGenderRecognition(const Params::ParamManager::InferenceParams& infer); + std::shared_ptr + createEmotionRecognition(const Params::ParamManager::InferenceParams& infer); + std::shared_ptr + createHeadPoseEstimation(const Params::ParamManager::InferenceParams& infer); + std::shared_ptr + createObjectDetection(const Params::ParamManager::InferenceParams& infer); + + std::map pipelines_; + // std::map plugins_for_devices_; +}; + +#endif // DYNAMIC_VINO_LIB__INFERENCES__INFERENCE_MANAGER_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/landmarks_detection.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/landmarks_detection.h new file mode 100644 index 00000000..a69e9e0f --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/landmarks_detection.h @@ -0,0 +1,112 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for LandmarksDetection Class + * @file landmarks_detection.hpp + */ +#ifndef DYNAMIC_VINO_LIB__INFERENCES__LANDMARKS_DETECTION_HPP_ +#define DYNAMIC_VINO_LIB__INFERENCES__LANDMARKS_DETECTION_HPP_ +#include +#include +#include +#include "dynamic_vino_lib/models/landmarks_detection_model.h" +#include "dynamic_vino_lib/engines/engine.h" +#include "dynamic_vino_lib/inferences/base_inference.h" +#include "inference_engine.hpp" +#include "opencv2/opencv.hpp" +// namespace +namespace dynamic_vino_lib +{ +/** + * @class LandmarksDetectionResult + * @brief Class for storing and processing landmarks detection result. + */ +class LandmarksDetectionResult : public Result +{ +public: + friend class LandmarksDetection; + explicit LandmarksDetectionResult(const cv::Rect& location); + std::vector getLandmarks() const + { + return landmark_points_; + } + +private: + std::vector landmark_points_; +}; +/** + * @class LandmarksDetection + * @brief Class to load landmarks detection model and perform landmarks detection. + */ +class LandmarksDetection : public BaseInference +{ +public: + using Result = dynamic_vino_lib::LandmarksDetectionResult; + LandmarksDetection(); + ~LandmarksDetection() override; + /** + * @brief Load the landmarks detection model. + */ + void loadNetwork(std::shared_ptr); + /** + * @brief Enqueue a frame to this class. + * The frame will be buffered but not infered yet. + * @param[in] frame The frame to be enqueued. + * @param[in] input_frame_loc The location of the enqueued frame with respect + * to the frame generated by the input device. + * @return Whether this operation is successful. + */ + bool enqueue(const cv::Mat&, const cv::Rect&) override; + /** + * @brief Start inference for all buffered frames. + * @return Whether this operation is successful. + */ + bool submitRequest() override; + /** + * @brief This function will fetch the results of the previous inference and + * stores the results in a result buffer array. All buffered frames will be + * cleared. + * @return Whether the Inference object fetches a result this time + */ + bool fetchResults() override; + /** + * @brief Get the length of the buffer result array. + * @return The length of the buffer result array. + */ + int getResultsLength() const override; + /** + * @brief Get the location of result with respect + * to the frame generated by the input device. + * @param[in] idx The index of the result. + */ + const dynamic_vino_lib::Result* getLocationResult(int idx) const override; + /** + * @brief Show the observed detection result either through image window + or ROS topic. + */ + void observeOutput(const std::shared_ptr& output); + /** + * @brief Get the name of the Inference instance. + * @return The name of the Inference instance. + */ + const std::string getName() const override; + const std::vector getFilteredROIs(const std::string filter_conditions) const override; + +private: + std::shared_ptr valid_model_; + std::vector results_; +}; +} // namespace dynamic_vino_lib +#endif // DYNAMIC_VINO_LIB__INFERENCES__LANDMARKS_DETECTION_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/license_plate_detection.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/license_plate_detection.h new file mode 100644 index 00000000..0d5b80ec --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/license_plate_detection.h @@ -0,0 +1,186 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for LicensePlateDetection Class + * @file license_plate_detection.hpp + */ +#ifndef DYNAMIC_VINO_LIB__INFERENCES__LICENSE_PLATE_DETECTION_HPP_ +#define DYNAMIC_VINO_LIB__INFERENCES__LICENSE_PLATE_DETECTION_HPP_ +#include +#include +#include +#include "dynamic_vino_lib/models/license_plate_detection_model.h" +#include "dynamic_vino_lib/engines/engine.h" +#include "dynamic_vino_lib/inferences/base_inference.h" +#include "inference_engine.hpp" +#include "opencv2/opencv.hpp" +// namespace +namespace dynamic_vino_lib +{ +/** + * @class LicensePlateDetectionResult + * @brief Class for storing and processing license plate detection result. + */ +class LicensePlateDetectionResult : public Result +{ +public: + friend class LicensePlateDetection; + explicit LicensePlateDetectionResult(const cv::Rect& location); + std::string getLicense() const + { + return license_; + } + +private: + std::string license_ = ""; +}; +/** + * @class LicensePlateDetection + * @brief Class to load license plate detection model and perform detection. + */ +class LicensePlateDetection : public BaseInference +{ +public: + using Result = dynamic_vino_lib::LicensePlateDetectionResult; + LicensePlateDetection(); + ~LicensePlateDetection() override; + /** + * @brief Load the license plate detection model. + */ + void loadNetwork(std::shared_ptr); + /** + * @brief Enqueue a frame to this class. + * The frame will be buffered but not infered yet. + * @param[in] frame The frame to be enqueued. + * @param[in] input_frame_loc The location of the enqueued frame with respect + * to the frame generated by the input device. + * @return Whether this operation is successful. + */ + bool enqueue(const cv::Mat&, const cv::Rect&) override; + /** + * @brief Set the sequence input blob + */ + void fillSeqBlob(); + /** + * @brief Start inference for all buffered frames. + * @return Whether this operation is successful. + */ + bool submitRequest() override; + /** + * @brief This function will fetch the results of the previous inference and + * stores the results in a result buffer array. All buffered frames will be + * cleared. + * @return Whether the Inference object fetches a result this time + */ + bool fetchResults() override; + /** + * @brief Get the length of the buffer result array. + * @return The length of the buffer result array. + */ + int getResultsLength() const override; + /** + * @brief Get the location of result with respect + * to the frame generated by the input device. + * @param[in] idx The index of the result. + */ + const dynamic_vino_lib::Result* getLocationResult(int idx) const override; + /** + * @brief Show the observed detection result either through image window + or ROS topic. + */ + void observeOutput(const std::shared_ptr& output); + /** + * @brief Get the name of the Inference instance. + * @return The name of the Inference instance. + */ + const std::string getName() const override; + const std::vector getFilteredROIs(const std::string filter_conditions) const override; + +private: + std::shared_ptr valid_model_; + std::vector results_; + const std::vector licenses_ = { "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z" }; +}; +} // namespace dynamic_vino_lib +#endif // DYNAMIC_VINO_LIB__INFERENCES__LICENSE_PLATE_DETECTION_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/object_detection.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/object_detection.h index 6eb50efa..8890c69b 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/object_detection.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/object_detection.h @@ -25,39 +25,118 @@ #include #include #include -#include "dynamic_vino_lib/models/object_detection_model.h" +#include #include "dynamic_vino_lib/engines/engine.h" #include "dynamic_vino_lib/inferences/base_inference.h" +#include "dynamic_vino_lib/inferences/base_filter.h" #include "inference_engine.hpp" #include "opencv2/opencv.hpp" // namespace -namespace dynamic_vino_lib { +namespace dynamic_vino_lib +{ /** * @class ObjectDetectionResult * @brief Class for storing and processing face detection result. */ -class ObjectDetectionResult : public Result { - public: +class ObjectDetectionResult : public Result +{ +public: friend class ObjectDetection; explicit ObjectDetectionResult(const cv::Rect& location); - std::string getLabel() const { return label_; } + std::string getLabel() const + { + return label_; + } + + void setLabel(const std::string& label) + { + label_ = label; + } /** * @brief Get the confidence that the detected area is a face. - * @return The confidence value. + * @return The confidence value. */ - float getConfidence() const { return confidence_; } - private: + float getConfidence() const + { + return confidence_; + } + + void setConfidence(const float& con) + { + confidence_ = con; + } + + bool operator<(const ObjectDetectionResult& s2) const + { + return this->confidence_ > s2.confidence_; + } + +private: std::string label_ = ""; float confidence_ = -1; }; + +/** + * @class ObjectDetectionResultFilter + * @brief Class for object detection result filter. + */ +class ObjectDetectionResultFilter : public BaseFilter +{ +public: + using Result = dynamic_vino_lib::ObjectDetectionResult; + + ObjectDetectionResultFilter(); + + /** + * @brief Initiate the object detection result filter. + */ + void init() override; + /** + * @brief Set the object detection results into filter. + * @param[in] The object detection results. + */ + void acceptResults(const std::vector& results); + /** + * @brief Get the filtered results' ROIs. + * @return The filtered ROIs. + */ + std::vector getFilteredLocations() override; + +private: + /** + * @brief Decide whether a result is valid for label filter condition. + * @param[in] Result to be decided, filter operator, target label value. + * @return True if valid, false if not. + */ + static bool isValidLabel(const Result& result, const std::string& op, const std::string& target); + /** + * @brief Decide whether a result is valid for confidence filter condition. + * @param[in] Result to be decided, filter operator, target confidence value. + * @return True if valid, false if not. + */ + static bool isValidConfidence(const Result& result, const std::string& op, const std::string& target); + + /** + * @brief Decide whether a result is valid. + * @param[in] Result to be decided. + * @return True if valid, false if not. + */ + bool isValidResult(const Result& result); + + std::map key_to_function_; + std::vector results_; +}; + /** * @class ObjectDetection * @brief Class to load face detection model and perform face detection. */ -class ObjectDetection : public BaseInference { - public: +class ObjectDetection : public BaseInference +{ +public: using Result = dynamic_vino_lib::ObjectDetectionResult; - explicit ObjectDetection(bool,double); + using Filter = dynamic_vino_lib::ObjectDetectionResultFilter; + explicit ObjectDetection(bool, double); ~ObjectDetection() override; /** * @brief Load the face detection model. @@ -72,11 +151,7 @@ class ObjectDetection : public BaseInference { * @return Whether this operation is successful. */ bool enqueue(const cv::Mat&, const cv::Rect&) override; - /** - * @brief Start inference for all buffered frames. - * @return Whether this operation is successful. - */ - bool submitRequest() override; + /** * @brief This function will fetch the results of the previous inference and * stores the results in a result buffer array. All buffered frames will be @@ -88,31 +163,41 @@ class ObjectDetection : public BaseInference { * @brief Get the length of the buffer result array. * @return The length of the buffer result array. */ - const int getResultsLength() const override; + int getResultsLength() const override; /** * @brief Get the location of result with respect * to the frame generated by the input device. * @param[in] idx The index of the result. */ - const dynamic_vino_lib::Result* getLocationResult(int idx) const override; + const Result* getLocationResult(int idx) const override; /** * @brief Show the observed detection result either through image window or ROS topic. */ - const void observeOutput(const std::shared_ptr& output); + void observeOutput(const std::shared_ptr& output); /** * @brief Get the name of the Inference instance. * @return The name of the Inference instance. */ const std::string getName() const override; - private: + + const std::vector getFilteredROIs(const std::string filter_conditions) const override; + /** + * @brief Calculate the IoU ratio for the given rectangles. + * @return IoU Ratio of the given rectangles. + */ + static double calcIoU(const cv::Rect& box_1, const cv::Rect& box_2); + +private: std::shared_ptr valid_model_; + std::shared_ptr result_filter_; std::vector results_; int width_ = 0; int height_ = 0; int max_proposal_count_; int object_size_; double show_output_thresh_ = 0; + bool enable_roi_constraint_ = false; }; } // namespace dynamic_vino_lib #endif // DYNAMIC_VINO_LIB_INFERENCES_OBJECT_DETECTION_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/object_segmentation.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/object_segmentation.h index e6100e1a..eac05d0c 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/object_segmentation.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/object_segmentation.h @@ -40,7 +40,7 @@ class ObjectSegmentationResult : public Result { public: friend class ObjectSegmentation; - explicit ObjectSegmentationResult(const cv::Rect & location); + explicit ObjectSegmentationResult(const cv::Rect& location); std::string getLabel() const { return label_; @@ -85,7 +85,11 @@ class ObjectSegmentation : public BaseInference * to the frame generated by the input device. * @return Whether this operation is successful. */ - bool enqueue(const cv::Mat &, const cv::Rect &) override; + bool enqueue(const cv::Mat&, const cv::Rect&) override; + + // Deprecated!! + bool enqueue_for_one_input(const cv::Mat&, const cv::Rect&); + /** * @brief Start inference for all buffered frames. * @return Whether this operation is successful. @@ -102,23 +106,24 @@ class ObjectSegmentation : public BaseInference * @brief Get the length of the buffer result array. * @return The length of the buffer result array. */ - const int getResultsLength() const override; + int getResultsLength() const override; /** * @brief Get the location of result with respect * to the frame generated by the input device. * @param[in] idx The index of the result. */ - const dynamic_vino_lib::Result * getLocationResult(int idx) const override; + const dynamic_vino_lib::Result* getLocationResult(int idx) const override; /** * @brief Show the observed detection result either through image window or ROS topic. */ - const void observeOutput(const std::shared_ptr & output); + void observeOutput(const std::shared_ptr& output); /** * @brief Get the name of the Inference instance. * @return The name of the Inference instance. */ const std::string getName() const override; + const std::vector getFilteredROIs(const std::string filter_conditions) const override; private: std::shared_ptr valid_model_; @@ -126,6 +131,13 @@ class ObjectSegmentation : public BaseInference int width_ = 0; int height_ = 0; double show_output_thresh_ = 0; + + std::vector colors_ = { { 128, 64, 128 }, { 232, 35, 244 }, { 70, 70, 70 }, { 156, 102, 102 }, + { 153, 153, 190 }, { 153, 153, 153 }, { 30, 170, 250 }, { 0, 220, 220 }, + { 35, 142, 107 }, { 152, 251, 152 }, { 180, 130, 70 }, { 60, 20, 220 }, + { 0, 0, 255 }, { 142, 0, 0 }, { 70, 0, 0 }, { 100, 60, 0 }, + { 90, 0, 0 }, { 230, 0, 0 }, { 32, 11, 119 }, { 0, 74, 111 }, + { 81, 0, 81 } }; }; } // namespace dynamic_vino_lib #endif // DYNAMIC_VINO_LIB__INFERENCES__OBJECT_SEGMENTATION_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/person_attribs_detection.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/person_attribs_detection.h new file mode 100644 index 00000000..ae170a57 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/person_attribs_detection.h @@ -0,0 +1,132 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for PersonAttribsDetection Class + * @file person_attribs_detection.hpp + */ +#ifndef DYNAMIC_VINO_LIB__INFERENCES__PERSON_ATTRIBS_DETECTION_H_ +#define DYNAMIC_VINO_LIB__INFERENCES__PERSON_ATTRIBS_DETECTION_H_ +#include +#include +#include +#include "dynamic_vino_lib/models/person_attribs_detection_model.h" +#include "dynamic_vino_lib/engines/engine.h" +#include "dynamic_vino_lib/inferences/base_inference.h" +#include "inference_engine.hpp" +#include "opencv2/opencv.hpp" +// namespace +namespace dynamic_vino_lib +{ +/** + * @class PersonAttribsDetectionResult + * @brief Class for storing and processing person attributes detection result. + */ +class PersonAttribsDetectionResult : public Result +{ +public: + friend class PersonAttribsDetection; + explicit PersonAttribsDetectionResult(const cv::Rect& location); + std::string getAttributes() const + { + return attributes_; + } + bool getMaleProbability() const + { + return male_probability_; + } + cv::Point2f getTopLocation() const + { + return top_point_; + } + cv::Point2f getBottomLocation() const + { + return bottom_point_; + } + +private: + float male_probability_; + float attributes_probability_[8]; + cv::Point2f top_point_; + cv::Point2f bottom_point_; + std::string attributes_ = ""; +}; +/** + * @class PersonAttribsDetection + * @brief Class to load person attributes detection model and perform person attributes detection. + */ +class PersonAttribsDetection : public BaseInference +{ +public: + using Result = dynamic_vino_lib::PersonAttribsDetectionResult; + explicit PersonAttribsDetection(double); + ~PersonAttribsDetection() override; + /** + * @brief Load the person attributes detection model. + */ + void loadNetwork(std::shared_ptr); + /** + * @brief Enqueue a frame to this class. + * The frame will be buffered but not infered yet. + * @param[in] frame The frame to be enqueued. + * @param[in] input_frame_loc The location of the enqueued frame with respect + * to the frame generated by the input device. + * @return Whether this operation is successful. + */ + bool enqueue(const cv::Mat&, const cv::Rect&) override; + /** + * @brief Start inference for all buffered frames. + * @return Whether this operation is successful. + */ + bool submitRequest() override; + /** + * @brief This function will fetch the results of the previous inference and + * stores the results in a result buffer array. All buffered frames will be + * cleared. + * @return Whether the Inference object fetches a result this time + */ + bool fetchResults() override; + /** + * @brief Get the length of the buffer result array. + * @return The length of the buffer result array. + */ + int getResultsLength() const override; + /** + * @brief Get the location of result with respect + * to the frame generated by the input device. + * @param[in] idx The index of the result. + */ + const dynamic_vino_lib::Result* getLocationResult(int idx) const override; + /** + * @brief Show the observed detection result either through image window + or ROS topic. + */ + void observeOutput(const std::shared_ptr& output); + /** + * @brief Get the name of the Inference instance. + * @return The name of the Inference instance. + */ + const std::string getName() const override; + const std::vector getFilteredROIs(const std::string filter_conditions) const override; + +private: + std::shared_ptr valid_model_; + std::vector results_; + double attribs_confidence_; + const std::vector net_attributes_ = { "is male", "has_bag", "has_backpack", + "has hat", "has longsleeves", "has longpants", + "has longhair", "has coat_jacket" }; +}; +} // namespace dynamic_vino_lib +#endif // DYNAMIC_VINO_LIB__INFERENCES__PERSON_ATTRIBS_DETECTION_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/person_reidentification.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/person_reidentification.h index 72b9b7a0..1caa9990 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/person_reidentification.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/person_reidentification.h @@ -24,6 +24,7 @@ #include "dynamic_vino_lib/models/person_reidentification_model.h" #include "dynamic_vino_lib/engines/engine.h" #include "dynamic_vino_lib/inferences/base_inference.h" +#include "dynamic_vino_lib/inferences/base_reidentification.h" #include "inference_engine.hpp" #include "opencv2/opencv.hpp" // namespace @@ -37,8 +38,11 @@ class PersonReidentificationResult : public Result { public: friend class PersonReidentification; - explicit PersonReidentificationResult(const cv::Rect & location); - std::string getPersonID() const {return person_id_;} + explicit PersonReidentificationResult(const cv::Rect& location); + std::string getPersonID() const + { + return person_id_; + } private: std::string person_id_ = "No.#"; @@ -65,7 +69,7 @@ class PersonReidentification : public BaseInference * to the frame generated by the input device. * @return Whether this operation is successful. */ - bool enqueue(const cv::Mat &, const cv::Rect &) override; + bool enqueue(const cv::Mat&, const cv::Rect&) override; /** * @brief Start inference for all buffered frames. * @return Whether this operation is successful. @@ -82,41 +86,29 @@ class PersonReidentification : public BaseInference * @brief Get the length of the buffer result array. * @return The length of the buffer result array. */ - const int getResultsLength() const override; + int getResultsLength() const override; /** * @brief Get the location of result with respect * to the frame generated by the input device. * @param[in] idx The index of the result. */ - const dynamic_vino_lib::Result * getLocationResult(int idx) const override; + const dynamic_vino_lib::Result* getLocationResult(int idx) const override; /** * @brief Show the observed detection result either through image window or ROS topic. */ - const void observeOutput(const std::shared_ptr & output); + void observeOutput(const std::shared_ptr& output); /** * @brief Get the name of the Inference instance. * @return The name of the Inference instance. */ const std::string getName() const override; - /** - * @brief Calculate the similarity between a new detected person with - * the already recorded persons. - * @return The similarity value. - */ - float calcSimilarity(const std::vector &, const std::vector &); - /** - * @brief Try to find the matched person from the recorded persons, if there are not, - * record it in the recorded persons. - * @return The id of the matched person (or the new person). - */ - std::string findMatchPerson(const std::vector &); + const std::vector getFilteredROIs(const std::string filter_conditions) const override; private: std::shared_ptr valid_model_; std::vector results_; - std::vector> recorded_persons_; - double match_thresh_ = 0; + std::shared_ptr person_tracker_; }; } // namespace dynamic_vino_lib #endif // DYNAMIC_VINO_LIB__INFERENCES__PERSON_REIDENTIFICATION_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inferences/vehicle_attribs_detection.h b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/vehicle_attribs_detection.h new file mode 100644 index 00000000..e32812b4 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inferences/vehicle_attribs_detection.h @@ -0,0 +1,119 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for VehicleAttribsDetection Class + * @file vehicle_attribs_detection.hpp + */ +#ifndef DYNAMIC_VINO_LIB__INFERENCES__VEHICLE_ATTRIBS_DETECTION_HPP_ +#define DYNAMIC_VINO_LIB__INFERENCES__VEHICLE_ATTRIBS_DETECTION_HPP_ +#include +#include +#include +#include "dynamic_vino_lib/models/vehicle_attribs_detection_model.h" +#include "dynamic_vino_lib/engines/engine.h" +#include "dynamic_vino_lib/inferences/base_inference.h" +#include "inference_engine.hpp" +#include "opencv2/opencv.hpp" +// namespace +namespace dynamic_vino_lib +{ +/** + * @class VehicleAttribsDetectionResult + * @brief Class for storing and processing vehicle attributes detection result. + */ +class VehicleAttribsDetectionResult : public Result +{ +public: + friend class VehicleAttribsDetection; + explicit VehicleAttribsDetectionResult(const cv::Rect& location); + std::string getColor() const + { + return color_; + } + std::string getType() const + { + return type_; + } + +private: + std::string color_ = ""; + std::string type_ = ""; +}; +/** + * @class VehicleAttribsDetection + * @brief Class to load vehicle attributes detection model and perform detection. + */ +class VehicleAttribsDetection : public BaseInference +{ +public: + using Result = dynamic_vino_lib::VehicleAttribsDetectionResult; + VehicleAttribsDetection(); + ~VehicleAttribsDetection() override; + /** + * @brief Load the vehicle attributes detection model. + */ + void loadNetwork(std::shared_ptr); + /** + * @brief Enqueue a frame to this class. + * The frame will be buffered but not infered yet. + * @param[in] frame The frame to be enqueued. + * @param[in] input_frame_loc The location of the enqueued frame with respect + * to the frame generated by the input device. + * @return Whether this operation is successful. + */ + bool enqueue(const cv::Mat&, const cv::Rect&) override; + /** + * @brief Start inference for all buffered frames. + * @return Whether this operation is successful. + */ + bool submitRequest() override; + /** + * @brief This function will fetch the results of the previous inference and + * stores the results in a result buffer array. All buffered frames will be + * cleared. + * @return Whether the Inference object fetches a result this time + */ + bool fetchResults() override; + /** + * @brief Get the length of the buffer result array. + * @return The length of the buffer result array. + */ + int getResultsLength() const override; + /** + * @brief Get the location of result with respect + * to the frame generated by the input device. + * @param[in] idx The index of the result. + */ + const dynamic_vino_lib::Result* getLocationResult(int idx) const override; + /** + * @brief Show the observed detection result either through image window + or ROS topic. + */ + void observeOutput(const std::shared_ptr& output); + /** + * @brief Get the name of the Inference instance. + * @return The name of the Inference instance. + */ + const std::string getName() const override; + const std::vector getFilteredROIs(const std::string filter_conditions) const override; + +private: + std::shared_ptr valid_model_; + std::vector results_; + const std::vector types_ = { "car", "van", "truck", "bus" }; + const std::vector colors_ = { "white", "gray", "yellow", "red", "green", "blue", "black" }; +}; +} // namespace dynamic_vino_lib +#endif // DYNAMIC_VINO_LIB__INFERENCES__VEHICLE_ATTRIBS_DETECTION_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/base_input.h b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/base_input.h index e300fdee..4d1cca7b 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/base_input.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/base_input.h @@ -31,9 +31,14 @@ */ namespace Input { -class BaseInputDevice : public Ros2Handler +struct Config { - public: + std::string path; +}; + +class BaseInputDevice : public RosHandler +{ +public: /** * @brief Initialize the input device, * for cameras, it will turn the camera on and get ready to read frames, @@ -41,12 +46,6 @@ class BaseInputDevice : public Ros2Handler * @return Whether the input device is successfully turned on. */ virtual bool initialize() = 0; - /** - * @brief (Only work for standard camera) - * Initialize camera by its index when multiple standard camera is connected. - * @return Whether the input device is successfully turned on. - */ - virtual bool initialize(int) = 0; /** * @brief Initialize the input device with given width and height. * @return Whether the input device is successfully turned on. @@ -57,7 +56,13 @@ class BaseInputDevice : public Ros2Handler * @return Whether the next frame is successfully read. */ virtual bool read(cv::Mat* frame) = 0; - virtual void config() = 0; //< TODO + virtual bool readService(cv::Mat* frame, std::string config_path) + { + return true; + } + virtual void config(const Config&) + { + } virtual ~BaseInputDevice() = default; /** * @brief Get the width of the frame read from input device. @@ -107,28 +112,11 @@ class BaseInputDevice : public Ros2Handler { is_init_ = is_init; } - /** - * @brief Set the frame_id of input device for ROSTopic outputs. - * @param[in] frame_id The frame_id of input device. - */ - inline void setFrameID(std::string frame_id) - { - frame_id_ = frame_id; - } - /** - * @brief Get the frame_id of input device. - * @return Frame_id of input device. - */ - inline std::string getFrameID() - { - return frame_id_; - } - private: - size_t width_ = 0; - size_t height_ = 0; +private: + size_t width_ = 0; // 0 means using the original size + size_t height_ = 0; // 0 means using the original size bool is_init_ = false; - std::string frame_id_; }; } // namespace Input #endif // DYNAMIC_VINO_LIB_INPUTS_BASE_INPUT_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/image_input.h b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/image_input.h index afdbd619..4711ddc9 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/image_input.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/image_input.h @@ -33,7 +33,7 @@ namespace Input */ class Image : public BaseInputDevice { - public: +public: explicit Image(const std::string&); /** * @brief Read an image file from the file path. @@ -41,15 +41,6 @@ class Image : public BaseInputDevice * @return Whether the input device is successfully turned on. */ bool initialize() override; - /** - * @brief (Only work for standard camera) - * No implementation for Image class. - * @return Whether the input device is successfully turned on. - */ - bool initialize(int t) override - { - return initialize(); - }; /** * @brief Initialize the input device with given width and height. * No implementation for Image class. @@ -58,15 +49,16 @@ class Image : public BaseInputDevice bool initialize(size_t width, size_t height) override { return initialize(); - }; + } /** * @brief Read next frame, and give the value to argument frame. * @return Whether the next frame is successfully read. */ bool read(cv::Mat* frame) override; - void config() override; - private: + void config(const Config&) override; + +private: cv::Mat image_; std::string file_; }; diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/image_topic.h b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/image_topic.h new file mode 100644 index 00000000..f06f4c5d --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/image_topic.h @@ -0,0 +1,57 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for ImageTopic class + * @file image_topic.h + */ + +#ifndef DYNAMIC_VINO_LIB__INPUTS__IMAGE_TOPIC_HPP_ +#define DYNAMIC_VINO_LIB__INPUTS__IMAGE_TOPIC_HPP_ + +#include "dynamic_vino_lib/inputs/base_input.h" +#include "dynamic_vino_lib/utils/mutex_counter.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace Input +{ +/** + * @class ImageTopic + * @brief Class for recieving a realsense camera topic as input. + */ +class ImageTopic : public BaseInputDevice +{ +public: + bool initialize() override; + bool initialize(size_t width, size_t height) override; + bool read(cv::Mat* frame) override; + +private: + ros::NodeHandle nh_; + image_transport::Subscriber sub_; + cv::Mat image_; + MutexCounter image_count_; + + // void cb(const sensor_msgs::ImageConstPtr& image_msg); + void cb(const sensor_msgs::Image::Ptr image_msg); +}; +} // namespace Input + +#endif // DYNAMIC_VINO_LIB__INPUTS__IMAGE_TOPIC_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/realsense_camera.h b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/realsense_camera.h index f4a87d0f..423dcb96 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/realsense_camera.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/realsense_camera.h @@ -34,22 +34,13 @@ namespace Input */ class RealSenseCamera : public BaseInputDevice { - public: +public: /** * @brief Initialize the input device, turn the * camera on and get ready to read frames. * @return Whether the input device is successfully turned on. */ bool initialize() override; - /** - * @brief (Only work for standard camera) - * Initialize camera by its index when multiple standard camera is connected. - * @return Whether the input device is successfully turned on. - */ - bool initialize(int t) override - { - return true; - }; /** * @brief Initialize the input device with given width and height. * @return Whether the input device is successfully turned on. @@ -60,12 +51,15 @@ class RealSenseCamera : public BaseInputDevice * @return Whether the next frame is successfully read. */ bool read(cv::Mat* frame) override; - void config() override; - private: +private: + void bypassFewFramesOnceInited(); + std::string getCameraSN(); + rs2::config cfg_; rs2::pipeline pipe_; bool first_read_ = true; + static int rscamera_count; }; } // namespace Input diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/realsense_camera_topic.h b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/realsense_camera_topic.h index 8d5fbf9d..98d5ea6e 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/realsense_camera_topic.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/realsense_camera_topic.h @@ -31,37 +31,18 @@ #include #include "dynamic_vino_lib/inputs/base_input.h" +#include "dynamic_vino_lib/inputs/image_topic.h" namespace Input { /** + * DEPRECATED! + * Using the new class ImageTopic to handle all image topics. * @class RealSenseCameraTopic * @brief Class for recieving a realsense camera topic as input. */ -class RealSenseCameraTopic : public BaseInputDevice -{ - public: - RealSenseCameraTopic(); - bool initialize() override; - bool initialize(int t) override - { - return true; - }; - bool initialize(size_t width, size_t height) override - { - return true; - }; - bool read(cv::Mat* frame) override; - void config() override; - - private: - ros::NodeHandle nh_; - image_transport::Subscriber sub_; - cv::Mat image; - cv::Mat last_image; +typedef ImageTopic RealSenseCameraTopic; - void cb(const sensor_msgs::ImageConstPtr& image_msg); -}; } // namespace Input #endif // DYNAMIC_VINO_LIB_INPUTS_REALSENSE_CAMERA_TOPIC_H_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/ros_handler.h b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/ros_handler.h index 4003b699..1fb5a9f2 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/ros_handler.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/ros_handler.h @@ -26,21 +26,40 @@ namespace Input { - -class Ros2Handler +class RosHandler { - public: +public: void setHandler(const std::shared_ptr& node) { node_ = node; } + std::shared_ptr getHandler() const { return node_; } - private: + // inline void setHeader(std::string frame_id) + // { + // header_.frame_id = frame_id; + // #if true //directly use RCLCPP api for time stamp generation. + // header_.stamp = ros::Time::now(); + // #else + // std::chrono::high_resolution_clock::time_point tp = std::chrono::high_resolution_clock::now(); + // int64 ns = tp.time_since_epoch().count(); + // header_.stamp.sec = ns / 1000000000; + // header_.stamp.nanosec = ns % 1000000000; + // #endif + // } + + // inline void setHeader(std_msgs::Header header) + // { + // header_ = header; + // } + +private: std::shared_ptr node_; + // std_msgs::Header header_; }; } // namespace Input diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/standard_camera.h b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/standard_camera.h index 8928a741..265493be 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/standard_camera.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/standard_camera.h @@ -23,7 +23,13 @@ #define DYNAMIC_VINO_LIB_INPUTS_STANDARD_CAMERA_H #include + #include "dynamic_vino_lib/inputs/base_input.h" +#include +#include +#include +#include +#include namespace Input { @@ -33,7 +39,7 @@ namespace Input */ class StandardCamera : public BaseInputDevice { - public: +public: /** * @brief Initialize the input device, * for cameras, it will turn the camera on and get ready to read frames, @@ -41,12 +47,6 @@ class StandardCamera : public BaseInputDevice * @return Whether the input device is successfully turned on. */ bool initialize() override; - /** - * @brief (Only work for standard camera) - * Initialize camera by its index when multiple standard camera is connected. - * @return Whether the input device is successfully turned on. - */ - bool initialize(int t) override; /** * @brief Initialize the input device with given width and height. * @return Whether the input device is successfully turned on. @@ -57,10 +57,11 @@ class StandardCamera : public BaseInputDevice * @return Whether the next frame is successfully read. */ bool read(cv::Mat* frame) override; - void config() override; - private: +private: + int getCameraId(); cv::VideoCapture cap; + int camera_id_ = -1; }; } // namespace Input #endif // DYNAMIC_VINO_LIB_INPUTS_STANDARD_CAMERA_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/video_input.h b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/video_input.h index d1ae6f47..8bac0265 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/inputs/video_input.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/inputs/video_input.h @@ -33,7 +33,7 @@ namespace Input */ class Video : public BaseInputDevice { - public: +public: explicit Video(const std::string&); /** * @brief Read a video file from the file path. @@ -41,15 +41,6 @@ class Video : public BaseInputDevice * @return Whether the input device is successfully turned on. */ bool initialize() override; - /** - * @brief (Only work for standard camera) - * No implementation for Video class. - * @return Whether the input device is successfully turned on. - */ - bool initialize(int t) override - { - return initialize(); - }; /** * @brief Initialize the input device with given width and height. * No implementation for Video class. @@ -61,9 +52,8 @@ class Video : public BaseInputDevice * @return Whether the next frame is successfully read. */ bool read(cv::Mat* frame) override; - void config() override; - private: +private: cv::VideoCapture cap; std::string video_; }; diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/age_gender_detection_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/age_gender_detection_model.h index 586b0e86..1bd47be8 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/models/age_gender_detection_model.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/age_gender_detection_model.h @@ -33,23 +33,20 @@ namespace Models */ class AgeGenderDetectionModel : public BaseModel { - public: - AgeGenderDetectionModel(const std::string&, int, int, int); +public: + AgeGenderDetectionModel(const std::string& model_loc, int batch_size = 1); /** * @brief Get the input name. * @return Input name. */ - inline const std::string getInputName() const - { - return input_; - } + /** * @brief Get the age from the detection reuslt. * @return Detected age. */ inline const std::string getOutputAgeName() const { - return output_age_; + return getOutputName("age"); } /** * @brief Get the gender from the detection reuslt. @@ -57,22 +54,16 @@ class AgeGenderDetectionModel : public BaseModel */ inline const std::string getOutputGenderName() const { - return output_gender_; + return getOutputName("gender"); } /** * @brief Get the name of this detection model. * @return Name of the model. */ - const std::string getModelName() const override; - - protected: - void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr&) override; - void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + const std::string getModelCategory() const override; - private: - std::string input_; - std::string output_age_; - std::string output_gender_; +protected: + bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; }; } // namespace Models diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/attributes/base_attribute.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/attributes/base_attribute.h new file mode 100644 index 00000000..61326acb --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/attributes/base_attribute.h @@ -0,0 +1,203 @@ +// Copyright (c) 2018-2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for ModelAttribute class. + * @file base_attribute.hpp + */ + +#ifndef DYNAMIC_VINO_LIB__MODELS__ATTRIBUTES_BASE_ATTRIBUTE_HPP_ +#define DYNAMIC_VINO_LIB__MODELS__ATTRIBUTES_BASE_ATTRIBUTE_HPP_ + +#include +#include +#include +#include + +#include "inference_engine.hpp" +#include "dynamic_vino_lib/slog.h" + +namespace Models +{ +/** + * @class ModelAttribute + * @brief This class represents the network given by .xml and .bin file + */ +class ModelAttribute +{ +public: + using Ptr = std::shared_ptr; + struct ModelAttr + { + int max_proposal_count = 0; + int object_size = 0; + int input_height = 0; + int input_width = 0; + std::string model_name; + std::map input_names; + std::map output_names; + std::vector labels; + }; + + ModelAttribute(const std::string model_name) + { + attr_.model_name = model_name; + } + + inline bool isVerified() + { + return (attr_.max_proposal_count > 0 && attr_.object_size > 0 && attr_.input_height > 0 && attr_.input_width > 0 && + attr_.input_names.empty() && attr_.output_names.empty()); + } + inline void printAttribute() + { + slog::info << "----Attributes for Model " << attr_.model_name << "----" << slog::endl; + slog::info << "| model_name: " << attr_.model_name << slog::endl; + slog::info << "| max_proposal_count: " << attr_.max_proposal_count << slog::endl; + slog::info << "| object_size: " << attr_.object_size << slog::endl; + slog::info << "| input_height: " << attr_.input_height << slog::endl; + slog::info << "| input_width: " << attr_.input_width << slog::endl; + slog::info << "| input_names: " << slog::endl; + for (auto& item : attr_.input_names) + { + slog::info << "| " << item.first << "-->" << item.second << slog::endl; + } + slog::info << "| output_names: " << slog::endl; + for (auto& item : attr_.output_names) + { + slog::info << "| " << item.first << "-->" << item.second << slog::endl; + } + + if (attr_.max_proposal_count <= 0 || attr_.object_size <= 0 || attr_.input_height <= 0 || attr_.input_width <= 0 || + attr_.input_names.empty() || attr_.output_names.empty()) + { + slog::info << "--------" << slog::endl; + slog::warn << "Not all attributes are set correctly! not 0 or empty is allowed in" + << " the above list." << slog::endl; + } + slog::info << "--------------------------------" << slog::endl; + } + + virtual bool updateLayerProperty(const InferenceEngine::CNNNetReader::Ptr&) + { + return false; + } + + inline std::string getModelName() const + { + return attr_.model_name; + } + + inline void setModelName(std::string name) + { + attr_.model_name = name; + } + + inline std::string getInputName(std::string name = "input") const + { + // std::map::iterator it; + auto it = attr_.input_names.find(name); + if (it == attr_.input_names.end()) + { + slog::warn << "No input named: " << name << slog::endl; + return std::string(""); + } + + return it->second; + } + + inline std::string getOutputName(std::string name = "output") const + { + // std::map::iterator it; + auto it = attr_.output_names.find(name); + if (it == attr_.output_names.end()) + { + slog::warn << "No output named: " << name << slog::endl; + return std::string(""); + } + + return it->second; + } + + inline int getMaxProposalCount() const + { + return attr_.max_proposal_count; + } + + inline int getObjectSize() const + { + return attr_.object_size; + } + + inline void loadLabelsFromFile(const std::string file_path) + { + std::ifstream input_file(file_path); + std::copy(std::istream_iterator(input_file), std::istream_iterator(), + std::back_inserter(attr_.labels)); + } + + inline std::vector& getLabels() + { + return attr_.labels; + } + + inline void addInputInfo(std::string key, std::string value) + { + attr_.input_names[key] = value; + } + + inline void addOutputInfo(std::string key, std::string value) + { + attr_.output_names[key] = value; + } + + inline void setInputHeight(const int height) + { + attr_.input_height = height; + } + + inline void setInputWidth(const int width) + { + attr_.input_width = width; + } + + inline void setMaxProposalCount(const int max) + { + attr_.max_proposal_count = max; + } + + inline void setObjectSize(const int size) + { + attr_.object_size = size; + } + +protected: + ModelAttr attr_; +}; + +#if 0 // not used +class SSDModelAttr : public ModelAttribute +{ +public: + explicit SSDModelAttr(const std::string model_name = "SSDNet-like"); + + bool updateLayerProperty( + const InferenceEngine::CNNNetReader::Ptr &); + +}; +#endif + +} // namespace Models + +#endif // DYNAMIC_VINO_LIB__MODELS__ATTRIBUTES_BASE_ATTRIBUTE_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/base_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/base_model.h index 13b580d2..b49859fd 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/models/base_model.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/base_model.h @@ -22,95 +22,124 @@ #ifndef DYNAMIC_VINO_LIB_MODELS_BASE_MODEL_H #define DYNAMIC_VINO_LIB_MODELS_BASE_MODEL_H +#include + #include #include -#include +#include +#include +#include #include "inference_engine.hpp" +#include "dynamic_vino_lib/models/attributes/base_attribute.h" namespace Engines { class Engine; } +namespace dynamic_vino_lib +{ +class ObjectDetectionResult; +} + namespace Models { /** * @class BaseModel * @brief This class represents the network given by .xml and .bin file */ -class BaseModel +class BaseModel : public ModelAttribute { +public: using Ptr = std::shared_ptr; - - public: /** - * @brief Initialize the class with given .xml, .bin and .labels file. It will - * also check whether the number of input and output are fit. - * @param[in] model_loc The location of model' s .xml file - * (model' s bin file should be the same as .xml file except for extension) - * @param[in] input_num The number of input the network should have. - * @param[in] output_num The number of output the network should have. - * @param[in] batch_size The number of batch size the network should have. - * @return Whether the input device is successfully turned on. - */ - BaseModel(const std::string& model_loc, int input_num, int output_num, - int batch_size); + * @brief Initialize the class with given .xml, .bin and .labels file. It will + * also check whether the number of input and output are fit. + * @param[in] model_loc The location of model' s .xml file + * (model' s bin file should be the same as .xml file except for extension) + * @param[in] input_num The number of input the network should have. + * @param[in] output_num The number of output the network should have. + * @param[in] batch_size The number of batch size (default: 1) the network should have. + * @return Whether the input device is successfully turned on. + */ + BaseModel(const std::string& model_loc, int batch_size = 1); + /** - * @brief Get the label vector. - * @return The label vector. - */ - inline std::vector& getLabels() + * @brief Get the maximum batch size of the model. + * @return The maximum batch size of the model. + */ + inline int getMaxBatchSize() const { - return labels_; + return max_batch_size_; } - /** - * @brief Get the maximum batch size of the model. - * @return The maximum batch size of the model. - */ - inline const int getMaxBatchSize() const + inline void setMaxBatchSize(int max_batch_size) { - return max_batch_size_; + max_batch_size_ = max_batch_size; + } + + virtual bool enqueue(const std::shared_ptr& engine, const cv::Mat& frame, + const cv::Rect& input_frame_loc) + { + return true; } /** - * @brief Initialize the model. During the process the class will check - * the network input, output size, check layer property and - * set layer property. - */ + * @brief Initialize the model. During the process the class will check + * the network input, output size, check layer property and + * set layer property. + */ void modelInit(); /** * @brief Get the name of the model. * @return The name of the model. */ - virtual const std::string getModelName() const = 0; + virtual const std::string getModelCategory() const = 0; + inline ModelAttr getAttribute() + { + return attr_; + } - protected: + inline InferenceEngine::CNNNetReader::Ptr getNetReader() const + { + return net_reader_; + } + +protected: /** - * @brief Check whether the layer property - * (output layer name, output layer type, etc.) is right - * @param[in] network_reader The reader of the network to be checked. + * New infterface to check and update Layer Property + * @brief Set the layer property (layer layout, layer precision, etc.). + * @param[in] network_reader The reader of the network to be set. */ - virtual void checkLayerProperty( - const InferenceEngine::CNNNetReader::Ptr& network_reader) = 0; - virtual void - /** - * @brief Set the layer property (layer layout, layer precision, etc.). - * @param[in] network_reader The reader of the network to be set. - */ - setLayerProperty(InferenceEngine::CNNNetReader::Ptr network_reader) = 0; - - private: - friend class Engines::Engine; - - void checkNetworkSize(unsigned int, unsigned int, - InferenceEngine::CNNNetReader::Ptr); + virtual bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr network_reader) = 0; + InferenceEngine::CNNNetReader::Ptr net_reader_; - std::vector labels_; - int input_num_; - int output_num_; - std::string model_loc_; + void setFrameSize(const int& w, const int& h) + { + frame_size_.width = w; + frame_size_.height = h; + } + cv::Size getFrameSize() + { + return frame_size_; + } + +private: int max_batch_size_; + std::string model_loc_; + cv::Size frame_size_; }; + +class ObjectDetectionModel : public BaseModel +{ +public: + ObjectDetectionModel(const std::string& model_loc, int batch_size = 1); + virtual bool fetchResults(const std::shared_ptr& engine, + std::vector& result, + const float& confidence_thresh = 0.3, const bool& enable_roi_constraint = false) = 0; + virtual bool matToBlob(const cv::Mat& orig_image, const cv::Rect&, float scale_factor, int batch_index, + const std::shared_ptr& engine) = 0; +}; + } // namespace Models -#endif // DYNAMIC_VINO_LIB_MODELS_BASE_MODEL_H +#endif // DYNAMIC_VINO_LIB__MODELS__BASE_MODEL_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/emotion_detection_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/emotion_detection_model.h index 65a1e490..1a64a28e 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/models/emotion_detection_model.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/emotion_detection_model.h @@ -33,29 +33,18 @@ namespace Models */ class EmotionDetectionModel : public BaseModel { - public: - EmotionDetectionModel(const std::string&, int, int, int); - inline const std::string getInputName() - { - return input_; - } - inline const std::string getOutputName() - { - return output_; - } +public: + EmotionDetectionModel(const std::string& model_loc, int batch_size = 1); + /** * @brief Get the name of this detection model. * @return Name of the model. */ - const std::string getModelName() const override; - - protected: - void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr&) override; - void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + const std::string getModelCategory() const override; + bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; - private: - std::string input_; - std::string output_; +private: + bool verifyOutputLayer(const InferenceEngine::DataPtr& ptr); }; } // namespace Models diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/face_detection_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/face_detection_model.h index 778c1db3..d384432b 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/models/face_detection_model.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/face_detection_model.h @@ -31,41 +31,16 @@ namespace Models * @class FaceDetectionModel * @brief This class generates the face detection model. */ -class FaceDetectionModel : public BaseModel +class FaceDetectionModel : public ObjectDetectionModel { - public: - FaceDetectionModel(const std::string&, int, int, int); - inline const int getMaxProposalCount() - { - return max_proposal_count_; - } - inline const int getObjectSize() - { - return object_size_; - } - inline const std::string getInputName() - { - return input_; - } - inline const std::string getOutputName() - { - return output_; - } +public: + FaceDetectionModel(const std::string& model_loc, int batch_size = 1); + // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; /** * @brief Get the name of this detection model. * @return Name of the model. */ - const std::string getModelName() const override; - - protected: - void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr&) override; - void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; - - private: - int max_proposal_count_; - int object_size_; - std::string input_; - std::string output_; + const std::string getModelCategory() const override; }; } // namespace Models diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/face_reidentification_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/face_reidentification_model.h new file mode 100644 index 00000000..b1d895a1 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/face_reidentification_model.h @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for FaceReidentificationModel Class + * @file person_reidentification_model.h + */ +#ifndef DYNAMIC_VINO_LIB__MODELS__FACE_REIDENTIFICATION_MODEL_H_ +#define DYNAMIC_VINO_LIB__MODELS__FACE_REIDENTIFICATION_MODEL_H_ +#include +#include "dynamic_vino_lib/models/base_model.h" +namespace Models +{ +/** + * @class FaceReidentificationModel + * @brief This class generates the face reidentification model. + */ +class FaceReidentificationModel : public BaseModel +{ +public: + FaceReidentificationModel(const std::string& model_loc, int batch_size = 1); + inline const std::string getInputName() + { + return input_; + } + inline const std::string getOutputName() + { + return output_; + } + /** + * @brief Get the name of this detection model. + * @return Name of the model. + */ + const std::string getModelCategory() const override; + +protected: + std::string input_; + std::string output_; +}; +} // namespace Models +#endif // DYNAMIC_VINO_LIB__MODELS__FACE_REIDENTIFICATION_MODEL_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/head_pose_detection_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/head_pose_detection_model.h index db3eb498..4297603d 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/models/head_pose_detection_model.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/head_pose_detection_model.h @@ -33,12 +33,9 @@ namespace Models */ class HeadPoseDetectionModel : public BaseModel { - public: - HeadPoseDetectionModel(const std::string&, int, int, int); - inline const std::string getInputName() const - { - return input_; - } +public: + HeadPoseDetectionModel(const std::string& model_loc, int batch_size = 1); + /** * @brief Get the output angle roll. * @return Roll value. @@ -67,14 +64,10 @@ class HeadPoseDetectionModel : public BaseModel * @brief Get the name of this detection model. * @return Name of the model. */ - const std::string getModelName() const override; - - protected: - void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr&) override; - void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + const std::string getModelCategory() const override; + bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; - private: - std::string input_; +private: std::string output_angle_r_ = "angle_r_fc"; std::string output_angle_p_ = "angle_p_fc"; std::string output_angle_y_ = "angle_y_fc"; diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/landmarks_detection_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/landmarks_detection_model.h new file mode 100644 index 00000000..24b72a26 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/landmarks_detection_model.h @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for LandmarksDetectionModel Class + * @file landmarks_detection_model.h + */ +#ifndef DYNAMIC_VINO_LIB__MODELS__LANDMARKS_DETECTION_MODEL_HPP_ +#define DYNAMIC_VINO_LIB__MODELS__LANDMARKS_DETECTION_MODEL_HPP_ +#include +#include "dynamic_vino_lib/models/base_model.h" +namespace Models +{ +/** + * @class LandmarksDetectionModel + * @brief This class generates the landmarks detection model. + */ +class LandmarksDetectionModel : public BaseModel +{ +public: + LandmarksDetectionModel(const std::string& model_loc, int batch_size = 1); + inline const std::string getInputName() + { + return input_; + } + inline const std::string getOutputName() + { + return output_; + } + /** + * @brief Get the name of this detection model. + * @return Name of the model. + */ + const std::string getModelCategory() const override; + +protected: + std::string input_; + std::string output_; +}; +} // namespace Models +#endif // DYNAMIC_VINO_LIB__MODELS__LANDMARKS_DETECTION_MODEL_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/license_plate_detection_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/license_plate_detection_model.h new file mode 100644 index 00000000..2eb0b782 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/license_plate_detection_model.h @@ -0,0 +1,64 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for LicensePlateDetectionModel Class + * @file vehicle_attribs_detection_model.h + */ +#ifndef DYNAMIC_VINO_LIB__MODELS__LICENSE_PLATE_DETECTION_MODEL_HPP_ +#define DYNAMIC_VINO_LIB__MODELS__LICENSE_PLATE_DETECTION_MODEL_HPP_ +#include +#include "dynamic_vino_lib/models/base_model.h" +namespace Models +{ +/** + * @class LicensePlateDetectionModel + * @brief This class generates the license plate detection model. + */ +class LicensePlateDetectionModel : public BaseModel +{ +public: + LicensePlateDetectionModel(const std::string& model_loc, int batch_size = 1); + inline const std::string getInputName() + { + return input_; + } + inline const std::string getSeqInputName() + { + return seq_input_; + } + inline const std::string getOutputName() + { + return output_; + } + inline int getMaxSequenceSize() const + { + return max_sequence_size_; + } + /** + * @brief Get the name of this detection model. + * @return Name of the model. + */ + const std::string getModelCategory() const override; + +protected: + bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + // up to 88 items per license plate, ended with "-1" + const int max_sequence_size_ = 88; + std::string input_; + std::string seq_input_; + std::string output_; +}; +} // namespace Models +#endif // DYNAMIC_VINO_LIB__MODELS__LICENSE_PLATE_DETECTION_MODEL_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/object_detection_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/object_detection_model.h deleted file mode 100644 index b0fe6c9f..00000000 --- a/dynamic_vino_lib/include/dynamic_vino_lib/models/object_detection_model.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2018 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * @brief A header file with declaration for ObjectDetectionModel Class - * @file face_detection_model.h - */ -#ifndef DYNAMIC_VINO_LIB_MODELS_OBJECT_DETECTION_MODEL_H -#define DYNAMIC_VINO_LIB_MODELS_OBJECT_DETECTION_MODEL_H -#include -#include "dynamic_vino_lib/models/base_model.h" -namespace Models { -/** - * @class ObjectDetectionModel - * @brief This class generates the face detection model. - */ -class ObjectDetectionModel : public BaseModel { - public: - ObjectDetectionModel(const std::string&, int, int, int); - inline const int getMaxProposalCount() { return max_proposal_count_; } - inline const int getObjectSize() { return object_size_; } - inline const std::string getInputName() { return input_; } - inline const std::string getOutputName() { return output_; } - /** - * @brief Get the name of this detection model. - * @return Name of the model. - */ - const std::string getModelName() const override; - protected: - void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr&) override; - void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; - - int max_proposal_count_; - int object_size_; - std::string input_; - std::string output_; -}; -} // namespace Models -#endif // DYNAMIC_VINO_LIB_MODELS_OBJECT_DETECTION_MODEL_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/object_detection_ssd_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/object_detection_ssd_model.h new file mode 100644 index 00000000..59d1c971 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/object_detection_ssd_model.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2018 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @brief A header file with declaration for ObjectDetectionModel Class + * @file face_detection_model.h + */ +#ifndef DYNAMIC_VINO_LIB_MODELS_OBJECT_DETECTION_SSD_MODEL_H +#define DYNAMIC_VINO_LIB_MODELS_OBJECT_DETECTION_SSD_MODEL_H +#include +#include "dynamic_vino_lib/models/base_model.h" +namespace Models +{ +/** + * @class ObjectDetectionModel + * @brief This class generates the face detection model. + */ +class ObjectDetectionSSDModel : public ObjectDetectionModel +{ + using Result = dynamic_vino_lib::ObjectDetectionResult; + +public: + ObjectDetectionSSDModel(const std::string& model_loc, int batch_size = 1); + + bool fetchResults(const std::shared_ptr& engine, + std::vector& results, const float& confidence_thresh = 0.3, + const bool& enable_roi_constraint = false) override; + + bool enqueue(const std::shared_ptr& engine, const cv::Mat& frame, + const cv::Rect& input_frame_loc) override; + + bool matToBlob(const cv::Mat& orig_image, const cv::Rect&, float scale_factor, int batch_index, + const std::shared_ptr& engine) override; + + /** + * @brief Get the name of this detection model. + * @return Name of the model. + */ + const std::string getModelCategory() const override; + + bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; +}; +} // namespace Models +#endif // DYNAMIC_VINO_LIB_MODELS_OBJECT_DETECTION_SSD_MODEL_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/object_detection_yolov2voc_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/object_detection_yolov2voc_model.h new file mode 100644 index 00000000..ff886514 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/object_detection_yolov2voc_model.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2018 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @brief A header file with declaration for ObjectDetectionModel Class + * @file face_detection_model.h + */ +#ifndef DYNAMIC_VINO_LIB_MODELS_OBJECT_DETECTION_YOLOV2VOC_MODEL_H +#define DYNAMIC_VINO_LIB_MODELS_OBJECT_DETECTION_YOLOV2VOC_MODEL_H +#include +#include "dynamic_vino_lib/models/base_model.h" +namespace Models +{ +/** + * @class ObjectDetectionModel + * @brief This class generates the face detection model. + */ +class ObjectDetectionYolov2Model : public ObjectDetectionModel +{ + using Result = dynamic_vino_lib::ObjectDetectionResult; + +public: + ObjectDetectionYolov2Model(const std::string& model_loc, int batch_size = 1); + + bool fetchResults(const std::shared_ptr& engine, + std::vector& results, const float& confidence_thresh = 0.3, + const bool& enable_roi_constraint = false) override; + + bool enqueue(const std::shared_ptr& engine, const cv::Mat& frame, + const cv::Rect& input_frame_loc) override; + + bool matToBlob(const cv::Mat& orig_image, const cv::Rect&, float scale_factor, int batch_index, + const std::shared_ptr& engine) override; + + /** + * @brief Get the name of this detection model. + * @return Name of the model. + */ + const std::string getModelCategory() const override; + bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + +protected: + int getEntryIndex(int side, int lcoords, int lclasses, int location, int entry); + InferenceEngine::InputInfo::Ptr input_info_ = nullptr; +}; +} // namespace Models +#endif // DYNAMIC_VINO_LIB_MODELS_OBJECT_DETECTION_YOLOV2VOC_MODEL_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/object_segmentation_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/object_segmentation_model.h index 38418824..e13a8ce9 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/models/object_segmentation_model.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/object_segmentation_model.h @@ -28,45 +28,32 @@ namespace Models class ObjectSegmentationModel : public BaseModel { public: - ObjectSegmentationModel(const std::string &, int, int, int); - inline const int getMaxProposalCount() + ObjectSegmentationModel(const std::string& model_loc, int batch_size = 1); + inline int getMaxProposalCount() const { return max_proposal_count_; } - inline const int getObjectSize() + inline int getObjectSize() const { return object_size_; } - inline const std::string getInputName() - { - return input_; - } - inline const std::string getDetectionOutputName() - { - return detection_output_; - } - inline const std::string getMaskOutputName() - { - return mask_output_; - } + + bool enqueue(const std::shared_ptr&, const cv::Mat&, const cv::Rect&) override; + + bool matToBlob(const cv::Mat&, const cv::Rect&, float, int, const std::shared_ptr&); /** * @brief Get the name of this segmentation model. * @return Name of the model. */ - const std::string getModelName() const override; - -protected: - void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; - void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; - //void checkNetworkSize(int input_size, int output_size, InferenceEngine::CNNNetReader::Ptr net_reader) override; + const std::string getModelCategory() const override; + bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; private: int max_proposal_count_; int object_size_; - std::string input_; - std::string mask_output_; - std::string detection_output_; + + InferenceEngine::InputsDataMap input_info_; }; } // namespace Models #endif // DYNAMIC_VINO_LIB__MODELS__OBJECT_SEGMENTATION_MODEL_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/person_attribs_detection_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/person_attribs_detection_model.h new file mode 100644 index 00000000..ebd90be8 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/person_attribs_detection_model.h @@ -0,0 +1,49 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for PersonAttribsDetectionModel Class + * @file person_attribs_detection_model.h + */ +#ifndef DYNAMIC_VINO_LIB__MODELS__PERSON_ATTRIBS_DETECTION_MODEL_H_ +#define DYNAMIC_VINO_LIB__MODELS__PERSON_ATTRIBS_DETECTION_MODEL_H_ +#include +#include "dynamic_vino_lib/models/base_model.h" +namespace Models +{ +/** + * @class PersonAttribsDetectionModel + * @brief This class generates the person attributes detection model. + */ +class PersonAttribsDetectionModel : public BaseModel +{ +public: + PersonAttribsDetectionModel(const std::string& model_loc, int batch_size = 1); + // inline const std::string getInputName() {return input_;} + // inline const std::string getOutputName() {return output_;} + /** + * @brief Get the name of this detection model. + * @return Name of the model. + */ + const std::string getModelCategory() const override; + +protected: + // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; + // void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + std::string input_; + std::string output_; +}; +} // namespace Models +#endif // DYNAMIC_VINO_LIB__MODELS__PERSON_ATTRIBS_DETECTION_MODEL_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/person_reidentification_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/person_reidentification_model.h index a9cff01f..b1f801f5 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/models/person_reidentification_model.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/person_reidentification_model.h @@ -29,18 +29,25 @@ namespace Models class PersonReidentificationModel : public BaseModel { public: - PersonReidentificationModel(const std::string &, int, int, int); - inline const std::string getInputName() {return input_;} - inline const std::string getOutputName() {return output_;} + PersonReidentificationModel(const std::string& model_loc, int batch_size = 1); + inline const std::string getInputName() + { + return input_; + } + inline const std::string getOutputName() + { + return output_; + } /** * @brief Get the name of this detection model. * @return Name of the model. */ - const std::string getModelName() const override; + const std::string getModelCategory() const override; protected: - void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; - void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; + // void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; std::string input_; std::string output_; }; diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/models/vehicle_attribs_detection_model.h b/dynamic_vino_lib/include/dynamic_vino_lib/models/vehicle_attribs_detection_model.h new file mode 100644 index 00000000..e716b5bb --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/models/vehicle_attribs_detection_model.h @@ -0,0 +1,60 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for VehicleAttribsDetectionModel Class + * @file vehicle_attribs_detection_model.h + */ +#ifndef DYNAMIC_VINO_LIB__MODELS__VEHICLE_ATTRIBS_DETECTION_MODEL_HPP_ +#define DYNAMIC_VINO_LIB__MODELS__VEHICLE_ATTRIBS_DETECTION_MODEL_HPP_ +#include +#include "dynamic_vino_lib/models/base_model.h" +namespace Models +{ +/** + * @class VehicleAttribsDetectionModel + * @brief This class generates the vehicle attributes detection model. + */ +class VehicleAttribsDetectionModel : public BaseModel +{ +public: + VehicleAttribsDetectionModel(const std::string& model_loc, int batch_size = 1); + inline const std::string getInputName() + { + return input_; + } + inline const std::string getColorOutputName() + { + return color_output_; + } + inline const std::string getTypeOutputName() + { + return type_output_; + } + /** + * @brief Get the name of this detection model. + * @return Name of the model. + */ + const std::string getModelCategory() const override; + +protected: + // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; + // void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + bool updateLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; + std::string input_; + std::string color_output_; + std::string type_output_; +}; +} // namespace Models +#endif // DYNAMIC_VINO_LIB__MODELS__VEHICLE_ATTRIBS_DETECTION_MODEL_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/outputs/base_output.h b/dynamic_vino_lib/include/dynamic_vino_lib/outputs/base_output.h index 00cafc1d..9a324ff3 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/outputs/base_output.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/outputs/base_output.h @@ -30,12 +30,24 @@ #include "dynamic_vino_lib/inferences/emotions_detection.h" #include "dynamic_vino_lib/inferences/face_detection.h" #include "dynamic_vino_lib/inferences/head_pose_detection.h" -#include "dynamic_vino_lib/inferences/object_detection.h" #include "dynamic_vino_lib/inferences/object_segmentation.h" #include "dynamic_vino_lib/inferences/person_reidentification.h" +#include "dynamic_vino_lib/inferences/landmarks_detection.h" +#include "dynamic_vino_lib/inferences/face_reidentification.h" +#include "dynamic_vino_lib/inferences/person_attribs_detection.h" +#include "dynamic_vino_lib/inferences/vehicle_attribs_detection.h" +#include "dynamic_vino_lib/inferences/license_plate_detection.h" #include "dynamic_vino_lib/services/frame_processing_server.h" #include "opencv2/opencv.hpp" +#include +#include +#include +#include +#include +#include +#include + class Pipeline; namespace Outputs { @@ -47,23 +59,53 @@ namespace Outputs */ class BaseOutput { - public: - // BaseOutput() = default; - BaseOutput() +public: + explicit BaseOutput() + { + } + explicit BaseOutput(std::string output_name) : output_name_(output_name) + { + } + /** + * @brief Generate output content according to the license plate detection result. + */ + virtual void accept(const std::vector&) + { + } + /** + * @brief Generate output content according to the vehicle attributes detection result. + */ + virtual void accept(const std::vector&) + { + } + /** + * @brief Generate output content according to the person atrribute detection result. + */ + virtual void accept(const std::vector&) + { + } + /** + * @brief Generate output content according to the face reidentification result. + */ + virtual void accept(const std::vector&) + { + } + /** + * @brief Generate output content according to the landmarks detection result. + */ + virtual void accept(const std::vector&) { - }; + } /** * @brief Generate output content according to the face detection result. */ - virtual void accept( - const std::vector&) + virtual void accept(const std::vector&) { } /** * @brief Generate output content according to the face detection result. */ - virtual void accept( - const std::vector&) + virtual void accept(const std::vector&) { } /** @@ -94,7 +136,7 @@ class BaseOutput /** * @brief Generate output content according to the person reidentification result. */ - virtual void accept(const std::vector &) + virtual void accept(const std::vector&) { } /** @@ -110,29 +152,40 @@ class BaseOutput virtual void handleOutput() = 0; void setPipeline(Pipeline* const pipeline); - virtual void setServiceResponse( - boost::shared_ptr response) {} - virtual void setServiceResponseForFace( - boost::shared_ptr response) {} - virtual void setServiceResponse( - boost::shared_ptr response) {} - virtual void setServiceResponse( - boost::shared_ptr response) {} - virtual void setServiceResponse( - boost::shared_ptr response) {} - virtual void setServiceResponse( - boost::shared_ptr response) {} - virtual void setServiceResponse( - boost::shared_ptr response) {} - virtual void setServiceResponse( - boost::shared_ptr response) {} + virtual void setServiceResponse(boost::shared_ptr response) + { + } + virtual void setServiceResponseForFace(boost::shared_ptr response) + { + } + virtual void setServiceResponse(boost::shared_ptr response) + { + } + virtual void setServiceResponse(boost::shared_ptr response) + { + } + virtual void setServiceResponse(boost::shared_ptr response) + { + } + virtual void setServiceResponse(boost::shared_ptr response) + { + } + virtual void setServiceResponse(boost::shared_ptr response) + { + } + virtual void setServiceResponse(boost::shared_ptr response) + { + } Pipeline* getPipeline() const; cv::Mat getFrame() const; - virtual void clearData() {} + virtual void clearData() + { + } - protected: +protected: cv::Mat frame_; Pipeline* pipeline_; + std::string output_name_; }; } // namespace Outputs #endif // DYNAMIC_VINO_LIB_OUTPUTS_BASE_OUTPUT_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/outputs/image_window_output.h b/dynamic_vino_lib/include/dynamic_vino_lib/outputs/image_window_output.h index 2bdca7c2..5e9faba7 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/outputs/image_window_output.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/outputs/image_window_output.h @@ -34,9 +34,8 @@ namespace Outputs */ class ImageWindowOutput : public BaseOutput { - public: - explicit ImageWindowOutput(const std::string& window_name, - int focal_length = 950); +public: + explicit ImageWindowOutput(const std::string& window_name, int focal_length = 950); /** * @brief Calculate the camera matrix of a frame for image * window output. @@ -52,13 +51,44 @@ class ImageWindowOutput : public BaseOutput * functions with image window. */ void handleOutput() override; + /** + * @brief Generate image window output content according to + * the license plate detection result. + * @param[in] A license plate detection result objetc. + */ + void accept(const std::vector&) override; + /** + * @brief Generate image window output content according to + * the vehicle attributes detection result. + * @param[in] A vehicle attributes detection result objetc. + */ + void accept(const std::vector&) override; + /** +*@brief Generate image window output content according to +*the landmarks detection result. +*@param[in] A landmarks detection result objetc. +*/ + void accept(const std::vector&) override; + /** + * @brief Generate image window output content according to + * the person attributes detection result. + * @param[in] A person attributes detection result objetc. + */ + void accept(const std::vector&) override; + /** + * @brief Generate image window output content according to + * the landmarks detetection result. + * the face reidentification result. + * @param[in] A face reidentification result objetc. + */ + void accept(const std::vector&) override; /** * @brief Generate image window output content according to * the face detection result. * @param[in] A face detection result objetc. */ - void accept( - const std::vector&) override; + + void accept(const std::vector&) override; /** * @brief Generate image window output content according to * the emotion detection result. @@ -89,21 +119,21 @@ class ImageWindowOutput : public BaseOutput * @param[in] An object segmentation result objetc. */ void accept(const std::vector&) override; - /** - * @brief Generate image window output content according to - * the person re-ID result. - * @param[in] An object segmentation result objetc. - */ - void accept(const std::vector &) override; - /** - * @brief Merge mask for image window ouput - * the object segmentation result. - * @param[in] An object segmentation result objetc. - */ - void mergeMask(const std::vector &); - private: + /** + * @brief Generate image window output content according to + * the person re-ID result. + * @param[in] An object segmentation result objetc. + */ + void accept(const std::vector&) override; + /** + * @brief Merge mask for image window ouput + * the object segmentation result. + * @param[in] An object segmentation result objetc. + */ + void mergeMask(const std::vector&); - unsigned findOutput(const cv::Rect &); +private: + unsigned findOutput(const cv::Rect&); void initOutputs(unsigned size); /** * @brief Calculate the axises of the coordinates for showing @@ -123,24 +153,26 @@ class ImageWindowOutput : public BaseOutput std::string desc; cv::Rect rect; cv::Scalar scalar; - cv::Point hp_cp; // for headpose, center point - cv::Point hp_x; // for headpose, end point of xAxis - cv::Point hp_y; // for headpose, end point of yAxis - cv::Point hp_zs; // for headpose, start point of zAxis - cv::Point hp_ze; // for headpose, end point of zAxis + cv::Point hp_cp; // for headpose, center point + cv::Point hp_x; // for headpose, end point of xAxis + cv::Point hp_y; // for headpose, end point of yAxis + cv::Point hp_zs; // for headpose, start point of zAxis + cv::Point hp_ze; // for headpose, end point of zAxis + cv::Point pa_top; // for person attributes, top position + cv::Point pa_bottom; // for person attributes, bottom position + std::vector landmarks; }; std::vector outputs_; const std::string window_name_; float focal_length_; cv::Mat camera_matrix_; - std::vector> colors_ = { - {128, 64, 128}, {232, 35, 244}, {70, 70, 70}, {156, 102, 102}, {153, 153, 190}, - {153, 153, 153}, {30, 170, 250}, {0, 220, 220}, {35, 142, 107}, {152, 251, 152}, - {180, 130, 70}, {60, 20, 220}, {0, 0, 255}, {142, 0, 0}, {70, 0, 0}, - {100, 60, 0}, {90, 0, 0}, {230, 0, 0}, {32, 11, 119}, {0, 74, 111}, - {81, 0, 81} - }; + std::vector> colors_ = { { 128, 64, 128 }, { 232, 35, 244 }, { 70, 70, 70 }, { 156, 102, 102 }, + { 153, 153, 190 }, { 153, 153, 153 }, { 30, 170, 250 }, { 0, 220, 220 }, + { 35, 142, 107 }, { 152, 251, 152 }, { 180, 130, 70 }, { 60, 20, 220 }, + { 0, 0, 255 }, { 142, 0, 0 }, { 70, 0, 0 }, { 100, 60, 0 }, + { 90, 0, 0 }, { 230, 0, 0 }, { 32, 11, 119 }, { 0, 74, 111 }, + { 81, 0, 81 } }; }; } // namespace Outputs #endif // DYNAMIC_VINO_LIB_OUTPUTS_IMAGE_WINDOW_OUTPUT_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/outputs/ros_service_output.h b/dynamic_vino_lib/include/dynamic_vino_lib/outputs/ros_service_output.h index 3ce30e74..cb0c0de7 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/outputs/ros_service_output.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/outputs/ros_service_output.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -38,11 +39,10 @@ #include #include #include -#include +#include #include #include - #include #include @@ -61,13 +61,17 @@ namespace Outputs class RosServiceOutput : public RosTopicOutput { public: - RosServiceOutput() {} + RosServiceOutput(std::string pipeline_name) : pipeline_name_(pipeline_name) + { + } /** * @brief Publish all the detected infomations generated by the accept * functions with ros topic. */ - void handleOutput() override {} + void handleOutput() override + { + } void clearData(); void setServiceResponse(boost::shared_ptr response); @@ -81,6 +85,7 @@ class RosServiceOutput : public RosTopicOutput private: const std::string service_name_; + const std::string pipeline_name_; }; } // namespace Outputs #endif // DYNAMIC_VINO_LIB__OUTPUTS__ROS_SERVICE_OUTPUT_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/outputs/ros_topic_output.h b/dynamic_vino_lib/include/dynamic_vino_lib/outputs/ros_topic_output.h index 583c9afd..077e2c84 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/outputs/ros_topic_output.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/outputs/ros_topic_output.h @@ -36,6 +36,14 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -53,8 +61,9 @@ namespace Outputs */ class RosTopicOutput : public BaseOutput { - public: - RosTopicOutput(); +public: + RosTopicOutput(){}; + RosTopicOutput(std::string pipeline_name); /** * @brief Calculate the camera matrix of a frame. * @param[in] A frame. @@ -65,25 +74,54 @@ class RosTopicOutput : public BaseOutput * functions with ros topic. */ void handleOutput() override; + /** + * @brief Generate image window output content according to + * the license plate detection result. + * @param[in] A license plate detection result objetc. + */ + void accept(const std::vector&) override; + /** + * @brief Generate image window output content according to + * the vehicle attributes detection result. + * @param[in] A vehicle attributes detection result objetc. + */ + void accept(const std::vector&) override; + /** + * @brief Generate ros topic infomation according to + * the person attributes detection result. + * @param[in] results a bundle of person attributes detection results. + */ + void accept(const std::vector&) override; + /** + * @brief Generate ros topic infomation according to + * the landmark detection result. + * @param[in] results a bundle of person landmarks detection result. + */ + void accept(const std::vector&) override; + /** + * @brief Generate ros topic infomation according to + * the face reidentification result. + * @param[in] results a bundle of face reidentification results. + */ + void accept(const std::vector&) override; /** * @brief Generate ros topic infomation according to * the person reidentification result. * @param[in] results a bundle of person reidentification results. */ - void accept(const std::vector &) override; + void accept(const std::vector&) override; /** * @brief Generate ros topic infomation according to * the object segmentation result. * @param[in] results a bundle of object segmentation results. */ - void accept(const std::vector &) override; + void accept(const std::vector&) override; /** * @brief Generate ros topic infomation according to * the face detection result. * @param[in] An face detection result objetc. */ - void accept( - const std::vector&) override; + void accept(const std::vector&) override; /** * @brief Generate ros topic infomation according to * the emotion detection result. @@ -95,7 +133,7 @@ class RosTopicOutput : public BaseOutput * the age gender detection result. * @param[in] An age gender detection result objetc. */ - void accept(const std::vector &) override; + void accept(const std::vector&) override; /**detected_objects_topic_ * @brief Generate ros topic infomation according to * the headpose detection result. @@ -109,28 +147,38 @@ class RosTopicOutput : public BaseOutput */ void accept(const std::vector&) override; - - private: +private: std_msgs::Header getHeader(); + const std::string pipeline_name_; const std::string topic_name_; cv::Mat frame_; ros::NodeHandle nh_; - protected: + +protected: ros::Publisher pub_face_; - std::shared_ptr faces_msg_ptr_; + std::shared_ptr faces_topic_; ros::Publisher pub_emotion_; - std::shared_ptr emotions_msg_ptr_; + std::shared_ptr emotions_topic_; ros::Publisher pub_age_gender_; - std::shared_ptr age_gender_msg_ptr_; + std::shared_ptr age_gender_topic_; ros::Publisher pub_headpose_; - std::shared_ptr headpose_msg_ptr_; + std::shared_ptr headpose_topic_; ros::Publisher pub_object_; - std::shared_ptr object_msg_ptr_; + std::shared_ptr detected_objects_topic_; ros::Publisher pub_person_reid_; std::shared_ptr person_reid_msg_ptr_; ros::Publisher pub_segmented_object_; - std::shared_ptr segmented_object_msg_ptr_; - + std::shared_ptr segmented_objects_topic_; + ros::Publisher pub_face_reid_; + std::shared_ptr face_reid_topic_; + ros::Publisher pub_person_attribs_; + std::shared_ptr person_attribs_topic_; + ros::Publisher pub_license_plate_; + std::shared_ptr license_plate_topic_; + ros::Publisher pub_vehicle_attribs_; + std::shared_ptr vehicle_attribs_topic_; + ros::Publisher pub_landmarks_; + std::shared_ptr landmarks_topic_; }; } // namespace Outputs #endif // DYNAMIC_VINO_LIB_OUTPUTS_ROS_TOPIC_OUTPUT_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/outputs/rviz_output.h b/dynamic_vino_lib/include/dynamic_vino_lib/outputs/rviz_output.h index 595e1864..3b62186e 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/outputs/rviz_output.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/outputs/rviz_output.h @@ -20,8 +20,6 @@ #ifndef DYNAMIC_VINO_LIB__OUTPUTS__RVIZ_OUTPUT_HPP_ #define DYNAMIC_VINO_LIB__OUTPUTS__RVIZ_OUTPUT_HPP_ - - #include #include #include @@ -38,47 +36,71 @@ namespace Outputs class RvizOutput : public BaseOutput { public: - RvizOutput(); + RvizOutput(std::string); /** * @brief Construct frame for rviz * @param[in] A frame. */ - void feedFrame(const cv::Mat &) override; + void feedFrame(const cv::Mat&) override; /** * @brief Show all the contents generated by the accept * functions with rviz. */ void handleOutput() override; + /** + * @brief Generate image window output content according to + * the license plate detection result. + * @param[in] A license plate detection result objetc. + */ + void accept(const std::vector&) override; + /** + * @brief Generate image window output content according to + * the vehicle attributes detection result. + * @param[in] A vehicle attributes detection result objetc. + */ + void accept(const std::vector&) override; + /** + * @brief Generate rviz output content according to + * the person attributes detection result. + * @param[in] A person attributes detection result objetc. + */ + void accept(const std::vector&) override; + /** + * @brief Generate rviz output content according to + * the face reidentification result. + * @param[in] A face reidentification result objetc. + */ + void accept(const std::vector&) override; /** * @brief Generate rviz output content according to * the face detection result. * @param[in] A face detection result objetc. */ - void accept(const std::vector &) override; + void accept(const std::vector&) override; /** * @brief Generate rviz output content according to * the object detection result. * @param[in] results A bundle of object detection results. */ - void accept(const std::vector &) override; + void accept(const std::vector&) override; /** * @brief Generate rviz output content according to * the object segmentation result. * @param[in] results A bundle of object segmentation results. */ - void accept(const std::vector &) override; + void accept(const std::vector&) override; /** * @brief Generate rviz output content according to * the age and gender detection result. * @param[in] A head pose detection result objetc. */ - void accept(const std::vector &) override; + void accept(const std::vector&) override; /** * @brief Generate rviz output content according to * the headpose detection result. * @param[in] An age gender detection result objetc. */ - void accept(const std::vector &) override; + void accept(const std::vector&) override; /** * @brief Generate rviz output content according to @@ -86,17 +108,17 @@ class RvizOutput : public BaseOutput * @param[in] An object segmentation result objetc. */ void accept(const std::vector&) override; - /** - * @brief Generate rviz output content according to - * the person re-ID result. - * @param[in] An object segmentation result objetc. - */ - void accept(const std::vector &) override; - /** - * @brief Merge mask for image window ouput - * the object segmentation result. - * @param[in] An object segmentation result objetc. - */ + /** + * @brief Generate rviz output content according to + * the person re-ID result. + * @param[in] An object segmentation result objetc. + */ + void accept(const std::vector&) override; + /** + * @brief Merge mask for image window ouput + * the object segmentation result. + * @param[in] An object segmentation result objetc. + */ private: std_msgs::Header getHeader(); @@ -107,4 +129,3 @@ class RvizOutput : public BaseOutput }; } // namespace Outputs #endif // DYNAMIC_VINO_LIB__OUTPUTS__RVIZ_OUTPUT_HPP_ - diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/pipeline.h b/dynamic_vino_lib/include/dynamic_vino_lib/pipeline.h index bf7394f2..f9597d6a 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/pipeline.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/pipeline.h @@ -43,7 +43,7 @@ */ class Pipeline { - public: +public: explicit Pipeline(const std::string& name = "pipeline"); /** * @brief Add input device to the pipeline. @@ -51,8 +51,7 @@ class Pipeline * @param[in] input_device the input device instance to be added. * @return whether the add operation is successful */ - bool add(const std::string& name, - std::shared_ptr input_device); + bool add(const std::string& name, std::shared_ptr input_device); /** * @brief Add inference network to the pipeline. * @param[in] parent name of the parent device or inference. @@ -68,8 +67,7 @@ class Pipeline * @param[in] inference the inference instance to be added. * @return whether the add operation is successful */ - bool add(const std::string & name, - std::shared_ptr inference); + bool add(const std::string& name, std::shared_ptr inference); /** * @brief Add output device to the pipeline. * @param[in] parent name of the parent inference. @@ -77,16 +75,14 @@ class Pipeline * @param[in] output the output instance to be added. * @return whether the add operation is successful */ - bool add(const std::string& parent, const std::string& name, - std::shared_ptr output); + bool add(const std::string& parent, const std::string& name, std::shared_ptr output); /** * @brief Add output device to the pipeline. * @param[in] name name of the current output device. * @param[in] output the output instance to be added. * @return whether the add operation is successful */ - bool add(const std::string& name, - std::shared_ptr output); + bool add(const std::string& name, std::shared_ptr output); /** * @brief Add inference network-output device edge to the pipeline. * @param[in] parent name of the parent inference. @@ -99,7 +95,7 @@ class Pipeline * @param[in] name of the instance. * @return the category order of this instance. */ - void addConnect(const std::string & parent, const std::string & name); + void addConnect(const std::string& parent, const std::string& name); /** * @brief Do the inference once. * Data flow from input device to inference network, then to output device. @@ -118,6 +114,14 @@ class Pipeline void setCallback(); void printPipeline(); + std::map> getOutputHandle() + { + return name_to_output_map_; + } + void setParams(PipelineParams pipeline_params) + { + params_ = std::make_shared(pipeline_params); + } const std::shared_ptr getParameters() { return params_; @@ -126,7 +130,10 @@ class Pipeline { return input_device_; } - + const std::multimap getPipelineDetail() + { + return next_; + } /** * @brief Get real time FPS (frames per second). */ @@ -134,12 +141,13 @@ class Pipeline { return fps_; } - std::map> getOutputHandle() + + std::string findFilterConditions(const std::string& input, const std::string& output) { - return name_to_output_map_; + return params_->findFilterConditions(input, output); } - private: +private: void initInferenceCounter(); void increaseInferenceCounter(); void decreaseInferenceCounter(); @@ -162,11 +170,8 @@ class Pipeline std::shared_ptr input_device_; std::string input_device_name_; std::multimap next_; - std::map> - name_to_detection_map_; - std::map> - name_to_output_map_; - + std::map> name_to_detection_map_; + std::map> name_to_output_map_; std::set output_names_; int width_ = 0; int height_ = 0; @@ -176,6 +181,8 @@ class Pipeline std::mutex counter_mutex_; std::condition_variable cv_; int fps_ = 0; + int frame_cnt_ = 0; + std::chrono::time_point t_start_; }; #endif // DYNAMIC_VINO_LIB_PIPELINE_H_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/pipeline_manager.h b/dynamic_vino_lib/include/dynamic_vino_lib/pipeline_manager.h index 3deb2aa9..d131c530 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/pipeline_manager.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/pipeline_manager.h @@ -31,81 +31,97 @@ #include #include "dynamic_vino_lib/pipeline.h" +#include "dynamic_vino_lib/engines/engine_manager.h" /** * @class PipelineManager * @brief This class manages the lifecycles of pipelines. */ -class PipelineManager { - public: +class PipelineManager +{ +public: /** * @brief Get the singleton instance of PipelineManager class. * The instance will be created when first call. * @return The reference of PipelineManager instance. */ - static PipelineManager& getInstance() { + static PipelineManager& getInstance() + { static PipelineManager manager_; return manager_; }; - std::shared_ptr createPipeline( - const Params::ParamManager::PipelineParams& params); + std::shared_ptr createPipeline(const Params::ParamManager::PipelineRawData& params); void removePipeline(const std::string& name); - PipelineManager& updatePipeline( - const std::string& name, - const Params::ParamManager::PipelineParams& params); - + PipelineManager& updatePipeline(const std::string& name, const Params::ParamManager::PipelineRawData& params); + void runAll(); void stopAll(); void joinAll(); - enum PipelineState { - PipelineState_ThreadNotCreated, - PipelineState_ThreadStopped, - PipelineState_ThreadRunning, - PipelineState_Error + enum PipelineState + { + PipelineState_ThreadNotCreated = 0, + PipelineState_ThreadStopped = 1, + PipelineState_ThreadRunning = 2, + PipelineState_ThreadPasued = 3, + PipelineState_Error = 4 }; - struct PipelineData { - Params::ParamManager::PipelineParams params; + + struct PipelineData + { + Params::ParamManager::PipelineRawData params; std::shared_ptr pipeline; std::vector> spin_nodes; std::shared_ptr thread; PipelineState state; }; - + void runService(); std::map getPipelines() { return pipelines_; } + std::map* getPipelinesPtr() + { + return &pipelines_; + } - - private: +private: PipelineManager(){}; PipelineManager(PipelineManager const&); void operator=(PipelineManager const&); void threadPipeline(const char* name); - std::map> - parseInputDevice(const Params::ParamManager::PipelineParams& params); - std::map> parseOutput( - const Params::ParamManager::PipelineParams& params); + std::map> parseInputDevice(const PipelineData& params); + std::map> parseOutput(const PipelineData& pdata); std::map> - parseInference(const Params::ParamManager::PipelineParams& params); - std::shared_ptr createFaceDetection( - const Params::ParamManager::InferenceParams& infer); - std::shared_ptr createAgeGenderRecognition( - const Params::ParamManager::InferenceParams& infer); - std::shared_ptr createEmotionRecognition( - const Params::ParamManager::InferenceParams& infer); - std::shared_ptr createHeadPoseEstimation( - const Params::ParamManager::InferenceParams& infer); - std::shared_ptr createObjectDetection( - const Params::ParamManager::InferenceParams& infer); - std::shared_ptr createObjectSegmentation( - const Params::ParamManager::InferenceParams& infer); - std::shared_ptr createPersonReidentification( - const Params::ParamManager::InferenceParams& infer); + parseInference(const Params::ParamManager::PipelineRawData& params); + std::shared_ptr + createFaceDetection(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createAgeGenderRecognition(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createEmotionRecognition(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createHeadPoseEstimation(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createObjectDetection(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createObjectSegmentation(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createPersonReidentification(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createFaceReidentification(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createPersonAttribsDetection(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createVehicleAttribsDetection(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createLicensePlateDetection(const Params::ParamManager::InferenceRawData& infer); + std::shared_ptr + createLandmarksDetection(const Params::ParamManager::InferenceRawData& infer); + std::map pipelines_; - std::map plugins_for_devices_; + Engines::EngineManager engine_manager_; }; -#endif // DYNAMIC_VINO_LIB__PIPELINE_MANAGER_HPP_ \ No newline at end of file +#endif // DYNAMIC_VINO_LIB__PIPELINE_MANAGER_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/pipeline_params.h b/dynamic_vino_lib/include/dynamic_vino_lib/pipeline_params.h index 4344a48c..2a8c4055 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/pipeline_params.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/pipeline_params.h @@ -35,11 +35,12 @@ #include "opencv2/opencv.hpp" #include "vino_param_lib/param_manager.h" - const char kInputType_Image[] = "Image"; const char kInputType_Video[] = "Video"; const char kInputType_StandardCamera[] = "StandardCamera"; +// const char kInputType_IpCamera[] = "IpCamera"; const char kInputType_CameraTopic[] = "RealSenseCameraTopic"; +const char kInputType_ImageTopic[] = "ImageTopic"; const char kInputType_RealSenseCamera[] = "RealSenseCamera"; const char kInputType_ServiceImage[] = "ServiceImage"; @@ -55,7 +56,13 @@ const char kInferTpye_HeadPoseEstimation[] = "HeadPoseEstimation"; const char kInferTpye_ObjectDetection[] = "ObjectDetection"; const char kInferTpye_ObjectSegmentation[] = "ObjectSegmentation"; const char kInferTpye_PersonReidentification[] = "PersonReidentification"; - +const char kInferTpye_ObjectDetectionTypeSSD[] = "SSD"; +const char kInferTpye_ObjectDetectionTypeYolov2voc[] = "yolov2-voc"; +const char kInferTpye_LandmarksDetection[] = "LandmarksDetection"; +const char kInferTpye_FaceReidentification[] = "FaceReidentification"; +const char kInferTpye_PersonAttribsDetection[] = "PersonAttribsDetection"; +const char kInferTpye_VehicleAttribsDetection[] = "VehicleAttribsDetection"; +const char kInferTpye_LicensePlateDetection[] = "LicensePlateDetection"; /** * @class PipelineParams * @brief This class is a pipeline parameter management that stores parameters @@ -63,22 +70,22 @@ const char kInferTpye_PersonReidentification[] = "PersonReidentification"; */ class PipelineParams { - public: +public: explicit PipelineParams(const std::string& name); - explicit PipelineParams(const Params::ParamManager::PipelineParams& params); - static Params::ParamManager::PipelineParams getPipeline( - const std::string& name); - PipelineParams& operator=(const Params::ParamManager::PipelineParams& params); + explicit PipelineParams(const Params::ParamManager::PipelineRawData& params); + Params::ParamManager::PipelineRawData getPipeline(const std::string& name); + PipelineParams& operator=(const Params::ParamManager::PipelineRawData& params); void update(); - void update(const Params::ParamManager::PipelineParams& params); + void update(const Params::ParamManager::PipelineRawData& params); bool isOutputTo(std::string& name); bool isGetFps(); + std::string findFilterConditions(const std::string& input, const std::string& output); const std::string kInputType_Image = "Image"; const std::string kOutputTpye_RViz = "RViz"; - private: - Params::ParamManager::PipelineParams params_; +private: + Params::ParamManager::PipelineRawData params_; }; #endif // DYNAMIC_VINO_LIB_PIPELINE_PARAMS_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/services/frame_processing_server.h b/dynamic_vino_lib/include/dynamic_vino_lib/services/frame_processing_server.h index 0284a47c..2a76a0dd 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/services/frame_processing_server.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/services/frame_processing_server.h @@ -29,8 +29,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -39,32 +41,20 @@ namespace vino_service { -template -class FrameProcessingServer //: public ros::NodeHandle +template +class FrameProcessingServer { public: - explicit FrameProcessingServer( - const std::string & service_name, - const std::string & config_path); - void initService(const std::string & config_path); + explicit FrameProcessingServer(const std::string& service_name, const std::string& config_path); + void initService(); -//private: - std::shared_ptr nh_; - -/* - void cbService( - std::shared_ptr request, - std::shared_ptr response); -*/ - bool cbService(ros::ServiceEvent& event); - // rclcpp::Service::SharedPtr service_; - //ros::NodeHandle nh_; - std::shared_ptr service_; - ros::ServiceServer srv; +private: std::string service_name_; std::string config_path_; + std::shared_ptr nh_; + std::shared_ptr service_; - bool add3(ros::ServiceEvent &); + bool cbService(ros::ServiceEvent& event); }; } // namespace vino_service -#endif // DYNAMIC_VINO_LIB__SERVICES__FRAME_PROCESSING_SERVER_HPP_ \ No newline at end of file +#endif // DYNAMIC_VINO_LIB__SERVICES__FRAME_PROCESSING_SERVER_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/services/pipeline_processing_server.h b/dynamic_vino_lib/include/dynamic_vino_lib/services/pipeline_processing_server.h new file mode 100644 index 00000000..76d35adc --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/services/pipeline_processing_server.h @@ -0,0 +1,68 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef DYNAMIC_VINO_LIB__SERVICES__PIPELINE_PROCESSING_SERVER_HPP_ +#define DYNAMIC_VINO_LIB__SERVICES__PIPELINE_PROCESSING_SERVER_HPP_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "dynamic_vino_lib/pipeline_manager.h" + +#include +#include +#include +#include + +namespace vino_service +{ +template +class PipelineProcessingServer +{ +public: + explicit PipelineProcessingServer(const std::string& service_name); + +private: + void initPipelineService(); + + std::shared_ptr nh_; + + bool cbService(ros::ServiceEvent& event); + + void setResponse(ros::ServiceEvent& event); + + void setPipelineByRequest(std::string pipeline_name, PipelineManager::PipelineState state); + + std::shared_ptr service_; + + std::map* pipelines_; + + std::string service_name_; +}; +} // namespace vino_service +#endif // DYNAMIC_VINO_LIB__SERVICES__FRAME_PROCESSING_SERVER_HPP_ diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/slog.h b/dynamic_vino_lib/include/dynamic_vino_lib/slog.h index 0466457a..1d81bf29 100644 --- a/dynamic_vino_lib/include/dynamic_vino_lib/slog.h +++ b/dynamic_vino_lib/include/dynamic_vino_lib/slog.h @@ -28,6 +28,37 @@ namespace slog { +#if 1 +enum COLOR +{ + RESET = 0, + BLUE = 1, + GREEN = 2, + YELLOW = 3, + RED = 4, +}; + +#else +// the following are UBUNTU/LINUX ONLY terminal color codes. +#define RESET "\033[0m" +#define BLACK "\033[30m" /* Black */ +#define RED "\033[31m" /* Red */ +#define GREEN "\033[32m" /* Green */ +#define YELLOW "\033[33m" /* Yellow */ +#define BLUE "\033[34m" /* Blue */ +#define MAGENTA "\033[35m" /* Magenta */ +#define CYAN "\033[36m" /* Cyan */ +#define WHITE "\033[37m" /* White */ +#define BOLDBLACK "\033[1m\033[30m" /* Bold Black */ +#define BOLDRED "\033[1m\033[31m" /* Bold Red */ +#define BOLDGREEN "\033[1m\033[32m" /* Bold Green */ +#define BOLDYELLOW "\033[1m\033[33m" /* Bold Yellow */ +#define BOLDBLUE "\033[1m\033[34m" /* Bold Blue */ +#define BOLDMAGENTA "\033[1m\033[35m" /* Bold Magenta */ +#define BOLDCYAN "\033[1m\033[36m" /* Bold Cyan */ +#define BOLDWHITE "\033[1m\033[37m" /* Bold White */ +#endif + /** * @class LogStreamEndLine * @brief The LogStreamEndLine class implements an end line marker for a log @@ -48,14 +79,15 @@ class LogStream std::string _prefix; std::ostream* _log_stream; bool _new_line; + int _color_id; - public: +public: /** * @brief A constructor. Creates an LogStream object * @param prefix The prefix to print */ - LogStream(const std::string& prefix, std::ostream& log_stream) - : _prefix(prefix), _new_line(true) + LogStream(const std::string& prefix, std::ostream& log_stream, const int color_id = -1) + : _prefix(prefix), _new_line(true), _color_id(color_id) { _log_stream = &log_stream; } @@ -69,6 +101,7 @@ class LogStream { if (_new_line) { + setLineColor(); (*_log_stream) << "[ " << _prefix << " ] "; _new_line = false; } @@ -81,15 +114,69 @@ class LogStream LogStream& operator<<(const LogStreamEndLine& arg) { _new_line = true; - + resetLineColor(); (*_log_stream) << std::endl; return *this; } + + void setLineColor() + { + switch (_color_id) + { + case BLUE: + (*_log_stream) << "\033[34m"; + break; + case GREEN: + (*_log_stream) << "\033[32m"; + break; + case YELLOW: + (*_log_stream) << "\033[33m"; + break; + case RED: + (*_log_stream) << "\033[31m"; + break; + default: + break; + } + } + + void resetLineColor() + { + if (_color_id > 0) + { + (*_log_stream) << "\033[0m"; // RESET + } + } +}; + +class NullStream +{ +public: + NullStream() + { + } + + NullStream(const std::string& prefix, std::ostream& log_stream) + { + (void)prefix; + (void)log_stream; + } + + template + NullStream& operator<<(const T& arg) + { + return *this; + } }; -static LogStream info("INFO", std::cout); -static LogStream warn("WARNING", std::cout); -static LogStream err("ERROR", std::cerr); +#ifdef LOG_LEVEL_DEBUG +static LogStream debug("DEBUG", std::cout, GREEN); +#else +static NullStream debug; +#endif +static LogStream info("INFO", std::cout, BLUE); +static LogStream warn("WARNING", std::cout, YELLOW); +static LogStream err("ERROR", std::cerr, RED); } // namespace slog #endif // DYNAMIC_VINO_LIB_SLOG_H diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/utils/mutex_counter.hpp b/dynamic_vino_lib/include/dynamic_vino_lib/utils/mutex_counter.hpp new file mode 100644 index 00000000..e1735576 --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/utils/mutex_counter.hpp @@ -0,0 +1,58 @@ +// Copyright (c) 2018-2019 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +// @brief a utility class for mutex counter (Thread Safe). +// @file mutex_counter.hpp +// + +#ifndef DYNAMIC_VINO_LIB__UTILS__MUTEX_COUNTER_HPP_ +#define DYNAMIC_VINO_LIB__UTILS__MUTEX_COUNTER_HPP_ + +#include +#include + +class MutexCounter +{ +public: + explicit MutexCounter(int init_counter = 0) + { + std::lock_guard lk(counter_mutex_); + counter_ = init_counter; + } + + void increaseCounter() + { + std::lock_guard lk(counter_mutex_); + ++counter_; + } + + void decreaseCounter() + { + std::lock_guard lk(counter_mutex_); + --counter_; + } + + int get() + { + return counter_; + } + +private: + std::atomic counter_; + std::mutex counter_mutex_; + std::condition_variable cv_; +}; + +#endif // DYNAMIC_VINO_LIB__UTILS__MUTEX_COUNTER_HPP_ \ No newline at end of file diff --git a/dynamic_vino_lib/include/dynamic_vino_lib/utils/version_info.hpp b/dynamic_vino_lib/include/dynamic_vino_lib/utils/version_info.hpp new file mode 100644 index 00000000..d928f67a --- /dev/null +++ b/dynamic_vino_lib/include/dynamic_vino_lib/utils/version_info.hpp @@ -0,0 +1,148 @@ +// Copyright (c) 2018-2019 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +// @brief a header file with version information about Inference Engine. +// @file version_info.hpp +// + +#ifndef DYNAMIC_VINO_LIB__UTILS__VERSION_INFO_HPP_ +#define DYNAMIC_VINO_LIB__UTILS__VERSION_INFO_HPP_ + +#include +#include +#include +#include +#if (defined(USE_OLD_E_PLUGIN_API)) +#include +#endif +#include +#include +#include +#include +#include +#include +#include + +#ifdef WIN32 +#define UNUSED +#else +#define UNUSED __attribute__((unused)) +#endif + +/** + * @brief Trims from both ends (in place) + * @param s - string to trim + * @return trimmed string + */ +inline std::string& trim(std::string& s) +{ + s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun(std::isspace)))); + s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun(std::isspace))).base(), s.end()); + return s; +} + +static std::ostream& operator<<(std::ostream& os, const InferenceEngine::Version* version) +{ + os << "\n\tAPI version ............ "; + if (nullptr == version) + { + os << "UNKNOWN"; + } + else + { + os << version->apiVersion.major << "." << version->apiVersion.minor; + if (nullptr != version->buildNumber) + { + os << "\n\t" + << "Build .................. " << version->buildNumber; + } + if (nullptr != version->description) + { + os << "\n\t" + << "Description ............ " << version->description; + } + } + return os; +} + +#if (defined(USE_OLD_E_PLUGIN_API)) +/** + * @class PluginVersion + * @brief A PluginVersion class stores plugin version and initialization status + */ +struct PluginVersion : public InferenceEngine::Version +{ + bool initialized = false; + + explicit PluginVersion(const InferenceEngine::Version* ver) + { + if (nullptr == ver) + { + return; + } + InferenceEngine::Version::operator=(*ver); + initialized = true; + } + + operator bool() const noexcept + { + return initialized; + } +}; + +static UNUSED std::ostream& operator<<(std::ostream& os, const PluginVersion& version) +{ + os << "\tPlugin version ......... "; + if (!version) + { + os << "UNKNOWN"; + } + else + { + os << version.apiVersion.major << "." << version.apiVersion.minor; + } + + os << "\n\tPlugin name ............ "; + if (!version || version.description == nullptr) + { + os << "UNKNOWN"; + } + else + { + os << version.description; + } + + os << "\n\tPlugin build ........... "; + if (!version || version.buildNumber == nullptr) + { + os << "UNKNOWN"; + } + else + { + os << version.buildNumber; + } + + return os; +} + +inline void printPluginVersion(InferenceEngine::InferenceEnginePluginPtr ptr, std::ostream& stream) +{ + const PluginVersion* pluginVersion = nullptr; + ptr->GetVersion((const InferenceEngine::Version*&)pluginVersion); + stream << pluginVersion << std::endl; +} +#endif // (defined(USE_OLD_E_PLUGIN_API)) + +#endif // DYNAMIC_VINO_LIB__UTILS__VERSION_INFO_HPP_ diff --git a/dynamic_vino_lib/src/engines/engine.cpp b/dynamic_vino_lib/src/engines/engine.cpp index db008926..04fe6d77 100644 --- a/dynamic_vino_lib/src/engines/engine.cpp +++ b/dynamic_vino_lib/src/engines/engine.cpp @@ -20,9 +20,14 @@ */ #include "dynamic_vino_lib/engines/engine.h" -Engines::Engine::Engine(InferenceEngine::InferencePlugin plg, - const Models::BaseModel::Ptr base_model) +#if (defined(USE_OLD_E_PLUGIN_API)) +Engines::Engine::Engine(InferenceEngine::InferencePlugin plg, const Models::BaseModel::Ptr base_model) { - request_ = (plg.LoadNetwork(base_model->net_reader_->getNetwork(), {})) - .CreateInferRequestPtr(); + request_ = (plg.LoadNetwork(base_model->getNetReader()->getNetwork(), {})).CreateInferRequestPtr(); +} +#endif + +Engines::Engine::Engine(InferenceEngine::InferRequest::Ptr& request) +{ + request_ = request; } diff --git a/dynamic_vino_lib/src/engines/engine_manager.cpp b/dynamic_vino_lib/src/engines/engine_manager.cpp new file mode 100644 index 00000000..591b12e2 --- /dev/null +++ b/dynamic_vino_lib/src/engines/engine_manager.cpp @@ -0,0 +1,110 @@ +// Copyright (c) 2018-2019 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with definition of Engine class + * @file engine.cpp + */ +#include "dynamic_vino_lib/engines/engine_manager.h" +#include "dynamic_vino_lib/engines/engine.h" +#include "dynamic_vino_lib/slog.h" +#include "dynamic_vino_lib/models/base_model.h" +#include "dynamic_vino_lib/utils/version_info.hpp" +#include +#include +#if (defined(USE_OLD_E_PLUGIN_API)) +#include +#endif + +std::shared_ptr Engines::EngineManager::createEngine(const std::string& device, + const std::shared_ptr& model) +{ +#if (defined(USE_OLD_E_PLUGIN_API)) + return createEngine_beforeV2019R2(device, model); +#else + return createEngine_V2019R2_plus(device, model); +#endif +} + +std::shared_ptr Engines::EngineManager::createEngine_V2019R2_plus( + const std::string& device, const std::shared_ptr& model) +{ + InferenceEngine::Core core; + auto executable_network = core.LoadNetwork(model->getNetReader()->getNetwork(), device); + auto request = executable_network.CreateInferRequestPtr(); + + return std::make_shared(request); +} + +#if (defined(USE_OLD_E_PLUGIN_API)) +std::shared_ptr Engines::EngineManager::createEngine_beforeV2019R2( + const std::string& device, const std::shared_ptr& model) +{ + if (plugins_for_devices_.find(device) == plugins_for_devices_.end()) + { + auto pcommon = Params::ParamManager::getInstance().getCommon(); + plugins_for_devices_[device] = *makePluginByName(device, pcommon.custom_cpu_library, pcommon.custom_cldnn_library, + pcommon.enable_performance_count); + slog::info << "Created plugin for " << device << slog::endl; + } + + auto executeable_network = plugins_for_devices_[device].LoadNetwork(model->getNetReader()->getNetwork(), {}); + auto request = executeable_network.CreateInferRequestPtr(); + + return std::make_shared(request); +} + +std::unique_ptr +Engines::EngineManager::makePluginByName(const std::string& device_name, const std::string& custom_cpu_library_message, + const std::string& custom_cldnn_message, bool performance_message) +{ + slog::info << "Creating plugin for " << device_name << slog::endl; + + InferenceEngine::InferencePlugin plugin = + InferenceEngine::PluginDispatcher({ "../../../lib/intel64", "" }).getPluginByDevice(device_name); + + /** Printing plugin version **/ + printPluginVersion(plugin, std::cout); + + /** Load extensions for the CPU plugin **/ + if ((device_name.find("CPU") != std::string::npos)) + { + plugin.AddExtension(std::make_shared()); + if (!custom_cpu_library_message.empty()) + { + slog::info << "custom cpu library is not empty, tyring to use this extension:" << custom_cpu_library_message + << slog::endl; + // CPU(MKLDNN) extensions are loaded as a shared library and passed as a + // pointer to base + // extension + auto extension_ptr = InferenceEngine::make_so_pointer(custom_cpu_library_message); + plugin.AddExtension(extension_ptr); + } + } + else if (!custom_cldnn_message.empty()) + { + slog::info << "custom cldnn library is not empty, tyring to use this extension:" << custom_cldnn_message + << slog::endl; + // Load Extensions for other plugins not CPU + plugin.SetConfig({ { InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE, custom_cldnn_message } }); + } + if (performance_message) + { + plugin.SetConfig( + { { InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES } }); + } + + return std::make_unique(InferenceEngine::InferenceEnginePluginPtr(plugin)); +} +#endif diff --git a/dynamic_vino_lib/src/factory.cpp b/dynamic_vino_lib/src/factory.cpp deleted file mode 100644 index d886124c..00000000 --- a/dynamic_vino_lib/src/factory.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2018 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @brief a header file with declaration of Factory class - * @file factory.cpp - */ - -#include -#include - -#include "dynamic_vino_lib/factory.h" -#include "dynamic_vino_lib/inputs/image_input.h" -#include "dynamic_vino_lib/inputs/realsense_camera.h" -#include "dynamic_vino_lib/inputs/realsense_camera_topic.h" -#include "dynamic_vino_lib/inputs/standard_camera.h" -#include "dynamic_vino_lib/inputs/video_input.h" - -std::shared_ptr Factory::makeInputDeviceByName( - const std::string& input_device_name, const std::string& input_file_path) -{ - std::cout << "InputDvice: " << input_device_name << std::endl; - if (input_device_name == "RealSenseCamera") - { - return std::make_shared(); - } - else if (input_device_name == "StandardCamera") - { - return std::make_shared(); - } - else if (input_device_name == "RealSenseCameraTopic") - { - std::cout << "tring to create instance for " << input_device_name - << std::endl; - return std::make_shared(); - } - else if (input_device_name == "Video") - { - return std::make_shared(input_file_path); - } - else if (input_device_name == "Image") - { - return std::make_shared(input_file_path); - } - else - { - throw std::logic_error("Unsupported input category"); - } -} - -std::shared_ptr Factory::makePluginByName( - const std::string& device_name, - const std::string& custom_cpu_library_message, // FLAGS_l - const std::string& custom_cldnn_message, // FLAGS_c - bool performance_message) -{ // FLAGS_pc - InferenceEngine::InferencePlugin plugin = InferenceEngine::PluginDispatcher({"../../../lib/intel64", ""}) - .getPluginByDevice(device_name); - /** Printing plugin version **/ - printPluginVersion(plugin, std::cout); - /** Load extensions for the CPU plugin **/ - if ((device_name.find("CPU") != std::string::npos)) - { - plugin.AddExtension(std::make_shared()); - if (!custom_cpu_library_message.empty()) - { - // CPU(MKLDNN) extensions are loaded as a shared library and passed as a - // pointer to base - // extension - auto extension_ptr = InferenceEngine::make_so_pointer( - custom_cpu_library_message); - plugin.AddExtension(extension_ptr); - } - } - else if (!custom_cldnn_message.empty()) - { - // Load Extensions for other plugins not CPU - plugin.SetConfig( - {{InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE, custom_cldnn_message}}); - } - if (performance_message) - { - plugin.SetConfig( - {{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES}}); - } - return std::make_shared( - InferenceEngine::InferenceEnginePluginPtr(plugin)); -} diff --git a/dynamic_vino_lib/src/inferences/age_gender_detection.cpp b/dynamic_vino_lib/src/inferences/age_gender_detection.cpp index b087ab2a..ea27b369 100644 --- a/dynamic_vino_lib/src/inferences/age_gender_detection.cpp +++ b/dynamic_vino_lib/src/inferences/age_gender_detection.cpp @@ -21,42 +21,39 @@ #include #include +#include #include "dynamic_vino_lib/inferences/age_gender_detection.h" #include "dynamic_vino_lib/outputs/base_output.h" // AgeGenderResult -dynamic_vino_lib::AgeGenderResult::AgeGenderResult(const cv::Rect& location) - : Result(location) +dynamic_vino_lib::AgeGenderResult::AgeGenderResult(const cv::Rect& location) : Result(location) { } // AgeGender Detection -dynamic_vino_lib::AgeGenderDetection::AgeGenderDetection() - : dynamic_vino_lib::BaseInference() +dynamic_vino_lib::AgeGenderDetection::AgeGenderDetection() : dynamic_vino_lib::BaseInference() { } dynamic_vino_lib::AgeGenderDetection::~AgeGenderDetection() = default; -void dynamic_vino_lib::AgeGenderDetection::loadNetwork( - std::shared_ptr network) +void dynamic_vino_lib::AgeGenderDetection::loadNetwork(std::shared_ptr network) { valid_model_ = network; setMaxBatchSize(network->getMaxBatchSize()); } -bool dynamic_vino_lib::AgeGenderDetection::enqueue( - const cv::Mat& frame, const cv::Rect& input_frame_loc) +bool dynamic_vino_lib::AgeGenderDetection::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) { if (getEnqueuedNum() == 0) { results_.clear(); } - bool succeed = dynamic_vino_lib::BaseInference::enqueue( - frame, input_frame_loc, 1, getResultsLength(), - valid_model_->getInputName()); - if (!succeed) return false; + bool succeed = dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, getResultsLength(), + valid_model_->getInputName()); + if (!succeed) + return false; Result r(input_frame_loc); results_.emplace_back(r); return true; @@ -70,12 +67,11 @@ bool dynamic_vino_lib::AgeGenderDetection::submitRequest() bool dynamic_vino_lib::AgeGenderDetection::fetchResults() { bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); - if (!can_fetch) return false; + if (!can_fetch) + return false; auto request = getEngine()->getRequest(); - InferenceEngine::Blob::Ptr genderBlob = - request->GetBlob(valid_model_->getOutputGenderName()); - InferenceEngine::Blob::Ptr ageBlob = - request->GetBlob(valid_model_->getOutputAgeName()); + InferenceEngine::Blob::Ptr genderBlob = request->GetBlob(valid_model_->getOutputGenderName()); + InferenceEngine::Blob::Ptr ageBlob = request->GetBlob(valid_model_->getOutputAgeName()); for (size_t i = 0; i < results_.size(); ++i) { @@ -85,27 +81,41 @@ bool dynamic_vino_lib::AgeGenderDetection::fetchResults() return true; } -const int dynamic_vino_lib::AgeGenderDetection::getResultsLength() const +int dynamic_vino_lib::AgeGenderDetection::getResultsLength() const { return static_cast(results_.size()); } -const dynamic_vino_lib::Result* -dynamic_vino_lib::AgeGenderDetection::getLocationResult(int idx) const +const dynamic_vino_lib::Result* dynamic_vino_lib::AgeGenderDetection::getLocationResult(int idx) const { return &(results_[idx]); } const std::string dynamic_vino_lib::AgeGenderDetection::getName() const { - return valid_model_->getModelName(); + return valid_model_->getModelCategory(); } -const void dynamic_vino_lib::AgeGenderDetection::observeOutput( - const std::shared_ptr& output) +void dynamic_vino_lib::AgeGenderDetection::observeOutput(const std::shared_ptr& output) { if (output != nullptr) { output->accept(results_); } } + +const std::vector +dynamic_vino_lib::AgeGenderDetection::getFilteredROIs(const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "Age gender detection does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/dynamic_vino_lib/src/inferences/base_filter.cpp b/dynamic_vino_lib/src/inferences/base_filter.cpp new file mode 100644 index 00000000..0e9a22b8 --- /dev/null +++ b/dynamic_vino_lib/src/inferences/base_filter.cpp @@ -0,0 +1,266 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief An implementation file with implementation for BaseFilter Class + * @file base_filter.cpp + */ + +#include "dynamic_vino_lib/inferences/base_filter.h" +#include +#include +#include + +dynamic_vino_lib::BaseFilter::BaseFilter() +{ +} + +bool dynamic_vino_lib::BaseFilter::isValidFilterConditions(const std::string& filter_conditions) +{ + return strip(filter_conditions) != ""; +} + +void dynamic_vino_lib::BaseFilter::acceptFilterConditions(const std::string& filter_conditions) +{ + if (striped_conditions_.empty()) + { + striped_conditions_ = strip(filter_conditions); + std::vector infix_conditions = split(striped_conditions_); + infixToSuffix(infix_conditions); + } +} + +bool dynamic_vino_lib::BaseFilter::isRelationOperator(const std::string& str) +{ + if (std::find(relation_operators_.begin(), relation_operators_.end(), str) != relation_operators_.end()) + { + return true; + } + return false; +} + +bool dynamic_vino_lib::BaseFilter::isLogicOperator(const std::string& str) +{ + if (std::find(logic_operators_.begin(), logic_operators_.end(), str) != logic_operators_.end()) + { + return true; + } + return false; +} + +bool dynamic_vino_lib::BaseFilter::isPriorTo(const std::string& operator1, const std::string& operator2) +{ + if (isRelationOperator(operator1) && isLogicOperator(operator2)) + { + return true; + } + return false; +} + +std::string dynamic_vino_lib::BaseFilter::boolToStr(bool value) +{ + if (value) + { + return "true"; + } + return "false"; +} + +bool dynamic_vino_lib::BaseFilter::strToBool(const std::string& value) +{ + if (!value.compare("true")) + { + return true; + } + else if (!value.compare("false")) + { + return false; + } + else + { + slog::err << "Invalid string: " << value << " for bool conversion!" << slog::endl; + } + return false; +} + +const std::vector& dynamic_vino_lib::BaseFilter::getSuffixConditions() const +{ + return suffix_conditons_; +} + +bool dynamic_vino_lib::BaseFilter::logicOperation(const std::string& logic1, const std::string& op, + const std::string& logic2) +{ + if (!op.compare("&&")) + { + return strToBool(logic1) && strToBool(logic2); + } + else if (!op.compare("||")) + { + return strToBool(logic1) || strToBool(logic2); + } + else + { + slog::err << "Invalid operator: " << op << " for logic operation!" << slog::endl; + return false; + } +} + +bool dynamic_vino_lib::BaseFilter::stringCompare(const std::string& candidate, const std::string& op, + const std::string& target) +{ + if (!op.compare("==")) + { + return !target.compare(candidate); + } + else if (!op.compare("!=")) + { + return target.compare(candidate); + } + else + { + slog::err << "Invalid operator " << op << " for label comparsion" << slog::endl; + return false; + } +} + +bool dynamic_vino_lib::BaseFilter::floatCompare(float candidate, const std::string& op, float target) +{ + if (!op.compare("<=")) + { + return candidate <= target; + } + else if (!op.compare(">=")) + { + return candidate >= target; + } + else if (!op.compare("<")) + { + return candidate < target; + } + else if (!op.compare(">")) + { + return candidate > target; + } + else + { + slog::err << "Invalid operator " << op << " for confidence comparsion" << slog::endl; + return false; + } +} + +float dynamic_vino_lib::BaseFilter::stringToFloat(const std::string& candidate) +{ + float result = 0; + try + { + result = std::stof(candidate); + } + catch (...) + { + slog::err << "Failed when converting string " << candidate << " to float" << slog::endl; + } + return result; +} + +std::vector dynamic_vino_lib::BaseFilter::split(const std::string& filter_conditions) +{ + std::vector seperators; + seperators.insert(seperators.end(), relation_operators_.begin(), relation_operators_.end()); + seperators.insert(seperators.end(), logic_operators_.begin(), logic_operators_.end()); + seperators.push_back("("); + seperators.push_back(")"); + std::vector infix_conditions; + int last_pos = 0, pos = 0; + for (pos = 0; pos < filter_conditions.length(); pos++) + { + for (auto sep : seperators) + { + if (!sep.compare(filter_conditions.substr(pos, sep.length()))) + { + std::string elem = filter_conditions.substr(last_pos, pos - last_pos); + if (!elem.empty()) + { + infix_conditions.push_back(elem); + } + elem = filter_conditions.substr(pos, sep.length()); + infix_conditions.push_back(elem); + last_pos = pos + sep.length(); + pos = last_pos - 1; + break; + } + } + } + if (last_pos != pos) + { + infix_conditions.push_back(filter_conditions.substr(last_pos, pos - last_pos)); + } + return infix_conditions; +} + +void dynamic_vino_lib::BaseFilter::infixToSuffix(std::vector& infix_conditions) +{ + std::stack operator_stack; + for (auto elem : infix_conditions) + { + if (!elem.compare("(")) + { + operator_stack.push(elem); + } + else if (!elem.compare(")")) + { + while (!operator_stack.empty() && operator_stack.top().compare("(")) + { + suffix_conditons_.push_back(operator_stack.top()); + operator_stack.pop(); + } + if (operator_stack.empty()) + { + slog::err << "Brackets mismatch in filter_conditions!" << slog::endl; + } + operator_stack.pop(); + } + else if (isRelationOperator(elem) || isLogicOperator(elem)) + { + while (!operator_stack.empty() && isPriorTo(operator_stack.top(), elem)) + { + suffix_conditons_.push_back(operator_stack.top()); + operator_stack.pop(); + } + operator_stack.push(elem); + } + else + { + suffix_conditons_.push_back(elem); + } + } + while (!operator_stack.empty()) + { + suffix_conditons_.push_back(operator_stack.top()); + operator_stack.pop(); + } +} + +std::string dynamic_vino_lib::BaseFilter::strip(const std::string& str) +{ + std::string stripped_string = ""; + for (auto character : str) + { + if (character != ' ') + { + stripped_string += character; + } + } + return stripped_string; +} diff --git a/dynamic_vino_lib/src/inferences/base_inference.cpp b/dynamic_vino_lib/src/inferences/base_inference.cpp index 4ca7d251..10058cd9 100644 --- a/dynamic_vino_lib/src/inferences/base_inference.cpp +++ b/dynamic_vino_lib/src/inferences/base_inference.cpp @@ -34,25 +34,59 @@ dynamic_vino_lib::BaseInference::BaseInference() = default; dynamic_vino_lib::BaseInference::~BaseInference() = default; -void dynamic_vino_lib::BaseInference::loadEngine( - const std::shared_ptr engine) +void dynamic_vino_lib::BaseInference::loadEngine(const std::shared_ptr engine) { engine_ = engine; } bool dynamic_vino_lib::BaseInference::submitRequest() { - if (engine_->getRequest() == nullptr) return false; - if (!enqueued_frames) return false; - enqueued_frames = 0; + if (engine_->getRequest() == nullptr) + { + return false; + } + if (!enqueued_frames_) + { + return false; + } + enqueued_frames_ = 0; results_fetched_ = false; engine_->getRequest()->StartAsync(); return true; } +bool dynamic_vino_lib::BaseInference::SynchronousRequest() +{ + if (engine_->getRequest() == nullptr) + { + return false; + } + if (!enqueued_frames_) + { + return false; + } + enqueued_frames_ = 0; + results_fetched_ = false; + engine_->getRequest()->Infer(); + return true; +} + bool dynamic_vino_lib::BaseInference::fetchResults() { - if (results_fetched_) return false; + if (results_fetched_) + { + return false; + } results_fetched_ = true; return true; } + +void dynamic_vino_lib::BaseInference::addCandidatedModel(std::shared_ptr model) +{ + slog::info << "TESTING in addCandidatedModel()" << slog::endl; + if (model != nullptr) + { + slog::info << "adding new Model Candidate..." << slog::endl; + candidated_models_.push_back(model); + } +} diff --git a/dynamic_vino_lib/src/inferences/base_reidentification.cpp b/dynamic_vino_lib/src/inferences/base_reidentification.cpp new file mode 100644 index 00000000..1e2c0a8d --- /dev/null +++ b/dynamic_vino_lib/src/inferences/base_reidentification.cpp @@ -0,0 +1,196 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with declaration of BaseReidentification class + * @file base_reidentification.cpp + */ +#include +#include +#include +#include +#include +#include +#include +#include "dynamic_vino_lib/inferences/base_reidentification.h" +#include "dynamic_vino_lib/slog.h" + +// Tracker +dynamic_vino_lib::Tracker::Tracker(int max_record_size, double same_track_thresh, double new_track_thresh) + : max_record_size_(max_record_size), same_track_thresh_(same_track_thresh), new_track_thresh_(new_track_thresh) +{ +} + +int dynamic_vino_lib::Tracker::processNewTrack(const std::vector& feature) +{ + int most_similar_id; + double similarity = findMostSimilarTrack(feature, most_similar_id); + if (similarity > same_track_thresh_) + { + updateMatchTrack(most_similar_id, feature); + } + else if (similarity < new_track_thresh_) + { + most_similar_id = addNewTrack(feature); + } + return most_similar_id; +} + +double dynamic_vino_lib::Tracker::findMostSimilarTrack(const std::vector& feature, int& most_similar_id) +{ + double max_similarity = 0; + most_similar_id = -1; + for (auto iter = recorded_tracks_.begin(); iter != recorded_tracks_.end(); iter++) + { + double sim = calcSimilarity(feature, iter->second.feature); + if (sim > max_similarity) + { + max_similarity = sim; + most_similar_id = iter->first; + } + } + return max_similarity; +} + +double dynamic_vino_lib::Tracker::calcSimilarity(const std::vector& feature_a, + const std::vector& feature_b) +{ + if (feature_a.size() != feature_b.size()) + { + slog::err << "cosine similarity can't be called for vectors of different lengths: " + << "feature_a size = " << std::to_string(feature_a.size()) + << "feature_b size = " << std::to_string(feature_b.size()) << slog::endl; + } + float mul_sum, denom_a, denom_b, value_a, value_b; + mul_sum = denom_a = denom_b = value_a = value_b = 0; + for (auto i = 0; i < feature_a.size(); i++) + { + value_a = feature_a[i]; + value_b = feature_b[i]; + mul_sum += value_a * value_b; + denom_a += value_a * value_a; + denom_b += value_b * value_b; + } + if (denom_a == 0 || denom_b == 0) + { + slog::err << "cosine similarity is not defined whenever one or both " + "input vectors are zero-vectors." + << slog::endl; + } + return mul_sum / (sqrt(denom_a) * sqrt(denom_b)); +} + +void dynamic_vino_lib::Tracker::updateMatchTrack(int track_id, const std::vector& feature) +{ + if (recorded_tracks_.find(track_id) != recorded_tracks_.end()) + { + recorded_tracks_[track_id].feature.assign(feature.begin(), feature.end()); + recorded_tracks_[track_id].lastest_update_time = getCurrentTime(); + } + else + { + slog::err << "updating a non-existing track." << slog::endl; + } +} + +void dynamic_vino_lib::Tracker::removeEarlestTrack() +{ + std::lock_guard lk(tracks_mtx_); + int64_t earlest_time = LONG_MAX; + auto remove_iter = recorded_tracks_.begin(); + for (auto iter = recorded_tracks_.begin(); iter != recorded_tracks_.end(); iter++) + { + if (iter->second.lastest_update_time < earlest_time) + { + earlest_time = iter->second.lastest_update_time; + remove_iter = iter; + } + } + recorded_tracks_.erase(remove_iter); +} + +int dynamic_vino_lib::Tracker::addNewTrack(const std::vector& feature) +{ + if (recorded_tracks_.size() >= max_record_size_) + { + std::thread remove_thread(std::bind(&Tracker::removeEarlestTrack, this)); + remove_thread.detach(); + } + std::lock_guard lk(tracks_mtx_); + Track track; + track.lastest_update_time = getCurrentTime(); + track.feature.assign(feature.begin(), feature.end()); + max_track_id_ += 1; + recorded_tracks_.insert(std::pair(max_track_id_, track)); + return max_track_id_; +} + +int64_t dynamic_vino_lib::Tracker::getCurrentTime() +{ + auto tp = std::chrono::time_point_cast(std::chrono::system_clock::now()); + return static_cast(tp.time_since_epoch().count()); +} + +bool dynamic_vino_lib::Tracker::saveTracksToFile(std::string filepath) +{ + std::ofstream outfile(filepath); + if (!outfile.is_open()) + { + slog::err << "file not exists in file path: " << filepath << slog::endl; + return false; + } + for (auto record : recorded_tracks_) + { + outfile << record.first << " " << record.second.lastest_update_time << " "; + for (auto elem : record.second.feature) + { + outfile << elem << " "; + } + outfile << "\n"; + } + outfile.close(); + slog::info << "sucessfully save tracks into file: " << filepath << slog::endl; + return true; +} + +bool dynamic_vino_lib::Tracker::loadTracksFromFile(std::string filepath) +{ + std::ifstream infile(filepath); + if (!infile.is_open()) + { + slog::err << "file not exists in file path: " << filepath << slog::endl; + return false; + } + recorded_tracks_.clear(); + while (!infile.eof()) + { + int track_id; + int64_t lastest_update_time; + std::vector feature; + infile >> track_id >> lastest_update_time; + for (int num = 0; num < 256; num++) + { + float elem; + infile >> elem; + feature.push_back(elem); + } + Track track; + track.lastest_update_time = lastest_update_time; + track.feature = feature; + recorded_tracks_[track_id] = track; + } + infile.close(); + slog::info << "sucessfully load tracks from file: " << filepath << slog::endl; + return true; +} diff --git a/dynamic_vino_lib/src/inferences/emotions_detection.cpp b/dynamic_vino_lib/src/inferences/emotions_detection.cpp index 9e10e3d3..54ef3d43 100644 --- a/dynamic_vino_lib/src/inferences/emotions_detection.cpp +++ b/dynamic_vino_lib/src/inferences/emotions_detection.cpp @@ -22,42 +22,38 @@ #include #include +#include #include "dynamic_vino_lib/inferences/emotions_detection.h" #include "dynamic_vino_lib/outputs/base_output.h" #include "dynamic_vino_lib/slog.h" // EmotionsResult -dynamic_vino_lib::EmotionsResult::EmotionsResult(const cv::Rect& location) - : Result(location) +dynamic_vino_lib::EmotionsResult::EmotionsResult(const cv::Rect& location) : Result(location) { } // Emotions Detection -dynamic_vino_lib::EmotionsDetection::EmotionsDetection() - : dynamic_vino_lib::BaseInference() +dynamic_vino_lib::EmotionsDetection::EmotionsDetection() : dynamic_vino_lib::BaseInference() { } dynamic_vino_lib::EmotionsDetection::~EmotionsDetection() = default; -void dynamic_vino_lib::EmotionsDetection::loadNetwork( - const std::shared_ptr network) +void dynamic_vino_lib::EmotionsDetection::loadNetwork(const std::shared_ptr network) { valid_model_ = network; setMaxBatchSize(network->getMaxBatchSize()); } -bool dynamic_vino_lib::EmotionsDetection::enqueue( - const cv::Mat& frame, const cv::Rect& input_frame_loc) +bool dynamic_vino_lib::EmotionsDetection::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) { if (getEnqueuedNum() == 0) { results_.clear(); } - bool succeed = dynamic_vino_lib::BaseInference::enqueue( - frame, input_frame_loc, 1, getResultsLength(), - valid_model_->getInputName()); + bool succeed = dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, getResultsLength(), + valid_model_->getInputName()); if (!succeed) { slog::err << "Failed enqueue Emotion frame." << slog::endl; @@ -77,21 +73,19 @@ bool dynamic_vino_lib::EmotionsDetection::submitRequest() bool dynamic_vino_lib::EmotionsDetection::fetchResults() { bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); - if (!can_fetch) return false; + if (!can_fetch) + return false; int label_length = static_cast(valid_model_->getLabels().size()); std::string output_name = valid_model_->getOutputName(); - InferenceEngine::Blob::Ptr emotions_blob = - getEngine()->getRequest()->GetBlob(output_name); + InferenceEngine::Blob::Ptr emotions_blob = getEngine()->getRequest()->GetBlob(output_name); /** emotions vector must have the same size as number of channels in model output. Default output format is NCHW so we check index 1 */ int64 num_of_channels = emotions_blob->getTensorDesc().getDims().at(1); if (num_of_channels != label_length) { - slog::err << "Output size (" << num_of_channels - << ") of the Emotions Recognition network is not equal " - << "to used emotions vector size (" << label_length << ")" - << slog::endl; + slog::err << "Output size (" << num_of_channels << ") of the Emotions Recognition network is not equal " + << "to used emotions vector size (" << label_length << ")" << slog::endl; throw std::logic_error("Output size (" + std::to_string(num_of_channels) + ") of the Emotions Recognition network is not equal " "to used emotions vector size (" + @@ -104,36 +98,48 @@ bool dynamic_vino_lib::EmotionsDetection::fetchResults() for (unsigned int idx = 0; idx < results_.size(); ++idx) { auto output_idx_pos = emotions_values + idx; - int64 max_prob_emotion_idx = - std::max_element(output_idx_pos, output_idx_pos + label_length) - - output_idx_pos; + int64 max_prob_emotion_idx = std::max_element(output_idx_pos, output_idx_pos + label_length) - output_idx_pos; results_[idx].label_ = valid_model_->getLabels()[max_prob_emotion_idx]; } return true; } -const int dynamic_vino_lib::EmotionsDetection::getResultsLength() const +int dynamic_vino_lib::EmotionsDetection::getResultsLength() const { return static_cast(results_.size()); } -const dynamic_vino_lib::Result* -dynamic_vino_lib::EmotionsDetection::getLocationResult(int idx) const +const dynamic_vino_lib::Result* dynamic_vino_lib::EmotionsDetection::getLocationResult(int idx) const { return &(results_[idx]); } const std::string dynamic_vino_lib::EmotionsDetection::getName() const { - return valid_model_->getModelName(); + return valid_model_->getModelCategory(); } -const void dynamic_vino_lib::EmotionsDetection::observeOutput( - const std::shared_ptr& output) +void dynamic_vino_lib::EmotionsDetection::observeOutput(const std::shared_ptr& output) { if (output != nullptr) { output->accept(results_); } } + +const std::vector +dynamic_vino_lib::EmotionsDetection::getFilteredROIs(const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "Emotion detection does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/dynamic_vino_lib/src/inferences/face_detection.cpp b/dynamic_vino_lib/src/inferences/face_detection.cpp index e7aa3c36..7fdb62a7 100644 --- a/dynamic_vino_lib/src/inferences/face_detection.cpp +++ b/dynamic_vino_lib/src/inferences/face_detection.cpp @@ -29,120 +29,12 @@ #include "dynamic_vino_lib/slog.h" // FaceDetectionResult -dynamic_vino_lib::FaceDetectionResult::FaceDetectionResult( - const cv::Rect& location) - : Result(location) +dynamic_vino_lib::FaceDetectionResult::FaceDetectionResult(const cv::Rect& location) : ObjectDetectionResult(location) { } // FaceDetection -dynamic_vino_lib::FaceDetection::FaceDetection(double show_output_thresh) - : dynamic_vino_lib::BaseInference(), - show_output_thresh_(show_output_thresh) +dynamic_vino_lib::FaceDetection::FaceDetection(bool enable_roi_constraint, double show_output_thresh) + : ObjectDetection(enable_roi_constraint, show_output_thresh) { } - -dynamic_vino_lib::FaceDetection::~FaceDetection() = default; - -void dynamic_vino_lib::FaceDetection::loadNetwork( - const std::shared_ptr network) -{ - valid_model_ = network; - max_proposal_count_ = network->getMaxProposalCount(); - object_size_ = network->getObjectSize(); - setMaxBatchSize(network->getMaxBatchSize()); -} - -bool dynamic_vino_lib::FaceDetection::enqueue(const cv::Mat& frame, - const cv::Rect& input_frame_loc) -{ - // slog::info << "Face-enqueue" << slog::endl; - if (width_ == 0 && height_ == 0) - { - width_ = frame.cols; - height_ = frame.rows; - } - if (!dynamic_vino_lib::BaseInference::enqueue( - frame, input_frame_loc, 1, 0, valid_model_->getInputName())) - { - return false; - } - Result r(input_frame_loc); - results_.clear(); - results_.emplace_back(r); - return true; -} - -bool dynamic_vino_lib::FaceDetection::submitRequest() -{ - // slog::info << "Face-submitRequest" << slog::endl; - return dynamic_vino_lib::BaseInference::submitRequest(); -} - -bool dynamic_vino_lib::FaceDetection::fetchResults() -{ - bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); - if (!can_fetch) return false; - bool found_result = false; - results_.clear(); - InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); - std::string output = valid_model_->getOutputName(); - const float* detections = request->GetBlob(output)->buffer().as(); - for (int i = 0; i < max_proposal_count_; i++) - { - float image_id = detections[i * object_size_ + 0]; - cv::Rect r; - auto label_num = static_cast(detections[i * object_size_ + 1]); - std::vector& labels = valid_model_->getLabels(); - - r.x = static_cast(detections[i * object_size_ + 3] * width_); - r.y = static_cast(detections[i * object_size_ + 4] * height_); - r.width = static_cast(detections[i * object_size_ + 5] * width_ - r.x); - r.height = - static_cast(detections[i * object_size_ + 6] * height_ - r.y); - Result result(r); - result.label_ = label_num < labels.size() - ? labels[label_num] - : std::string("label #") + std::to_string(label_num); - result.confidence_ = detections[i * object_size_ + 2]; - if (result.confidence_ <= show_output_thresh_) - { - continue; - } - - if (image_id < 0) - { - break; - } - found_result = true; - results_.emplace_back(result); - } - if (!found_result) results_.clear(); - - return true; -} - -const int dynamic_vino_lib::FaceDetection::getResultsLength() const -{ - return static_cast(results_.size()); -} - -const dynamic_vino_lib::Result* -dynamic_vino_lib::FaceDetection::getLocationResult(int idx) const -{ - return &(results_[idx]); -} - -const std::string dynamic_vino_lib::FaceDetection::getName() const -{ - return valid_model_->getModelName(); -} - -const void dynamic_vino_lib::FaceDetection::observeOutput( - const std::shared_ptr& output) -{ - if (output != nullptr) - { - output->accept(results_); - } -} diff --git a/dynamic_vino_lib/src/inferences/face_reidentification.cpp b/dynamic_vino_lib/src/inferences/face_reidentification.cpp new file mode 100644 index 00000000..c8f3d83b --- /dev/null +++ b/dynamic_vino_lib/src/inferences/face_reidentification.cpp @@ -0,0 +1,130 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with declaration of FaceReidentification class and + * FaceReidentificationResult class + * @file face_reidentification.cpp + */ +#include +#include +#include +#include "dynamic_vino_lib/inferences/face_reidentification.h" +#include "dynamic_vino_lib/outputs/base_output.h" +#include "dynamic_vino_lib/slog.h" + +// FaceReidentificationResult +dynamic_vino_lib::FaceReidentificationResult::FaceReidentificationResult(const cv::Rect& location) : Result(location) +{ +} + +// FaceReidentification +dynamic_vino_lib::FaceReidentification::FaceReidentification(double match_thresh) : dynamic_vino_lib::BaseInference() +{ + face_tracker_ = std::make_shared(1000, match_thresh, 0.3); +} + +dynamic_vino_lib::FaceReidentification::~FaceReidentification() = default; +void dynamic_vino_lib::FaceReidentification::loadNetwork( + const std::shared_ptr network) +{ + valid_model_ = network; + setMaxBatchSize(network->getMaxBatchSize()); +} + +bool dynamic_vino_lib::FaceReidentification::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) +{ + if (getEnqueuedNum() == 0) + { + results_.clear(); + } + if (!dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, 0, valid_model_->getInputName())) + { + return false; + } + Result r(input_frame_loc); + results_.emplace_back(r); + return true; +} + +bool dynamic_vino_lib::FaceReidentification::submitRequest() +{ + return dynamic_vino_lib::BaseInference::submitRequest(); +} + +bool dynamic_vino_lib::FaceReidentification::fetchResults() +{ + bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); + if (!can_fetch) + { + return false; + } + bool found_result = false; + InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + std::string output = valid_model_->getOutputName(); + const float* output_values = request->GetBlob(output)->buffer().as(); + int result_length = request->GetBlob(output)->getTensorDesc().getDims()[1]; + for (int i = 0; i < getResultsLength(); i++) + { + std::vector new_face = + std::vector(output_values + result_length * i, output_values + result_length * (i + 1)); + std::string face_id = "No." + std::to_string(face_tracker_->processNewTrack(new_face)); + results_[i].face_id_ = face_id; + found_result = true; + } + if (!found_result) + { + results_.clear(); + } + return true; +} + +int dynamic_vino_lib::FaceReidentification::getResultsLength() const +{ + return static_cast(results_.size()); +} + +const dynamic_vino_lib::Result* dynamic_vino_lib::FaceReidentification::getLocationResult(int idx) const +{ + return &(results_[idx]); +} + +const std::string dynamic_vino_lib::FaceReidentification::getName() const +{ + return valid_model_->getModelCategory(); +} + +void dynamic_vino_lib::FaceReidentification::observeOutput(const std::shared_ptr& output) +{ + if (output != nullptr) + { + output->accept(results_); + } +} + +const std::vector +dynamic_vino_lib::FaceReidentification::getFilteredROIs(const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "Face reidentification does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/dynamic_vino_lib/src/inferences/head_pose_detection.cpp b/dynamic_vino_lib/src/inferences/head_pose_detection.cpp index 490936cf..9b5e4d06 100644 --- a/dynamic_vino_lib/src/inferences/head_pose_detection.cpp +++ b/dynamic_vino_lib/src/inferences/head_pose_detection.cpp @@ -26,37 +26,33 @@ #include "dynamic_vino_lib/outputs/base_output.h" // HeadPoseResult -dynamic_vino_lib::HeadPoseResult::HeadPoseResult(const cv::Rect& location) - : Result(location) +dynamic_vino_lib::HeadPoseResult::HeadPoseResult(const cv::Rect& location) : Result(location) { } // Head Pose Detection -dynamic_vino_lib::HeadPoseDetection::HeadPoseDetection() - : dynamic_vino_lib::BaseInference() +dynamic_vino_lib::HeadPoseDetection::HeadPoseDetection() : dynamic_vino_lib::BaseInference() { } dynamic_vino_lib::HeadPoseDetection::~HeadPoseDetection() = default; -void dynamic_vino_lib::HeadPoseDetection::loadNetwork( - std::shared_ptr network) +void dynamic_vino_lib::HeadPoseDetection::loadNetwork(std::shared_ptr network) { valid_model_ = network; setMaxBatchSize(network->getMaxBatchSize()); } -bool dynamic_vino_lib::HeadPoseDetection::enqueue( - const cv::Mat& frame, const cv::Rect& input_frame_loc) +bool dynamic_vino_lib::HeadPoseDetection::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) { if (getEnqueuedNum() == 0) { results_.clear(); } - bool succeed = dynamic_vino_lib::BaseInference::enqueue( - frame, input_frame_loc, 1, getResultsLength(), - valid_model_->getInputName()); - if (!succeed) return false; + bool succeed = dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, getResultsLength(), + valid_model_->getInputName()); + if (!succeed) + return false; Result r(input_frame_loc); results_.emplace_back(r); return true; @@ -70,14 +66,12 @@ bool dynamic_vino_lib::HeadPoseDetection::submitRequest() bool dynamic_vino_lib::HeadPoseDetection::fetchResults() { bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); - if (!can_fetch) return false; + if (!can_fetch) + return false; auto request = getEngine()->getRequest(); - InferenceEngine::Blob::Ptr angle_r = - request->GetBlob(valid_model_->getOutputOutputAngleR()); - InferenceEngine::Blob::Ptr angle_p = - request->GetBlob(valid_model_->getOutputOutputAngleP()); - InferenceEngine::Blob::Ptr angle_y = - request->GetBlob(valid_model_->getOutputOutputAngleY()); + InferenceEngine::Blob::Ptr angle_r = request->GetBlob(valid_model_->getOutputOutputAngleR()); + InferenceEngine::Blob::Ptr angle_p = request->GetBlob(valid_model_->getOutputOutputAngleP()); + InferenceEngine::Blob::Ptr angle_y = request->GetBlob(valid_model_->getOutputOutputAngleY()); for (int i = 0; i < getResultsLength(); ++i) { @@ -88,27 +82,41 @@ bool dynamic_vino_lib::HeadPoseDetection::fetchResults() return true; } -const int dynamic_vino_lib::HeadPoseDetection::getResultsLength() const +int dynamic_vino_lib::HeadPoseDetection::getResultsLength() const { return static_cast(results_.size()); } -const dynamic_vino_lib::Result* -dynamic_vino_lib::HeadPoseDetection::getLocationResult(int idx) const +const dynamic_vino_lib::Result* dynamic_vino_lib::HeadPoseDetection::getLocationResult(int idx) const { return &(results_[idx]); } const std::string dynamic_vino_lib::HeadPoseDetection::getName() const { - return valid_model_->getModelName(); + return valid_model_->getModelCategory(); } -const void dynamic_vino_lib::HeadPoseDetection::observeOutput( - const std::shared_ptr& output) +void dynamic_vino_lib::HeadPoseDetection::observeOutput(const std::shared_ptr& output) { if (output != nullptr) { output->accept(results_); } } + +const std::vector +dynamic_vino_lib::HeadPoseDetection::getFilteredROIs(const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "Headpose detection does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/dynamic_vino_lib/src/inferences/landmarks_detection.cpp b/dynamic_vino_lib/src/inferences/landmarks_detection.cpp new file mode 100644 index 00000000..81e40ef7 --- /dev/null +++ b/dynamic_vino_lib/src/inferences/landmarks_detection.cpp @@ -0,0 +1,133 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with declaration of LandmarksDetection class and + * LandmarksDetectionResult class + * @file landmarks_detection.cpp + */ +#include +#include +#include +#include "dynamic_vino_lib/inferences/landmarks_detection.h" +#include "dynamic_vino_lib/outputs/base_output.h" + +// LandmarksDetectionResult +dynamic_vino_lib::LandmarksDetectionResult::LandmarksDetectionResult(const cv::Rect& location) : Result(location) +{ +} + +// LandmarksDetection +dynamic_vino_lib::LandmarksDetection::LandmarksDetection() : dynamic_vino_lib::BaseInference() +{ +} + +dynamic_vino_lib::LandmarksDetection::~LandmarksDetection() = default; +void dynamic_vino_lib::LandmarksDetection::loadNetwork(const std::shared_ptr network) +{ + valid_model_ = network; + setMaxBatchSize(network->getMaxBatchSize()); +} + +bool dynamic_vino_lib::LandmarksDetection::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) +{ + if (getEnqueuedNum() == 0) + { + results_.clear(); + } + if (!dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, 0, valid_model_->getInputName())) + { + return false; + } + Result r(input_frame_loc); + results_.emplace_back(r); + return true; +} + +bool dynamic_vino_lib::LandmarksDetection::submitRequest() +{ + return dynamic_vino_lib::BaseInference::submitRequest(); +} + +bool dynamic_vino_lib::LandmarksDetection::fetchResults() +{ + bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); + if (!can_fetch) + { + return false; + } + bool found_result = false; + InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + std::string output = valid_model_->getOutputName(); + const float* output_values = request->GetBlob(output)->buffer().as(); + int result_length = request->GetBlob(output)->getTensorDesc().getDims()[1]; + for (int i = 0; i < getResultsLength(); i++) + { + std::vector coordinates = + std::vector(output_values + result_length * i, output_values + result_length * (i + 1)); + for (int j = 0; j < result_length; j += 2) + { + cv::Rect rect = results_[i].getLocation(); + int col = static_cast(coordinates[j] * rect.width); + int row = static_cast(coordinates[j + 1] * rect.height); + cv::Point landmark_point(rect.x + col, rect.y + row); + results_[i].landmark_points_.push_back(landmark_point); + } + found_result = true; + } + if (!found_result) + { + results_.clear(); + } + return true; +} + +int dynamic_vino_lib::LandmarksDetection::getResultsLength() const +{ + return static_cast(results_.size()); +} + +const dynamic_vino_lib::Result* dynamic_vino_lib::LandmarksDetection::getLocationResult(int idx) const +{ + return &(results_[idx]); +} + +const std::string dynamic_vino_lib::LandmarksDetection::getName() const +{ + return valid_model_->getModelCategory(); +} + +void dynamic_vino_lib::LandmarksDetection::observeOutput(const std::shared_ptr& output) +{ + if (output != nullptr) + { + output->accept(results_); + } +} + +const std::vector +dynamic_vino_lib::LandmarksDetection::getFilteredROIs(const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "Landmarks detection does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/dynamic_vino_lib/src/inferences/license_plate_detection.cpp b/dynamic_vino_lib/src/inferences/license_plate_detection.cpp new file mode 100644 index 00000000..76e43e72 --- /dev/null +++ b/dynamic_vino_lib/src/inferences/license_plate_detection.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a realization file with declaration of LicensePlateDetection class and + * LicensePlateDetectionResult class + * @file license_plate_detection.cpp + */ +#include +#include +#include +#include "dynamic_vino_lib/inferences/license_plate_detection.h" +#include "dynamic_vino_lib/outputs/base_output.h" +#include "dynamic_vino_lib/slog.h" + +// LicensePlateDetectionResult +dynamic_vino_lib::LicensePlateDetectionResult::LicensePlateDetectionResult(const cv::Rect& location) : Result(location) +{ +} + +// LicensePlateDetection +dynamic_vino_lib::LicensePlateDetection::LicensePlateDetection() : dynamic_vino_lib::BaseInference() +{ +} + +dynamic_vino_lib::LicensePlateDetection::~LicensePlateDetection() = default; +void dynamic_vino_lib::LicensePlateDetection::loadNetwork( + const std::shared_ptr network) +{ + valid_model_ = network; + setMaxBatchSize(network->getMaxBatchSize()); +} + +void dynamic_vino_lib::LicensePlateDetection::fillSeqBlob() +{ + InferenceEngine::Blob::Ptr seq_blob = getEngine()->getRequest()->GetBlob(valid_model_->getSeqInputName()); + int max_sequence_size = seq_blob->getTensorDesc().getDims()[0]; + // second input is sequence, which is some relic from the training + // it should have the leading 0.0f and rest 1.0f + float* blob_data = seq_blob->buffer().as(); + blob_data[0] = 0.0f; + std::fill(blob_data + 1, blob_data + max_sequence_size, 1.0f); +} + +bool dynamic_vino_lib::LicensePlateDetection::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) +{ + if (getEnqueuedNum() == 0) + { + results_.clear(); + } + if (!dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, 0, valid_model_->getInputName())) + { + return false; + } + fillSeqBlob(); + Result r(input_frame_loc); + results_.emplace_back(r); + return true; +} + +bool dynamic_vino_lib::LicensePlateDetection::submitRequest() +{ + return dynamic_vino_lib::BaseInference::submitRequest(); +} + +bool dynamic_vino_lib::LicensePlateDetection::fetchResults() +{ + bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); + if (!can_fetch) + { + return false; + } + bool found_result = false; + InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + std::string output = valid_model_->getOutputName(); + const float* output_values = request->GetBlob(output)->buffer().as(); + for (int i = 0; i < getResultsLength(); i++) + { + std::string license = ""; + int max_size = valid_model_->getMaxSequenceSize(); + for (int j = 0; j < max_size; j++) + { + if (output_values[i * max_size + j] == -1) + { + break; + } + license += licenses_[output_values[i * max_size + j]]; + } + results_[i].license_ = license; + found_result = true; + } + if (!found_result) + { + results_.clear(); + } + return true; +} + +int dynamic_vino_lib::LicensePlateDetection::getResultsLength() const +{ + return static_cast(results_.size()); +} + +const dynamic_vino_lib::Result* dynamic_vino_lib::LicensePlateDetection::getLocationResult(int idx) const +{ + return &(results_[idx]); +} + +const std::string dynamic_vino_lib::LicensePlateDetection::getName() const +{ + return valid_model_->getModelCategory(); +} + +void dynamic_vino_lib::LicensePlateDetection::observeOutput(const std::shared_ptr& output) +{ + if (output != nullptr) + { + output->accept(results_); + } +} + +const std::vector +dynamic_vino_lib::LicensePlateDetection::getFilteredROIs(const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "License plate detection does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/dynamic_vino_lib/src/inferences/object_detection.cpp b/dynamic_vino_lib/src/inferences/object_detection.cpp index f25259bc..46f0559d 100644 --- a/dynamic_vino_lib/src/inferences/object_detection.cpp +++ b/dynamic_vino_lib/src/inferences/object_detection.cpp @@ -21,92 +21,165 @@ #include #include #include +#include +#include #include "dynamic_vino_lib/inferences/object_detection.h" #include "dynamic_vino_lib/outputs/base_output.h" #include "dynamic_vino_lib/slog.h" // ObjectDetectionResult -dynamic_vino_lib::ObjectDetectionResult::ObjectDetectionResult( - const cv::Rect& location) - : Result(location){} +dynamic_vino_lib::ObjectDetectionResult::ObjectDetectionResult(const cv::Rect& location) : Result(location) +{ +} + // ObjectDetection -dynamic_vino_lib::ObjectDetection::ObjectDetection(bool enable_roi_constraint, - double show_output_thresh) - : dynamic_vino_lib::BaseInference(), - show_output_thresh_(show_output_thresh){} +dynamic_vino_lib::ObjectDetection::ObjectDetection(bool enable_roi_constraint, double show_output_thresh) + : show_output_thresh_(show_output_thresh) + , enable_roi_constraint_(enable_roi_constraint) + , dynamic_vino_lib::BaseInference() +{ + result_filter_ = std::make_shared(); + result_filter_->init(); +} + dynamic_vino_lib::ObjectDetection::~ObjectDetection() = default; -void dynamic_vino_lib::ObjectDetection::loadNetwork( - const std::shared_ptr network) { + +void dynamic_vino_lib::ObjectDetection::loadNetwork(std::shared_ptr network) +{ valid_model_ = network; - max_proposal_count_ = network->getMaxProposalCount(); - object_size_ = network->getObjectSize(); + setMaxBatchSize(network->getMaxBatchSize()); } -bool dynamic_vino_lib::ObjectDetection::enqueue(const cv::Mat& frame, - const cv::Rect& input_frame_loc) { - if (width_ == 0 && height_ == 0) { - width_ = frame.cols; - height_ = frame.rows; +bool dynamic_vino_lib::ObjectDetection::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) +{ + if (valid_model_ == nullptr || getEngine() == nullptr) + { + return false; } - if (!dynamic_vino_lib::BaseInference::enqueue( - frame, input_frame_loc, 1, 0, valid_model_->getInputName())) { + + if (enqueued_frames_ >= valid_model_->getMaxBatchSize()) + { + slog::warn << "Number of " << getName() << "input more than maximum(" << max_batch_size_ + << ") processed by inference" << slog::endl; return false; } - Result r(input_frame_loc); - results_.clear(); - results_.emplace_back(r); + + if (!valid_model_->enqueue(getEngine(), frame, input_frame_loc)) + { + return false; + } + + // nonsense!! + // Result r(input_frame_loc); + // results_.clear(); + // results_.emplace_back(r); + enqueued_frames_ += 1; return true; } -bool dynamic_vino_lib::ObjectDetection::submitRequest() { - return dynamic_vino_lib::BaseInference::submitRequest(); -} -bool dynamic_vino_lib::ObjectDetection::fetchResults() { + +bool dynamic_vino_lib::ObjectDetection::fetchResults() +{ bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); - if (!can_fetch) return false; - bool found_result = false; - results_.clear(); - InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); - std::string output = valid_model_->getOutputName(); - const float* detections = request->GetBlob(output)->buffer().as(); - for (int i = 0; i < max_proposal_count_; i++) { - float image_id = detections[i * object_size_ + 0]; - cv::Rect r; - auto label_num = static_cast(detections[i * object_size_ + 1]); - std::vector& labels = valid_model_->getLabels(); - r.x = static_cast(detections[i * object_size_ + 3] * width_); - r.y = static_cast(detections[i * object_size_ + 4] * height_); - r.width = static_cast(detections[i * object_size_ + 5] * width_ - r.x); - r.height = - static_cast(detections[i * object_size_ + 6] * height_ - r.y); - Result result(r); - result.label_ = label_num < labels.size() - ? labels[label_num] - : std::string("label #") + std::to_string(label_num); - result.confidence_ = detections[i * object_size_ + 2]; - if (result.confidence_ <= show_output_thresh_) { - continue; - } - if (image_id < 0) { - break; - } - found_result = true; - results_.emplace_back(result); + if (!can_fetch) + { + return false; } - if (!found_result) results_.clear(); - return true; + + results_.clear(); + + return (valid_model_ != nullptr) && + valid_model_->fetchResults(getEngine(), results_, show_output_thresh_, enable_roi_constraint_); } -const int dynamic_vino_lib::ObjectDetection::getResultsLength() const { + +int dynamic_vino_lib::ObjectDetection::getResultsLength() const +{ return static_cast(results_.size()); } -const dynamic_vino_lib::Result* -dynamic_vino_lib::ObjectDetection::getLocationResult(int idx) const { + +const dynamic_vino_lib::ObjectDetection::Result* dynamic_vino_lib::ObjectDetection::getLocationResult(int idx) const +{ return &(results_[idx]); } -const std::string dynamic_vino_lib::ObjectDetection::getName() const { - return valid_model_->getModelName(); + +const std::string dynamic_vino_lib::ObjectDetection::getName() const +{ + return valid_model_->getModelCategory(); } -const void dynamic_vino_lib::ObjectDetection::observeOutput( - const std::shared_ptr& output) { - if (output != nullptr) { + +void dynamic_vino_lib::ObjectDetection::observeOutput(const std::shared_ptr& output) +{ + if (output != nullptr) + { output->accept(results_); } } + +const std::vector +dynamic_vino_lib::ObjectDetection::getFilteredROIs(const std::string filter_conditions) const +{ + if (!result_filter_->isValidFilterConditions(filter_conditions)) + { + std::vector filtered_rois; + for (auto result : results_) + { + filtered_rois.push_back(result.getLocation()); + } + return filtered_rois; + } + result_filter_->acceptResults(results_); + result_filter_->acceptFilterConditions(filter_conditions); + return result_filter_->getFilteredLocations(); +} + +// ObjectDetectionResultFilter +dynamic_vino_lib::ObjectDetectionResultFilter::ObjectDetectionResultFilter() +{ +} + +void dynamic_vino_lib::ObjectDetectionResultFilter::init() +{ + key_to_function_.insert(std::make_pair("label", isValidLabel)); + key_to_function_.insert(std::make_pair("confidence", isValidConfidence)); +} + +void dynamic_vino_lib::ObjectDetectionResultFilter::acceptResults(const std::vector& results) +{ + results_ = results; +} + +std::vector dynamic_vino_lib::ObjectDetectionResultFilter::getFilteredLocations() +{ + std::vector locations; + for (auto result : results_) + { + if (isValidResult(result)) + { + locations.push_back(result.getLocation()); + } + } + return locations; +} + +bool dynamic_vino_lib::ObjectDetectionResultFilter::isValidLabel(const Result& result, const std::string& op, + const std::string& target) +{ + return stringCompare(result.getLabel(), op, target); +} + +bool dynamic_vino_lib::ObjectDetectionResultFilter::isValidConfidence(const Result& result, const std::string& op, + const std::string& target) +{ + return floatCompare(result.getConfidence(), op, stringToFloat(target)); +} + +bool dynamic_vino_lib::ObjectDetectionResultFilter::isValidResult(const Result& result) +{ + ISVALIDRESULT(key_to_function_, result); +} + +double dynamic_vino_lib::ObjectDetection::calcIoU(const cv::Rect& box_1, const cv::Rect& box_2) +{ + cv::Rect i = box_1 & box_2; + cv::Rect u = box_1 | box_2; + + return static_cast(i.area()) / static_cast(u.area()); +} diff --git a/dynamic_vino_lib/src/inferences/object_segmentation.cpp b/dynamic_vino_lib/src/inferences/object_segmentation.cpp index 69b7fd23..0dab5d42 100644 --- a/dynamic_vino_lib/src/inferences/object_segmentation.cpp +++ b/dynamic_vino_lib/src/inferences/object_segmentation.cpp @@ -21,42 +21,44 @@ #include #include #include +#include #include "dynamic_vino_lib/inferences/object_segmentation.h" #include "dynamic_vino_lib/outputs/base_output.h" #include "dynamic_vino_lib/slog.h" // ObjectSegmentationResult -dynamic_vino_lib::ObjectSegmentationResult::ObjectSegmentationResult(const cv::Rect & location) -: Result(location) +dynamic_vino_lib::ObjectSegmentationResult::ObjectSegmentationResult(const cv::Rect& location) : Result(location) { } // ObjectSegmentation dynamic_vino_lib::ObjectSegmentation::ObjectSegmentation(double show_output_thresh) -: show_output_thresh_(show_output_thresh), dynamic_vino_lib::BaseInference() + : show_output_thresh_(show_output_thresh), dynamic_vino_lib::BaseInference() { } dynamic_vino_lib::ObjectSegmentation::~ObjectSegmentation() = default; -void dynamic_vino_lib::ObjectSegmentation::loadNetwork( - const std::shared_ptr network) +void dynamic_vino_lib::ObjectSegmentation::loadNetwork(const std::shared_ptr network) { + slog::info << "Loading Network: " << network->getModelCategory() << slog::endl; valid_model_ = network; setMaxBatchSize(network->getMaxBatchSize()); } -bool dynamic_vino_lib::ObjectSegmentation::enqueue( - const cv::Mat & frame, - const cv::Rect & input_frame_loc) +/** + * Deprecated! + * This function only support OpenVINO version <=2018R5 + */ +bool dynamic_vino_lib::ObjectSegmentation::enqueue_for_one_input(const cv::Mat& frame, const cv::Rect& input_frame_loc) { - if (width_ == 0 && height_ == 0) { + if (width_ == 0 && height_ == 0) + { width_ = frame.cols; height_ = frame.rows; } - if (!dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, 0, - valid_model_->getInputName())) + if (!dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, 0, valid_model_->getInputName())) { return false; } @@ -66,6 +68,36 @@ bool dynamic_vino_lib::ObjectSegmentation::enqueue( return true; } +bool dynamic_vino_lib::ObjectSegmentation::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) +{ + if (width_ == 0 && height_ == 0) + { + width_ = frame.cols; + height_ = frame.rows; + } + + if (valid_model_ == nullptr || getEngine() == nullptr) + { + throw std::logic_error("Model or Engine is not set correctly!"); + return false; + } + + if (enqueued_frames_ >= valid_model_->getMaxBatchSize()) + { + slog::warn << "Number of " << getName() << "input more than maximum(" << getMaxBatchSize() + << ") processed by inference" << slog::endl; + return false; + } + + if (!valid_model_->enqueue(getEngine(), frame, input_frame_loc)) + { + return false; + } + + enqueued_frames_ += 1; + return true; +} + bool dynamic_vino_lib::ObjectSegmentation::submitRequest() { return dynamic_vino_lib::BaseInference::submitRequest(); @@ -74,81 +106,126 @@ bool dynamic_vino_lib::ObjectSegmentation::submitRequest() bool dynamic_vino_lib::ObjectSegmentation::fetchResults() { bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); - if (!can_fetch) { + if (!can_fetch) + { return false; } bool found_result = false; results_.clear(); InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); - std::string detection_output = valid_model_->getDetectionOutputName(); - std::string mask_output = valid_model_->getMaskOutputName(); - const auto do_blob = request->GetBlob(detection_output.c_str()); - const auto do_data = do_blob->buffer().as(); + slog::debug << "Analyzing Detection results..." << slog::endl; + std::string detection_output = valid_model_->getOutputName("detection"); + std::string mask_output = valid_model_->getOutputName("masks"); + + const InferenceEngine::Blob::Ptr do_blob = request->GetBlob(detection_output.c_str()); + const auto do_data = do_blob->buffer().as(); const auto masks_blob = request->GetBlob(mask_output.c_str()); - const auto masks_data = masks_blob->buffer().as(); - // amount of elements in each detected box description (batch, label, prob, x1, y1, x2, y2) - size_t box_num = masks_blob->dims().at(3); - size_t label_num = masks_blob->dims().at(2); - size_t box_description_size = do_blob->dims().at(0); - size_t H = masks_blob->dims().at(1); - size_t W = masks_blob->dims().at(0); - size_t box_stride = W * H * label_num; - for (size_t box = 0; box < box_num; ++box) { - float * box_info = do_data + box * box_description_size; - float batch = box_info[0]; - if (batch < 0) { - break; - } - float prob = box_info[2]; - if (prob > show_output_thresh_) { - float x1 = std::min(std::max(0.0f, box_info[3] * width_), static_cast(width_)); - float y1 = std::min(std::max(0.0f, box_info[4] * height_), static_cast(height_)); - float x2 = std::min(std::max(0.0f, box_info[5] * width_), static_cast(width_)); - float y2 = std::min(std::max(0.0f, box_info[6] * height_), static_cast(height_)); - int box_width = std::min(static_cast(std::max(0.0f, x2 - x1)), width_); - int box_height = std::min(static_cast(std::max(0.0f, y2 - y1)), height_); - int class_id = static_cast(box_info[1] + 1e-6f); - float * mask_arr = masks_data + box_stride * box + H * W * (class_id - 1); - cv::Mat mask_mat(H, W, CV_32FC1, mask_arr); - cv::Rect roi = cv::Rect(static_cast(x1), static_cast(y1), box_width, box_height); - cv::Mat resized_mask_mat(box_height, box_width, CV_32FC1); - cv::resize(mask_mat, resized_mask_mat, cv::Size(box_width, box_height)); - Result result(roi); - result.confidence_ = prob; - std::vector & labels = valid_model_->getLabels(); - result.label_ = class_id < labels.size() ? labels[class_id] : - std::string("label #") + std::to_string(class_id); - result.mask_ = resized_mask_mat; - found_result = true; - results_.emplace_back(result); + const auto masks_data = masks_blob->buffer().as(); + const size_t output_w = masks_blob->getTensorDesc().getDims().at(3); + const size_t output_h = masks_blob->getTensorDesc().getDims().at(2); + const size_t output_des = masks_blob->getTensorDesc().getDims().at(1); + const size_t output_extra = masks_blob->getTensorDesc().getDims().at(0); + + slog::debug << "output w " << output_w << slog::endl; + slog::debug << "output h " << output_h << slog::endl; + slog::debug << "output description " << output_des << slog::endl; + slog::debug << "output extra " << output_extra << slog::endl; + + const float* detections = request->GetBlob(detection_output)->buffer().as(); + std::vector& labels = valid_model_->getLabels(); + slog::debug << "label size " << labels.size() << slog::endl; + + cv::Mat inImg, resImg, maskImg(output_h, output_w, CV_8UC3); + cv::Mat colored_mask(output_h, output_w, CV_8UC3); + cv::Rect roi = cv::Rect(0, 0, output_w, output_h); + + for (int rowId = 0; rowId < output_h; ++rowId) + { + for (int colId = 0; colId < output_w; ++colId) + { + std::size_t classId = 0; + float maxProb = -1.0f; + if (output_des < 2) + { // assume the output is already ArgMax'ed + classId = static_cast(detections[rowId * output_w + colId]); + for (int ch = 0; ch < colored_mask.channels(); ++ch) + { + colored_mask.at(rowId, colId)[ch] = colors_[classId][ch]; + } + // classId = static_cast(predictions[rowId * output_w + colId]); + } + else + { + for (int chId = 0; chId < output_des; ++chId) + { + float prob = detections[chId * output_h * output_w + rowId * output_w + colId]; + // float prob = predictions[chId * output_h * output_w + rowId * output_w+ colId]; + if (prob > maxProb) + { + classId = chId; + maxProb = prob; + } + } + while (classId >= colors_.size()) + { + static std::mt19937 rng(classId); + std::uniform_int_distribution distr(0, 255); + cv::Vec3b color(distr(rng), distr(rng), distr(rng)); + colors_.push_back(color); + } + if (maxProb > 0.5) + { + for (int ch = 0; ch < colored_mask.channels(); ++ch) + { + colored_mask.at(rowId, colId)[ch] = colors_[classId][ch]; + } + } + } } } - if (!found_result) { - results_.clear(); - } + const float alpha = 0.7f; + Result result(roi); + result.mask_ = colored_mask; + found_result = true; + results_.emplace_back(result); return true; } -const int dynamic_vino_lib::ObjectSegmentation::getResultsLength() const +int dynamic_vino_lib::ObjectSegmentation::getResultsLength() const { return static_cast(results_.size()); } -const dynamic_vino_lib::Result * -dynamic_vino_lib::ObjectSegmentation::getLocationResult(int idx) const +const dynamic_vino_lib::Result* dynamic_vino_lib::ObjectSegmentation::getLocationResult(int idx) const { return &(results_[idx]); } const std::string dynamic_vino_lib::ObjectSegmentation::getName() const { - return valid_model_->getModelName(); + return valid_model_->getModelCategory(); } -const void dynamic_vino_lib::ObjectSegmentation::observeOutput( - const std::shared_ptr & output) +void dynamic_vino_lib::ObjectSegmentation::observeOutput(const std::shared_ptr& output) { - if (output != nullptr) { + if (output != nullptr) + { output->accept(results_); } } + +const std::vector +dynamic_vino_lib::ObjectSegmentation::getFilteredROIs(const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "Object segmentation does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/dynamic_vino_lib/src/inferences/person_attribs_detection.cpp b/dynamic_vino_lib/src/inferences/person_attribs_detection.cpp new file mode 100644 index 00000000..24200cb9 --- /dev/null +++ b/dynamic_vino_lib/src/inferences/person_attribs_detection.cpp @@ -0,0 +1,176 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with declaration of PersonAttribsDetection class and + * PersonAttribsDetectionResult class + * @file person_attribs_detection.cpp + */ +#include +#include +#include +#include "dynamic_vino_lib/inferences/person_attribs_detection.h" +#include "dynamic_vino_lib/outputs/base_output.h" +#include "dynamic_vino_lib/slog.h" + +// PersonAttribsDetectionResult +dynamic_vino_lib::PersonAttribsDetectionResult::PersonAttribsDetectionResult(const cv::Rect& location) + : Result(location) +{ +} + +// PersonAttribsDetection +dynamic_vino_lib::PersonAttribsDetection::PersonAttribsDetection(double attribs_confidence) + : attribs_confidence_(attribs_confidence), dynamic_vino_lib::BaseInference() +{ +} + +dynamic_vino_lib::PersonAttribsDetection::~PersonAttribsDetection() = default; +void dynamic_vino_lib::PersonAttribsDetection::loadNetwork( + const std::shared_ptr network) +{ + valid_model_ = network; + setMaxBatchSize(network->getMaxBatchSize()); +} + +bool dynamic_vino_lib::PersonAttribsDetection::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) +{ + if (getEnqueuedNum() == 0) + { + results_.clear(); + } + if (!dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, 0, valid_model_->getInputName())) + { + return false; + } + Result r(input_frame_loc); + results_.emplace_back(r); + return true; +} + +bool dynamic_vino_lib::PersonAttribsDetection::submitRequest() +{ + return dynamic_vino_lib::BaseInference::submitRequest(); +} +/* +bool dynamic_vino_lib::PersonAttribsDetection::fetchResults() +{ + bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); + if (!can_fetch) {return false;} + bool found_result = false; + InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + std::string output = valid_model_->getOutputName(); + const float * output_values = request->GetBlob(output)->buffer().as(); + int net_attrib_length = net_attributes_.size(); + for (int i = 0; i < getResultsLength(); i++) { + results_[i].male_probability_ = output_values[i * net_attrib_length]; + std::string attrib = ""; + for (int j = 1; j < net_attrib_length; j++) { + attrib += (output_values[i * net_attrib_length + j] > attribs_confidence_) ? + net_attributes_[j] + ", " : ""; + } + results_[i].attributes_ = attrib; + found_result = true; + } + if (!found_result) {results_.clear();} + return true; +} +*/ +bool dynamic_vino_lib::PersonAttribsDetection::fetchResults() +{ + bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); + if (!can_fetch) + { + return false; + } + bool found_result = false; + InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + slog::debug << "Analyzing Attributes Detection results..." << slog::endl; + std::string attribute_output = valid_model_->getOutputName("attributes_output_"); + std::string top_output = valid_model_->getOutputName("top_output_"); + std::string bottom_output = valid_model_->getOutputName("bottom_output_"); + + /*auto attri_values = request->GetBlob(attribute_output)->buffer().as(); + auto top_values = request->GetBlob(top_output)->buffer().as(); + auto bottom_values = request->GetBlob(bottom_output)->buffer().as();*/ + InferenceEngine::Blob::Ptr attribBlob = request->GetBlob(attribute_output); + InferenceEngine::Blob::Ptr topBlob = request->GetBlob(top_output); + InferenceEngine::Blob::Ptr bottomBlob = request->GetBlob(bottom_output); + + auto attri_values = attribBlob->buffer().as(); + auto top_values = topBlob->buffer().as(); + auto bottom_values = bottomBlob->buffer().as(); + + int net_attrib_length = net_attributes_.size(); + for (int i = 0; i < getResultsLength(); i++) + { + results_[i].male_probability_ = attri_values[i * net_attrib_length]; + results_[i].top_point_.x = top_values[i]; + results_[i].top_point_.y = top_values[i + 1]; + results_[i].bottom_point_.x = bottom_values[i]; + results_[i].bottom_point_.y = bottom_values[i + 1]; + std::string attrib = ""; + for (int j = 1; j < net_attrib_length; j++) + { + attrib += (attri_values[i * net_attrib_length + j] > attribs_confidence_) ? net_attributes_[j] + ", " : ""; + } + results_[i].attributes_ = attrib; + + found_result = true; + } + if (!found_result) + { + results_.clear(); + } + return true; +} + +int dynamic_vino_lib::PersonAttribsDetection::getResultsLength() const +{ + return static_cast(results_.size()); +} + +const dynamic_vino_lib::Result* dynamic_vino_lib::PersonAttribsDetection::getLocationResult(int idx) const +{ + return &(results_[idx]); +} + +const std::string dynamic_vino_lib::PersonAttribsDetection::getName() const +{ + return valid_model_->getModelCategory(); +} + +void dynamic_vino_lib::PersonAttribsDetection::observeOutput(const std::shared_ptr& output) +{ + if (output != nullptr) + { + output->accept(results_); + } +} + +const std::vector +dynamic_vino_lib::PersonAttribsDetection::getFilteredROIs(const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "Person attributes detection does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/dynamic_vino_lib/src/inferences/person_reidentification.cpp b/dynamic_vino_lib/src/inferences/person_reidentification.cpp index 005a128d..772c63ed 100644 --- a/dynamic_vino_lib/src/inferences/person_reidentification.cpp +++ b/dynamic_vino_lib/src/inferences/person_reidentification.cpp @@ -25,30 +25,33 @@ #include "dynamic_vino_lib/slog.h" // PersonReidentificationResult -dynamic_vino_lib::PersonReidentificationResult::PersonReidentificationResult( - const cv::Rect & location) -: Result(location) {} +dynamic_vino_lib::PersonReidentificationResult::PersonReidentificationResult(const cv::Rect& location) + : Result(location) +{ +} // PersonReidentification dynamic_vino_lib::PersonReidentification::PersonReidentification(double match_thresh) -: match_thresh_(match_thresh), dynamic_vino_lib::BaseInference() {} + : dynamic_vino_lib::BaseInference() +{ + person_tracker_ = std::make_shared(1000, match_thresh, 0.3); +} dynamic_vino_lib::PersonReidentification::~PersonReidentification() = default; void dynamic_vino_lib::PersonReidentification::loadNetwork( - const std::shared_ptr network) + const std::shared_ptr network) { valid_model_ = network; setMaxBatchSize(network->getMaxBatchSize()); } -bool dynamic_vino_lib::PersonReidentification::enqueue( - const cv::Mat & frame, const cv::Rect & input_frame_loc) +bool dynamic_vino_lib::PersonReidentification::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) { - if (getEnqueuedNum() == 0) { + if (getEnqueuedNum() == 0) + { results_.clear(); } - if (!dynamic_vino_lib::BaseInference::enqueue( - frame, input_frame_loc, 1, 0, valid_model_->getInputName())) + if (!dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, 0, valid_model_->getInputName())) { return false; } @@ -65,89 +68,63 @@ bool dynamic_vino_lib::PersonReidentification::submitRequest() bool dynamic_vino_lib::PersonReidentification::fetchResults() { bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); - if (!can_fetch) {return false;} + if (!can_fetch) + { + return false; + } bool found_result = false; InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); std::string output = valid_model_->getOutputName(); - const float * output_values = request->GetBlob(output)->buffer().as(); - for (int i = 0; i < getResultsLength(); i++) { - std::vector new_person = std::vector( - output_values + 256 * i, output_values + 256 * i + 256); - std::string person_id = findMatchPerson(new_person); + const float* output_values = request->GetBlob(output)->buffer().as(); + for (int i = 0; i < getResultsLength(); i++) + { + std::vector new_person = std::vector(output_values + 256 * i, output_values + 256 * i + 256); + std::string person_id = "No." + std::to_string(person_tracker_->processNewTrack(new_person)); results_[i].person_id_ = person_id; found_result = true; } - if (!found_result) {results_.clear();} - return true; -} - -float dynamic_vino_lib::PersonReidentification::calcSimilarity( - const std::vector & person_a, const std::vector & person_b) -{ - if (person_a.size() != person_b.size()) { - throw std::logic_error("cosine similarity can't be called for vectors of different lengths: " - "person_a size = " + std::to_string(person_a.size()) + - "person_b size = " + std::to_string(person_b.size())); - } - float mul_sum, denom_a, denom_b, value_a, value_b; - mul_sum = denom_a = denom_b = value_a = value_b = 0; - for (auto i = 0; i < person_a.size(); i++) { - value_a = person_a[i]; - value_b = person_b[i]; - mul_sum += value_a * value_b; - denom_a += value_a * value_a; - denom_b += value_b * value_b; - } - if (denom_a == 0 || denom_b == 0) { - throw std::logic_error("cosine similarity is not defined whenever one or both " - "input vectors are zero-vectors."); - } - return mul_sum / (sqrt(denom_a) * sqrt(denom_b)); -} - -std::string dynamic_vino_lib::PersonReidentification::findMatchPerson( - const std::vector & new_person) -{ - auto size = recorded_persons_.size(); - std::string id = "No."; - float best_match_sim = 0; - int best_match_ind = -1; - for (auto i = 0; i < size; ++i) { - float cos_sim = calcSimilarity(new_person, recorded_persons_[i]); - if (cos_sim > best_match_sim) { - best_match_sim = cos_sim; - best_match_ind = i; - } - } - if (best_match_sim > match_thresh_) { - recorded_persons_[best_match_ind] = new_person; - return id + std::to_string(best_match_ind); - } else { - recorded_persons_.push_back(new_person); - return id + std::to_string(size); + if (!found_result) + { + results_.clear(); } + return true; } -const int dynamic_vino_lib::PersonReidentification::getResultsLength() const +int dynamic_vino_lib::PersonReidentification::getResultsLength() const { return static_cast(results_.size()); } -const dynamic_vino_lib::Result * -dynamic_vino_lib::PersonReidentification::getLocationResult(int idx) const +const dynamic_vino_lib::Result* dynamic_vino_lib::PersonReidentification::getLocationResult(int idx) const { return &(results_[idx]); } const std::string dynamic_vino_lib::PersonReidentification::getName() const { - return valid_model_->getModelName(); + return valid_model_->getModelCategory(); } -const void dynamic_vino_lib::PersonReidentification::observeOutput( - const std::shared_ptr & output) +void dynamic_vino_lib::PersonReidentification::observeOutput(const std::shared_ptr& output) { - if (output != nullptr) { + if (output != nullptr) + { output->accept(results_); } } + +const std::vector +dynamic_vino_lib::PersonReidentification::getFilteredROIs(const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "Person reidentification does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/dynamic_vino_lib/src/inferences/vehicle_attribs_detection.cpp b/dynamic_vino_lib/src/inferences/vehicle_attribs_detection.cpp new file mode 100644 index 00000000..bba84f33 --- /dev/null +++ b/dynamic_vino_lib/src/inferences/vehicle_attribs_detection.cpp @@ -0,0 +1,135 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a realization file with declaration of VehicleAttribsDetection class and + * VehicleAttribsDetectionResult class + * @file vehicle_attribs_detection.cpp + */ +#include +#include +#include +#include "dynamic_vino_lib/inferences/vehicle_attribs_detection.h" +#include "dynamic_vino_lib/outputs/base_output.h" +#include "dynamic_vino_lib/slog.h" + +// VehicleAttribsDetectionResult +dynamic_vino_lib::VehicleAttribsDetectionResult::VehicleAttribsDetectionResult(const cv::Rect& location) + : Result(location) +{ +} + +// VehicleAttribsDetection +dynamic_vino_lib::VehicleAttribsDetection::VehicleAttribsDetection() : dynamic_vino_lib::BaseInference() +{ +} + +dynamic_vino_lib::VehicleAttribsDetection::~VehicleAttribsDetection() = default; +void dynamic_vino_lib::VehicleAttribsDetection::loadNetwork( + const std::shared_ptr network) +{ + valid_model_ = network; + setMaxBatchSize(network->getMaxBatchSize()); +} + +bool dynamic_vino_lib::VehicleAttribsDetection::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) +{ + if (getEnqueuedNum() == 0) + { + results_.clear(); + } + if (!dynamic_vino_lib::BaseInference::enqueue(frame, input_frame_loc, 1, 0, valid_model_->getInputName())) + { + return false; + } + Result r(input_frame_loc); + results_.emplace_back(r); + return true; +} + +bool dynamic_vino_lib::VehicleAttribsDetection::submitRequest() +{ + return dynamic_vino_lib::BaseInference::submitRequest(); +} + +bool dynamic_vino_lib::VehicleAttribsDetection::fetchResults() +{ + bool can_fetch = dynamic_vino_lib::BaseInference::fetchResults(); + if (!can_fetch) + { + return false; + } + bool found_result = false; + InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + // std::string color_name = valid_model_->getColorOutputName(); + // std::string type_name = valid_model_->getTypeOutputName(); + std::string color_name = valid_model_->getOutputName("color_output_"); + std::string type_name = valid_model_->getOutputName("type_output_"); + const float* color_values = request->GetBlob(color_name)->buffer().as(); + const float* type_values = request->GetBlob(type_name)->buffer().as(); + for (int i = 0; i < getResultsLength(); i++) + { + auto color_id = std::max_element(color_values, color_values + 7) - color_values; + auto type_id = std::max_element(type_values, type_values + 4) - type_values; + color_values += 7; + type_values += 4; + results_[i].color_ = colors_[color_id]; + results_[i].type_ = types_[type_id]; + found_result = true; + } + if (!found_result) + { + results_.clear(); + } + return true; +} + +int dynamic_vino_lib::VehicleAttribsDetection::getResultsLength() const +{ + return static_cast(results_.size()); +} + +const dynamic_vino_lib::Result* dynamic_vino_lib::VehicleAttribsDetection::getLocationResult(int idx) const +{ + return &(results_[idx]); +} + +const std::string dynamic_vino_lib::VehicleAttribsDetection::getName() const +{ + return valid_model_->getModelCategory(); +} + +void dynamic_vino_lib::VehicleAttribsDetection::observeOutput(const std::shared_ptr& output) +{ + if (output != nullptr) + { + output->accept(results_); + } +} + +const std::vector +dynamic_vino_lib::VehicleAttribsDetection::getFilteredROIs(const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "Vehicle attributes detection does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/dynamic_vino_lib/src/inputs/image_input.cpp b/dynamic_vino_lib/src/inputs/image_input.cpp index c2790f0e..f59070b0 100644 --- a/dynamic_vino_lib/src/inputs/image_input.cpp +++ b/dynamic_vino_lib/src/inputs/image_input.cpp @@ -20,6 +20,7 @@ */ #include "dynamic_vino_lib/inputs/image_input.h" +#include "dynamic_vino_lib/slog.h" #include // Image @@ -30,7 +31,6 @@ Input::Image::Image(const std::string& file) bool Input::Image::initialize() { - setFrameID("image_frame"); image_ = cv::imread(file_); if (image_.data != NULL) { @@ -52,10 +52,16 @@ bool Input::Image::read(cv::Mat* frame) return false; } *frame = image_; + // setHeader("image_frame"); return true; } -void Input::Image::config() +void Input::Image::config(const Input::Config& config) { - // TODO(weizhi): config + if (config.path != "") + { + file_.assign(config.path); + initialize(); + slog::info << "Image Input device was reinitialized with new file:" << config.path.c_str() << slog::endl; + } } diff --git a/dynamic_vino_lib/src/inputs/image_topic.cpp b/dynamic_vino_lib/src/inputs/image_topic.cpp new file mode 100644 index 00000000..1295dbbe --- /dev/null +++ b/dynamic_vino_lib/src/inputs/image_topic.cpp @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2018 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @brief a header file with declaration of ImageTopic class + * @file image_topic.cpp + */ + +#include "dynamic_vino_lib/inputs/realsense_camera_topic.h" +#include "dynamic_vino_lib/slog.h" +#include +#include +#include + +#define INPUT_TOPIC "/camera/color/image_raw" + +bool Input::ImageTopic::initialize() +{ + slog::info << "before Image Topic init" << slog::endl; + std::shared_ptr it = std::make_shared(nh_); + // sub_ = it->subscribe((INPUT_TOPIC, 1, &ImageTopic::cb, this)); + + return true; +} + +bool Input::ImageTopic::initialize(size_t width, size_t height) +{ + slog::warn << "BE CAREFUL: nothing for resolution is done when calling " + "initialize(width, height)" + << " for Image Topic" << slog::endl; + return initialize(); +} + +void Input::ImageTopic::cb(const sensor_msgs::Image::Ptr image_msg) +{ + slog::debug << "Receiving a new image from Camera topic." << slog::endl; + // setHeader(image_msg->header); + + image_ = cv_bridge::toCvCopy(image_msg, "bgr8")->image; + // Suppose Image Topic is sent within BGR order, so the below line would work. + // image_ = cv::Mat(image_msg->height, image_msg->width, CV_8UC3, + // const_cast(&image_msg->data[0]), image_msg->step); + + image_count_.increaseCounter(); +} + +bool Input::ImageTopic::read(cv::Mat* frame) +{ + if (image_count_.get() < 0 || image_.empty()) + { + slog::debug << "No data received in CameraTopic instance" << slog::endl; + return false; + } + + *frame = image_; + // lockHeader(); + image_count_.decreaseCounter(); + return true; +} diff --git a/dynamic_vino_lib/src/inputs/realsense_camera.cpp b/dynamic_vino_lib/src/inputs/realsense_camera.cpp index 8ebd273c..22ab9637 100644 --- a/dynamic_vino_lib/src/inputs/realsense_camera.cpp +++ b/dynamic_vino_lib/src/inputs/realsense_camera.cpp @@ -24,69 +24,32 @@ // RealSenseCamera bool Input::RealSenseCamera::initialize() { - cfg_.enable_stream(RS2_STREAM_COLOR, 640, 480, RS2_FORMAT_BGR8, 30); - setInitStatus(pipe_.start(cfg_)); - setWidth(640); - setHeight(480); - if (!isInit()) - { - return false; - } - if (first_read_) - { - rs2::frameset frames; - for (int i = 0; i < 30; i++) - { - // Wait for all configured streams to produce a frame - try - { - frames = pipe_.wait_for_frames(); - } - catch (...) - { - return false; - } - } - first_read_ = false; - } - return true; + return initialize(640, 480); } bool Input::RealSenseCamera::initialize(size_t width, size_t height) { if (3 * width != 4 * height) { - slog::err << "The aspect ratio must be 4:3 when using RealSense camera" - << slog::endl; + slog::err << "The aspect ratio must be 4:3 when using RealSense camera" << slog::endl; + throw std::runtime_error("The aspect ratio must be 4:3 when using RealSense camera!"); return false; } - cfg_.enable_stream(RS2_STREAM_COLOR, static_cast(width), - static_cast(height), RS2_FORMAT_BGR8, 30); + + auto devSerialNumber = getCameraSN(); + slog::info << "RealSense Serial number : " << devSerialNumber << slog::endl; + + cfg_.enable_device(devSerialNumber); + cfg_.enable_stream(RS2_STREAM_COLOR, static_cast(width), static_cast(height), RS2_FORMAT_BGR8, 30); + setInitStatus(pipe_.start(cfg_)); setWidth(width); setHeight(height); - if (!isInit()) - { - return false; - } - if (first_read_) - { - rs2::frameset frames; - for (int i = 0; i < 30; i++) - { - // Wait for all configured streams to produce a frame - try - { - frames = pipe_.wait_for_frames(); - } - catch (...) - { - return false; - } - } - first_read_ = false; - } - return true; + + // bypass RealSense's bug: several captured frames after HW is inited are with wrong data. + bypassFewFramesOnceInited(); + + return isInit(); } bool Input::RealSenseCamera::read(cv::Mat* frame) @@ -95,23 +58,51 @@ bool Input::RealSenseCamera::read(cv::Mat* frame) { return false; } - rs2::frameset data = - pipe_.wait_for_frames(); // Wait for next set of frames from the camera - rs2::frame color_frame; + try { + rs2::frameset data = pipe_.wait_for_frames(); // Wait for next set of frames from the camera + rs2::frame color_frame; color_frame = data.get_color_frame(); + + cv::Mat(cv::Size(static_cast(getWidth()), static_cast(getHeight())), CV_8UC3, + const_cast(color_frame.get_data()), cv::Mat::AUTO_STEP) + .copyTo(*frame); } catch (...) { return false; } - cv::Mat(cv::Size(static_cast(getWidth()), static_cast(getHeight())), - CV_8UC3, (void*)color_frame.get_data(), cv::Mat::AUTO_STEP) - .copyTo(*frame); + + // setHeader("realsense_camera_frame"); return true; } -void Input::RealSenseCamera::config() + +std::string Input::RealSenseCamera::getCameraSN() +{ + static int rscamera_count = 0; + // Get all devices connected + rs2::context cxt; + auto device = cxt.query_devices(); + size_t device_count = device.size(); + slog::info << "Find RealSense num:" << device_count << slog::endl; + auto hardware = device[rscamera_count]; + auto devSerialNumber = hardware.get_info(RS2_CAMERA_INFO_SERIAL_NUMBER); + rscamera_count++; + return devSerialNumber; +} + +void Input::RealSenseCamera::bypassFewFramesOnceInited() { - // TODO(weizhi): config + if (!isInit() || !first_read_) + { + return; + } + + rs2::frameset frames; + for (int i = 0; i < 30; i++) + { + frames = pipe_.wait_for_frames(); + } + first_read_ = false; } diff --git a/dynamic_vino_lib/src/inputs/realsense_camera_topic.cpp b/dynamic_vino_lib/src/inputs/realsense_camera_topic.cpp deleted file mode 100644 index 900ffa07..00000000 --- a/dynamic_vino_lib/src/inputs/realsense_camera_topic.cpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2018 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @brief a header file with declaration of RealSenseCamera class - * @file realsense_camera_topic.cpp - */ - -#include "dynamic_vino_lib/inputs/realsense_camera_topic.h" -#include -#include "dynamic_vino_lib/slog.h" - -#include - -#define INPUT_TOPIC "/camera/color/image_raw" - -Input::RealSenseCameraTopic::RealSenseCameraTopic() -{ -} - -bool Input::RealSenseCameraTopic::initialize() -{ - slog::info << "before cameraTOpic init" << slog::endl; - - std::shared_ptr it = - std::make_shared(nh_); - sub_ = it->subscribe("/camera/color/image_raw", 1, &RealSenseCameraTopic::cb, - this); - return true; -} - -void Input::RealSenseCameraTopic::cb( - const sensor_msgs::ImageConstPtr& image_msg) -{ - image = cv_bridge::toCvCopy(image_msg, "bgr8")->image; -} - -bool Input::RealSenseCameraTopic::read(cv::Mat* frame) -{ - ros::spinOnce(); - //nothing in topics from begining - if (image.empty() && last_image.empty()) - { - slog::warn << "No data received in CameraTopic instance" << slog::endl; - return false; - } - if(image.empty()) { - *frame = last_image; - } - else - { - *frame = image; - } - return true; -} - -void Input::RealSenseCameraTopic::config() -{ - // TODO(weizhi): config -} diff --git a/dynamic_vino_lib/src/inputs/standard_camera.cpp b/dynamic_vino_lib/src/inputs/standard_camera.cpp index d8649144..65ddd2d6 100644 --- a/dynamic_vino_lib/src/inputs/standard_camera.cpp +++ b/dynamic_vino_lib/src/inputs/standard_camera.cpp @@ -23,30 +23,18 @@ // StandardCamera bool Input::StandardCamera::initialize() { - setInitStatus(cap.open(0)); - setWidth((size_t)cap.get(CV_CAP_PROP_FRAME_WIDTH)); - setHeight((size_t)cap.get(CV_CAP_PROP_FRAME_HEIGHT)); - return isInit(); -} - -bool Input::StandardCamera::initialize(int camera_num) -{ - setInitStatus(cap.open(camera_num)); - setWidth((size_t)cap.get(CV_CAP_PROP_FRAME_WIDTH)); - setHeight((size_t)cap.get(CV_CAP_PROP_FRAME_HEIGHT)); - return isInit(); + return initialize(640, 480); } bool Input::StandardCamera::initialize(size_t width, size_t height) { + auto id = getCameraId(); + setInitStatus(cap.open(id)); + cap.set(cv::CAP_PROP_FRAME_WIDTH, width); + cap.set(cv::CAP_PROP_FRAME_HEIGHT, height); setWidth(width); setHeight(height); - setInitStatus(cap.open(0)); - if (isInit()) - { - cap.set(CV_CAP_PROP_FRAME_WIDTH, width); - cap.set(CV_CAP_PROP_FRAME_HEIGHT, height); - } + return isInit(); } @@ -57,11 +45,33 @@ bool Input::StandardCamera::read(cv::Mat* frame) return false; } cap.grab(); + // setHeader("standard_camera_frame"); return cap.retrieve(*frame); } -void Input::StandardCamera::config() +int Input::StandardCamera::getCameraId() { - // TODO(weizhi): config -} + // In case this function is invoked more than once. + if (camera_id_ >= 0) + { + return camera_id_; + } + static int STANDARD_CAMERA_COUNT = -1; + int fd; // A file descriptor to the video device + struct v4l2_capability cap; + char file[20]; + // if it is a realsense camera then skip it until we meet a standard camera + do + { + STANDARD_CAMERA_COUNT++; + sprintf(file, "/dev/video%d", STANDARD_CAMERA_COUNT); // format filename + fd = open(file, O_RDWR); + ioctl(fd, VIDIOC_QUERYCAP, &cap); + close(fd); + std::cout << "!!camera: " << cap.card << std::endl; + } while (!strcmp((char*)cap.card, "Intel(R) RealSense(TM) Depth Ca")); + + camera_id_ = STANDARD_CAMERA_COUNT; + return STANDARD_CAMERA_COUNT; +} diff --git a/dynamic_vino_lib/src/inputs/video_input.cpp b/dynamic_vino_lib/src/inputs/video_input.cpp index d493731b..68d1530a 100644 --- a/dynamic_vino_lib/src/inputs/video_input.cpp +++ b/dynamic_vino_lib/src/inputs/video_input.cpp @@ -22,6 +22,7 @@ #include #include "dynamic_vino_lib/inputs/video_input.h" +#include "dynamic_vino_lib/slog.h" // Video Input::Video::Video(const std::string& video) @@ -32,8 +33,8 @@ Input::Video::Video(const std::string& video) bool Input::Video::initialize() { setInitStatus(cap.open(video_)); - setWidth((size_t)cap.get(CV_CAP_PROP_FRAME_WIDTH)); - setHeight((size_t)cap.get(CV_CAP_PROP_FRAME_HEIGHT)); + setWidth((size_t)cap.get(cv::CAP_PROP_FRAME_WIDTH)); + setHeight((size_t)cap.get(cv::CAP_PROP_FRAME_HEIGHT)); return isInit(); } @@ -44,8 +45,8 @@ bool Input::Video::initialize(size_t width, size_t height) setInitStatus(cap.open(video_)); if (isInit()) { - cap.set(CV_CAP_PROP_FRAME_WIDTH, width); - cap.set(CV_CAP_PROP_FRAME_HEIGHT, height); + cap.set(cv::CAP_PROP_FRAME_WIDTH, width); + cap.set(cv::CAP_PROP_FRAME_HEIGHT, height); } return isInit(); } @@ -57,10 +58,6 @@ bool Input::Video::read(cv::Mat* frame) return false; } cap.grab(); + // setHeader("video_frame"); return cap.retrieve(*frame); } - -void Input::Video::config() -{ - // TODO(weizhi): config -} diff --git a/dynamic_vino_lib/src/models/age_gender_detection_model.cpp b/dynamic_vino_lib/src/models/age_gender_detection_model.cpp index 2a1122ad..1520aea8 100644 --- a/dynamic_vino_lib/src/models/age_gender_detection_model.cpp +++ b/dynamic_vino_lib/src/models/age_gender_detection_model.cpp @@ -26,73 +26,75 @@ #include "dynamic_vino_lib/slog.h" // Validated Age Gender Classification Network -Models::AgeGenderDetectionModel::AgeGenderDetectionModel( - const std::string& model_loc, int input_num, int output_num, - int max_batch_size) - : BaseModel(model_loc, input_num, output_num, max_batch_size) -{ -} - -void Models::AgeGenderDetectionModel::setLayerProperty( - InferenceEngine::CNNNetReader::Ptr net_reader) +// Models::AgeGenderDetectionModel::AgeGenderDetectionModel(const std::string& model_loc, int input_num, int output_num, +// int max_batch_size) +// : BaseModel(model_loc, input_num, output_num, max_batch_size) +// { +// } +bool Models::AgeGenderDetectionModel::updateLayerProperty(InferenceEngine::CNNNetReader::Ptr net_reader) { + slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; // set input property - InferenceEngine::InputsDataMap input_info_map( - net_reader->getNetwork().getInputsInfo()); + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + if (input_info_map.size() != 1) + { + slog::warn << "This model seems not Age-Gender-like, which should have only one input," + << " but we got " << std::to_string(input_info_map.size()) << "inputs" << slog::endl; + return false; + } InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; input_info->setPrecision(InferenceEngine::Precision::FP32); input_info->setLayout(InferenceEngine::Layout::NCHW); + addInputInfo("input", input_info_map.begin()->first); // set output property - InferenceEngine::OutputsDataMap output_info_map( - net_reader->getNetwork().getOutputsInfo()); + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + if (output_info_map.size() != 2) + { + // throw std::logic_error("Age/Gender Recognition network should have two output layers"); + slog::warn << "This model seems not Age-gender like, which should have and only have 2" + " outputs, but we got " + << std::to_string(output_info_map.size()) << "outputs" << slog::endl; + return false; + } auto it = output_info_map.begin(); InferenceEngine::DataPtr age_output_ptr = (it++)->second; InferenceEngine::DataPtr gender_output_ptr = (it++)->second; - age_output_ptr->setPrecision(InferenceEngine::Precision::FP32); - age_output_ptr->setLayout(InferenceEngine::Layout::NCHW); - gender_output_ptr->setPrecision(InferenceEngine::Precision::FP32); - gender_output_ptr->setLayout(InferenceEngine::Layout::NCHW); - // set input and output layer name - input_ = input_info_map.begin()->first; - output_age_ = age_output_ptr->name; - output_gender_ = gender_output_ptr->name; -} -void Models::AgeGenderDetectionModel::checkLayerProperty( - const InferenceEngine::CNNNetReader::Ptr& net_reader) -{ - slog::info << "Checking Age Gender Detection outputs" << slog::endl; - InferenceEngine::OutputsDataMap output_info( - net_reader->getNetwork().getOutputsInfo()); - auto it = output_info.begin(); - InferenceEngine::DataPtr age_output_ptr = (it++)->second; - InferenceEngine::DataPtr gender_output_ptr = (it++)->second; - // output layer of age should be Convolution type + // Check More Configuration: if (gender_output_ptr->getCreatorLayer().lock()->type == "Convolution") { std::swap(age_output_ptr, gender_output_ptr); } if (age_output_ptr->getCreatorLayer().lock()->type != "Convolution") { - throw std::logic_error("In Age Gender network, age layer (" + - age_output_ptr->getCreatorLayer().lock()->name + - ") should be a Convolution, but was: " + - age_output_ptr->getCreatorLayer().lock()->type); + slog::err << "In Age Gender network, age layer (" << age_output_ptr->getCreatorLayer().lock()->name + << ") should be a Convolution, but was: " << age_output_ptr->getCreatorLayer().lock()->type << slog::endl; + return false; } if (gender_output_ptr->getCreatorLayer().lock()->type != "SoftMax") { - throw std::logic_error("In Age Gender network, gender layer (" + - gender_output_ptr->getCreatorLayer().lock()->name + - ") should be a SoftMax, but was: " + - gender_output_ptr->getCreatorLayer().lock()->type); + slog::err << "In Age Gender network, gender layer (" << gender_output_ptr->getCreatorLayer().lock()->name + << ") should be a SoftMax, but was: " << gender_output_ptr->getCreatorLayer().lock()->type << slog::endl; + return false; } - slog::info << "Age layer: " << age_output_ptr->getCreatorLayer().lock()->name - << slog::endl; - slog::info << "Gender layer: " - << gender_output_ptr->getCreatorLayer().lock()->name << slog::endl; + slog::info << "Age layer: " << age_output_ptr->getCreatorLayer().lock()->name << slog::endl; + slog::info << "Gender layer: " << gender_output_ptr->getCreatorLayer().lock()->name << slog::endl; + + age_output_ptr->setPrecision(InferenceEngine::Precision::FP32); + age_output_ptr->setLayout(InferenceEngine::Layout::NCHW); + gender_output_ptr->setPrecision(InferenceEngine::Precision::FP32); + gender_output_ptr->setLayout(InferenceEngine::Layout::NCHW); + + // output_age_ = age_output_ptr->name; + addOutputInfo("age", age_output_ptr->getName()); + // output_gender_ = gender_output_ptr->name; + addOutputInfo("gender", gender_output_ptr->getName()); + + printAttribute(); + return true; } -const std::string Models::AgeGenderDetectionModel::getModelName() const +const std::string Models::AgeGenderDetectionModel::getModelCategory() const { return "Age Gender Detection"; } diff --git a/dynamic_vino_lib/src/models/base_model.cpp b/dynamic_vino_lib/src/models/base_model.cpp index 08153777..4b3c64bf 100644 --- a/dynamic_vino_lib/src/models/base_model.cpp +++ b/dynamic_vino_lib/src/models/base_model.cpp @@ -28,17 +28,14 @@ #include "dynamic_vino_lib/slog.h" // Validated Base Network -Models::BaseModel::BaseModel(const std::string& model_loc, int input_num, - int output_num, int max_batch_size) - : input_num_(input_num), - output_num_(output_num), - model_loc_(model_loc), - max_batch_size_(max_batch_size) +Models::BaseModel::BaseModel(const std::string& model_loc, int max_batch_size) + : model_loc_(model_loc), max_batch_size_(max_batch_size), ModelAttribute(model_loc) { if (model_loc.empty()) { throw std::logic_error("model file name is empty!"); } + net_reader_ = std::make_shared(); } @@ -47,9 +44,6 @@ void Models::BaseModel::modelInit() slog::info << "Loading network files" << slog::endl; // Read network model net_reader_->ReadNetwork(model_loc_); - // Set batch size to given max_batch_size_ - slog::info << "Batch size is set to " << max_batch_size_ << slog::endl; - net_reader_->getNetwork().setBatchSize(max_batch_size_); // Extract model name and load it's weights // remove extension size_t last_index = model_loc_.find_last_of("."); @@ -58,34 +52,39 @@ void Models::BaseModel::modelInit() net_reader_->ReadWeights(bin_file_name); // Read labels (if any) std::string label_file_name = raw_name + ".labels"; - std::ifstream input_file(label_file_name); - std::copy(std::istream_iterator(input_file), - std::istream_iterator(), std::back_inserter(labels_)); - checkNetworkSize(input_num_, output_num_, net_reader_); + loadLabelsFromFile(label_file_name); + + // Set batch size to given max_batch_size_ + slog::info << "Batch size is set to " << max_batch_size_ << slog::endl; + net_reader_->getNetwork().setBatchSize(max_batch_size_); + /** DEPRECATED! checkLayerProperty(net_reader_); - setLayerProperty(net_reader_); + setLayerProperty(net_reader_); */ + updateLayerProperty(net_reader_); } -void Models::BaseModel::checkNetworkSize( - unsigned int input_size, unsigned int output_size, - InferenceEngine::CNNNetReader::Ptr net_reader) +#if 0 +bool Models::BaseModel::updateLayerProperty( + InferenceEngine::CNNNetReader::Ptr net_reader) { - // check input size - slog::info << "Checking input size" << slog::endl; - InferenceEngine::InputsDataMap input_info( - net_reader->getNetwork().getInputsInfo()); - if (input_info.size() != input_size) - { - throw std::logic_error(getModelName() + " should have only one input"); +#if 0 + if (!updateLayerProperty(net_reader)){ + slog::warn << "The model(name: " << getModelName() << ") failed to update Layer Property!" + << slog::endl; + return false; } - // check output size - slog::info << "Checking output size" << slog::endl; - InferenceEngine::OutputsDataMap output_info( - net_reader->getNetwork().getOutputsInfo()); - if (output_info.size() != output_size) - { - throw std::logic_error(getModelName() + - "network should have only one output"); +#endif + if(!isVerified()){ + slog::warn << "The model(name: " << getModelName() << ") does NOT pass Attribute Check!" + << slog::endl; + return false; } - // InferenceEngine::DataPtr& output_data_ptr = output_info.begin()->second; + + return true; +} +#endif + +Models::ObjectDetectionModel::ObjectDetectionModel(const std::string& model_loc, int max_batch_size) + : BaseModel(model_loc, max_batch_size) +{ } diff --git a/dynamic_vino_lib/src/models/emotion_detection_model.cpp b/dynamic_vino_lib/src/models/emotion_detection_model.cpp index d1e6150d..bfb07481 100644 --- a/dynamic_vino_lib/src/models/emotion_detection_model.cpp +++ b/dynamic_vino_lib/src/models/emotion_detection_model.cpp @@ -24,54 +24,60 @@ #include "dynamic_vino_lib/slog.h" // Validated Emotions Detection Network -Models::EmotionDetectionModel::EmotionDetectionModel( - const std::string& model_loc, int input_num, int output_num, - int max_batch_size) - : BaseModel(model_loc, input_num, output_num, max_batch_size) -{ -} +// Models::EmotionDetectionModel::EmotionDetectionModel(const std::string& model_loc, int input_num, int output_num, +// int max_batch_size) +// : BaseModel(model_loc, input_num, output_num, max_batch_size) +// { +// } -void Models::EmotionDetectionModel::setLayerProperty( - InferenceEngine::CNNNetReader::Ptr net_reader) +bool Models::EmotionDetectionModel::updateLayerProperty(InferenceEngine::CNNNetReader::Ptr net_reader) { + slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; // set input property - InferenceEngine::InputsDataMap input_info_map( - net_reader->getNetwork().getInputsInfo()); + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + if (input_info_map.size() != 1) + { + slog::warn << "This model seems not Age-Gender-like, which should have only one input," + << " but we got " << std::to_string(input_info_map.size()) << "inputs" << slog::endl; + return false; + } InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; input_info->setPrecision(InferenceEngine::Precision::FP32); input_info->setLayout(InferenceEngine::Layout::NCHW); + addInputInfo("input", input_info_map.begin()->first); + // set output property - InferenceEngine::OutputsDataMap output_info_map( - net_reader->getNetwork().getOutputsInfo()); + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + if (output_info_map.size() != 1) + { + // throw std::logic_error("Age/Gender Recognition network should have two output layers"); + slog::warn << "This model should have and only have 1 output, but we got " << std::to_string(output_info_map.size()) + << "outputs" << slog::endl; + return false; + } InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; + slog::info << "Emotions layer: " << output_data_ptr->getCreatorLayer().lock()->name << slog::endl; output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); output_data_ptr->setLayout(InferenceEngine::Layout::NCHW); - // set input and output layer name - input_ = input_info_map.begin()->first; - output_ = output_info_map.begin()->first; + addOutputInfo("output", output_info_map.begin()->first); + + printAttribute(); + return verifyOutputLayer(output_data_ptr); } -void Models::EmotionDetectionModel::checkLayerProperty( - const InferenceEngine::CNNNetReader::Ptr& net_reader) +bool Models::EmotionDetectionModel::verifyOutputLayer(const InferenceEngine::DataPtr& ptr) { - slog::info << "Checking Emotions Detection outputs" << slog::endl; - InferenceEngine::OutputsDataMap output_info( - net_reader->getNetwork().getOutputsInfo()); - InferenceEngine::DataPtr emotions_output_ptr = output_info.begin()->second; - // output layer should be SoftMax type - if (emotions_output_ptr->getCreatorLayer().lock()->type != "SoftMax") + if (ptr->getCreatorLayer().lock()->type != "SoftMax") { - throw std::logic_error("In Emotions Recognition network, Emotion layer (" + - emotions_output_ptr->getCreatorLayer().lock()->name + - ") should be a SoftMax, but was: " + - emotions_output_ptr->getCreatorLayer().lock()->type); + slog::err << "In Emotion network, gender layer (" << ptr->getCreatorLayer().lock()->name + << ") should be a SoftMax, but was: " << ptr->getCreatorLayer().lock()->type << slog::endl; + return false; } - slog::info << "Emotions layer: " - << emotions_output_ptr->getCreatorLayer().lock()->name - << slog::endl; + + return true; } -const std::string Models::EmotionDetectionModel::getModelName() const +const std::string Models::EmotionDetectionModel::getModelCategory() const { return "Emotions Detection"; } diff --git a/dynamic_vino_lib/src/models/face_detection_model.cpp b/dynamic_vino_lib/src/models/face_detection_model.cpp index caadb0e2..35b613b7 100644 --- a/dynamic_vino_lib/src/models/face_detection_model.cpp +++ b/dynamic_vino_lib/src/models/face_detection_model.cpp @@ -26,92 +26,75 @@ #include "dynamic_vino_lib/slog.h" // Validated Face Detection Network -Models::FaceDetectionModel::FaceDetectionModel(const std::string& model_loc, - int input_num, int output_num, - int max_batch_size) - : BaseModel(model_loc, input_num, output_num, max_batch_size) +Models::FaceDetectionModel::FaceDetectionModel(const std::string& model_loc, int max_batch_size) + : ObjectDetectionModel(model_loc, max_batch_size) { } -void Models::FaceDetectionModel::setLayerProperty( - InferenceEngine::CNNNetReader::Ptr net_reader) -{ - // set input property - InferenceEngine::InputsDataMap input_info_map( - net_reader->getNetwork().getInputsInfo()); - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::U8); - input_info->setLayout(InferenceEngine::Layout::NCHW); - // set output property - InferenceEngine::OutputsDataMap output_info_map( - net_reader->getNetwork().getOutputsInfo()); - InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; - output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); - output_data_ptr->setLayout(InferenceEngine::Layout::NCHW); - // set input and output layer name - input_ = input_info_map.begin()->first; - output_ = output_info_map.begin()->first; -} - +#if 0 void Models::FaceDetectionModel::checkLayerProperty( const InferenceEngine::CNNNetReader::Ptr& net_reader) { + slog::info << "Checking Face Detection inputs" << slog::endl; + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + if (input_info_map.size() != 1) { + slog::err << "Face Detection network should have only one input, but we got " + << std::to_string(input_info_map.size()) << "inputs" << slog::endl; + throw std::logic_error("Face Detection network should have only one input"); + } + slog::info << "Checking Face Detection outputs" << slog::endl; - InferenceEngine::OutputsDataMap output_info_map( - net_reader->getNetwork().getOutputsInfo()); - InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + slog::info << "Checking Face Detection outputs ..." << slog::endl; + if (output_info_map.size() != 1) { + throw std::logic_error("This sample accepts networks having only one output"); + } + InferenceEngine::DataPtr & output_data_ptr = output_info_map.begin()->second; output_ = output_info_map.begin()->first; + slog::info << "Checking Object Detection output ... Name=" << output_ << slog::endl; + const InferenceEngine::CNNLayerPtr output_layer = - net_reader->getNetwork().getLayerByName(output_.c_str()); - // output layer should be DetectionOutput type - if (output_layer->type != "DetectionOutput") - { - throw std::logic_error( - "Face Detection network output layer(" + output_layer->name + - ") should be DetectionOutput, but was " + output_layer->type); - } + net_reader->getNetwork().getLayerByName(output_.c_str()); // output layer should have attribute called num_classes - if (output_layer->params.find("num_classes") == output_layer->params.end()) - { - throw std::logic_error("Face Detection network output layer (" + output_ + - ") should have num_classes integer attribute"); + slog::info << "Checking Object Detection num_classes" << slog::endl; + if (output_layer->params.find("num_classes") == output_layer->params.end()) { + throw std::logic_error("Object Detection network output layer (" + output_ + + ") should have num_classes integer attribute"); } // class number should be equal to size of label vector // if network has default "background" class, fake is used - const size_t num_classes = output_layer->GetParamAsInt("num_classes"); - if (getLabels().size() != num_classes) - { - if (getLabels().size() == (num_classes - 1)) - { + const int num_classes = output_layer->GetParamAsInt("num_classes"); + + slog::info << "Checking Object Detection output ... num_classes=" << num_classes << slog::endl; + if (getLabels().size() != num_classes) { + if (getLabels().size() == (num_classes - 1)) { getLabels().insert(getLabels().begin(), "fake"); - } - else - { + } else { getLabels().clear(); } } // last dimension of output layer should be 7 - const InferenceEngine::SizeVector output_dims = - output_data_ptr->getTensorDesc().getDims(); + const InferenceEngine::SizeVector output_dims = output_data_ptr->getTensorDesc().getDims(); max_proposal_count_ = static_cast(output_dims[2]); slog::info << "max proposal count is: " << max_proposal_count_ << slog::endl; - object_size_ = static_cast(output_dims[3]); - if (object_size_ != 7) - { - throw std::logic_error( - "Face Detection network output layer should have 7 as a last " - "dimension"); + + auto object_size = static_cast(output_dims[3]); + if (object_size != 7) { + throw std::logic_error("Object Detection network output layer should have 7 as a last " + "dimension"); } - if (output_dims.size() != 4) - { - throw std::logic_error( - "Face Detection network output dimensions not compatible shoulld be 4, " - "but was " + - std::to_string(output_dims.size())); + setObjectSize(object_size); + + if (output_dims.size() != 4) { + throw std::logic_error("Object Detection network output dimensions not compatible shoulld be " + "4, " + "but was " + + std::to_string(output_dims.size())); } } +#endif -const std::string Models::FaceDetectionModel::getModelName() const +const std::string Models::FaceDetectionModel::getModelCategory() const { return "Face Detection"; } diff --git a/dynamic_vino_lib/src/models/face_reidentification_model.cpp b/dynamic_vino_lib/src/models/face_reidentification_model.cpp new file mode 100644 index 00000000..add2b5fb --- /dev/null +++ b/dynamic_vino_lib/src/models/face_reidentification_model.cpp @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with declaration of FaceReidentificationModel class + * @file face_reidentification_model.cpp + */ +#include +#include "dynamic_vino_lib/models/face_reidentification_model.h" +#include "dynamic_vino_lib/slog.h" +// Validated Face Reidentification Network +Models::FaceReidentificationModel::FaceReidentificationModel(const std::string& model_loc, int max_batch_size) + : BaseModel(model_loc, max_batch_size) +{ +} + +void Models::FaceReidentificationModel::setLayerProperty(InferenceEngine::CNNNetReader::Ptr net_reader) +{ + // set input property + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; + input_info->setPrecision(InferenceEngine::Precision::U8); + input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); + // set output property + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; + output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); + output_data_ptr->setLayout(InferenceEngine::Layout::NCHW); + // set input and output layer name + input_ = input_info_map.begin()->first; + output_ = output_info_map.begin()->first; +} + +void Models::FaceReidentificationModel::checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr& net_reader) +{ +} + +const std::string Models::FaceReidentificationModel::getModelCategory() const +{ + return "Face Reidentification"; +} diff --git a/dynamic_vino_lib/src/models/head_pose_detection_model.cpp b/dynamic_vino_lib/src/models/head_pose_detection_model.cpp index 2012deb2..2761174e 100644 --- a/dynamic_vino_lib/src/models/head_pose_detection_model.cpp +++ b/dynamic_vino_lib/src/models/head_pose_detection_model.cpp @@ -27,77 +27,52 @@ #include "dynamic_vino_lib/slog.h" // Validated Head Pose Network -Models::HeadPoseDetectionModel::HeadPoseDetectionModel( - const std::string& model_loc, int input_num, int output_num, - int max_batch_size) - : BaseModel(model_loc, input_num, output_num, max_batch_size) +Models::HeadPoseDetectionModel::HeadPoseDetectionModel(const std::string& model_loc, int max_batch_size) + : BaseModel(model_loc, max_batch_size) { } -void Models::HeadPoseDetectionModel::checkLayerProperty( - const InferenceEngine::CNNNetReader::Ptr& net_reader) +bool Models::HeadPoseDetectionModel::updateLayerProperty(InferenceEngine::CNNNetReader::Ptr net_reader) { - slog::info << "Checking Head Pose network outputs 1" << slog::endl; - slog::info << "Checked Head Pose network outputs" << slog::endl; - - InferenceEngine::OutputsDataMap outputInfo( - net_reader->getNetwork().getOutputsInfo()); - std::map layerNames = {{output_angle_r_, false}, - {output_angle_p_, false}, - {output_angle_y_, false}}; - - for (auto&& output : outputInfo) + slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; + // set input property + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + if (input_info_map.size() != 1) { - InferenceEngine::CNNLayerPtr layer = - output.second->getCreatorLayer().lock(); - if (layerNames.find(layer->name) == layerNames.end()) - { - throw std::logic_error("Head Pose network output layer unknown: " + - layer->name + ", should be " + output_angle_r_ + - " or " + output_angle_p_ + " or " + - output_angle_y_); - } - if (layer->type != "FullyConnected") - { - throw std::logic_error("Head Pose network output layer (" + layer->name + - ") has invalid type: " + layer->type + - ", should be FullyConnected"); - } - auto fc = dynamic_cast(layer.get()); - if (fc->_out_num != 1) - { - throw std::logic_error("Head Pose network output layer (" + layer->name + - ") has invalid out-size=" + - std::to_string(fc->_out_num) + ", should be 1"); - } - layerNames[layer->name] = true; + slog::warn << "This model should have only one input, but we got" << std::to_string(input_info_map.size()) + << "inputs" << slog::endl; + return false; } -} - -void Models::HeadPoseDetectionModel::setLayerProperty( - InferenceEngine::CNNNetReader::Ptr net_reader) -{ - // set input property - InferenceEngine::InputsDataMap input_info_map( - net_reader->getNetwork().getInputsInfo()); - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; input_info->setPrecision(InferenceEngine::Precision::U8); input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); - input_ = input_info_map.begin()->first; + addInputInfo("input", input_info_map.begin()->first); // set output property - InferenceEngine::OutputsDataMap output_info_map( - net_reader->getNetwork().getOutputsInfo()); - + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); for (auto& output : output_info_map) { output.second->setPrecision(InferenceEngine::Precision::FP32); output.second->setLayout(InferenceEngine::Layout::NC); } + + for (const std::string& outName : { output_angle_r_, output_angle_p_, output_angle_y_ }) + { + if (output_info_map.find(outName) == output_info_map.end()) + { + throw std::logic_error("There is no " + outName + " output in Head Pose Estimation network"); + } + else + { + addOutputInfo(outName, outName); + } + } + + printAttribute(); + return true; } -const std::string Models::HeadPoseDetectionModel::getModelName() const +const std::string Models::HeadPoseDetectionModel::getModelCategory() const { return "Head Pose Network"; } diff --git a/dynamic_vino_lib/src/models/landmarks_detection_model.cpp b/dynamic_vino_lib/src/models/landmarks_detection_model.cpp new file mode 100644 index 00000000..1d6c0781 --- /dev/null +++ b/dynamic_vino_lib/src/models/landmarks_detection_model.cpp @@ -0,0 +1,61 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with declaration of LandmarksDetectionModel class + * @file landmarks_detection_model.cpp + */ +#include +#include "dynamic_vino_lib/models/landmarks_detection_model.h" +// Validated Landmarks Detection Network +Models::LandmarksDetectionModel::LandmarksDetectionModel(const std::string& model_loc, int max_batch_size) + : BaseModel(model_loc, max_batch_size) +{ +} + +void Models::LandmarksDetectionModel::setLayerProperty(InferenceEngine::CNNNetReader::Ptr net_reader) +{ + // set input property + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; + input_info->setPrecision(InferenceEngine::Precision::U8); + input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); + // set output property + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; + output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); + output_data_ptr->setLayout(InferenceEngine::Layout::NCHW); + // set input and output layer name + input_ = input_info_map.begin()->first; + output_ = output_info_map.begin()->first; +} + +void Models::LandmarksDetectionModel::checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr& net_reader) +{ + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + if (input_info_map.size() != 1) + { + throw std::logic_error("Landmarks Detection topology should have only one input"); + } + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + if (output_info_map.size() != 1) + { + throw std::logic_error("Landmarks Detection Network expects networks having one output"); + } +} + +const std::string Models::LandmarksDetectionModel::getModelCategory() const +{ + return "Landmarks Detection"; +} diff --git a/dynamic_vino_lib/src/models/license_plate_detection_model.cpp b/dynamic_vino_lib/src/models/license_plate_detection_model.cpp new file mode 100644 index 00000000..5c1e1a2f --- /dev/null +++ b/dynamic_vino_lib/src/models/license_plate_detection_model.cpp @@ -0,0 +1,62 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a realization file with declaration of LicensePlateDetectionModel class + * @file license_plate_detection_model.cpp + */ +#include +#include "dynamic_vino_lib/models/license_plate_detection_model.h" +#include "dynamic_vino_lib/slog.h" +// Validated Vehicle Attributes Detection Network +Models::LicensePlateDetectionModel::LicensePlateDetectionModel(const std::string& model_loc, int max_batch_size) + : BaseModel(model_loc, max_batch_size) +{ +} + +bool Models::LicensePlateDetectionModel::updateLayerProperty(const InferenceEngine::CNNNetReader::Ptr net_reader) +{ + slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + if (input_info_map.size() != 2) + { + throw std::logic_error("Vehicle Attribs topology should have only two inputs"); + } + auto sequence_input = (++input_info_map.begin()); + if (sequence_input->second->getTensorDesc().getDims()[0] != getMaxSequenceSize()) + { + throw std::logic_error("License plate detection max sequence size dismatch"); + } + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + if (output_info_map.size() != 1) + { + throw std::logic_error("Vehicle Attribs Network expects networks having one output"); + } + + InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; + input_info->setPrecision(InferenceEngine::Precision::U8); + input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); + + // set input and output layer name + input_ = input_info_map.begin()->first; + seq_input_ = (++input_info_map.begin())->first; + output_ = output_info_map.begin()->first; + + return true; +} + +const std::string Models::LicensePlateDetectionModel::getModelCategory() const +{ + return "Vehicle Attributes Detection"; +} diff --git a/dynamic_vino_lib/src/models/object_detection_model.cpp b/dynamic_vino_lib/src/models/object_detection_model.cpp deleted file mode 100644 index e16ddc20..00000000 --- a/dynamic_vino_lib/src/models/object_detection_model.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2018 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * @brief a header file with declaration of ObjectDetectionModel class - * @file object_detection_model.cpp - */ -#include -#include "dynamic_vino_lib/models/object_detection_model.h" -#include "dynamic_vino_lib/slog.h" -// Validated Object Detection Network -Models::ObjectDetectionModel::ObjectDetectionModel(const std::string& model_loc, - int input_num, int output_num, - int max_batch_size) - : BaseModel(model_loc, input_num, output_num, max_batch_size){} -void Models::ObjectDetectionModel::setLayerProperty( - InferenceEngine::CNNNetReader::Ptr net_reader) { - // set input property - InferenceEngine::InputsDataMap input_info_map( - net_reader->getNetwork().getInputsInfo()); - if (input_info_map.size() != 1) { - throw std::logic_error("This sample accepts networks having only one input"); - } - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::U8); - input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); - // set output property - InferenceEngine::OutputsDataMap output_info_map( - net_reader->getNetwork().getOutputsInfo()); - if (output_info_map.size() != 1) { - throw std::logic_error("This sample accepts networks having only one output"); - } - InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; - output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); - output_data_ptr->setLayout(InferenceEngine::Layout::NCHW); - // set input and output layer name - input_ = input_info_map.begin()->first; - output_ = output_info_map.begin()->first; -} -void Models::ObjectDetectionModel::checkLayerProperty( - const InferenceEngine::CNNNetReader::Ptr& net_reader) { - slog::info << "Checking Object Detection outputs" << slog::endl; - InferenceEngine::OutputsDataMap output_info_map( - net_reader->getNetwork().getOutputsInfo()); - slog::info << "Checking Object Detection outputs ..." << slog::endl; - if (output_info_map.size() != 1) { - throw std::logic_error("This sample accepts networks having only one output"); - } - InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; - output_ = output_info_map.begin()->first; - slog::info << "Checking Object Detection output ... Name=" << output_ << slog::endl; - - const InferenceEngine::CNNLayerPtr output_layer = - net_reader->getNetwork().getLayerByName(output_.c_str()); - // output layer should have attribute called num_classes - slog::info << "Checking Object Detection num_classes" << slog::endl; - if (output_layer->params.find("num_classes") == output_layer->params.end()) { - throw std::logic_error("Object Detection network output layer (" + output_ + - ") should have num_classes integer attribute"); - } - // class number should be equal to size of label vector - // if network has default "background" class, fake is used - const unsigned int num_classes = output_layer->GetParamAsInt("num_classes"); - - slog::info << "Checking Object Detection output ... num_classes=" << num_classes << slog::endl; - if (getLabels().size() != num_classes) { - if (getLabels().size() == (num_classes - 1)) { - getLabels().insert(getLabels().begin(), "fake"); - } else { - getLabels().clear(); - } - } - // last dimension of output layer should be 7 - const InferenceEngine::SizeVector output_dims = - output_data_ptr->getTensorDesc().getDims(); - max_proposal_count_ = static_cast(output_dims[2]); - slog::info << "max proposal count is: " << max_proposal_count_ << slog::endl; - object_size_ = static_cast(output_dims[3]); - if (object_size_ != 7) { - throw std::logic_error( - "Object Detection network output layer should have 7 as a last " - "dimension"); - } - if (output_dims.size() != 4) { - throw std::logic_error( - "Object Detection network output dimensions not compatible shoulld be 4, " - "but was " + - std::to_string(output_dims.size())); - } -} -const std::string Models::ObjectDetectionModel::getModelName() const { - return "Object Detection"; -} diff --git a/dynamic_vino_lib/src/models/object_detection_ssd_model.cpp b/dynamic_vino_lib/src/models/object_detection_ssd_model.cpp new file mode 100644 index 00000000..96c998d6 --- /dev/null +++ b/dynamic_vino_lib/src/models/object_detection_ssd_model.cpp @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2018 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @brief a header file with declaration of ObjectDetectionSSDModel class + * @file object_detection_model.cpp + */ +#include +#include "dynamic_vino_lib/models/object_detection_ssd_model.h" +#include "dynamic_vino_lib/slog.h" +// Validated Object Detection Network +Models::ObjectDetectionSSDModel::ObjectDetectionSSDModel(const std::string& model_loc, int max_batch_size) + : ObjectDetectionModel(model_loc, max_batch_size) +{ + slog::debug << "TESTING: in ObjectDetectionSSDModel" << slog::endl; + // addCandidatedAttr(std::make_shared()); +} + +const std::string Models::ObjectDetectionSSDModel::getModelCategory() const +{ + return "Object Detection SSD"; +} + +bool Models::ObjectDetectionSSDModel::enqueue(const std::shared_ptr& engine, const cv::Mat& frame, + const cv::Rect& input_frame_loc) +{ + if (!this->matToBlob(frame, input_frame_loc, 1, 0, engine)) + { + return false; + } + + setFrameSize(frame.cols, frame.rows); + return true; +} + +bool Models::ObjectDetectionSSDModel::matToBlob(const cv::Mat& orig_image, const cv::Rect&, float scale_factor, + int batch_index, const std::shared_ptr& engine) +{ + if (engine == nullptr) + { + slog::err << "A frame is trying to be enqueued in a NULL Engine." << slog::endl; + return false; + } + + std::string input_name = getInputName(); + slog::debug << "add input image to blob: " << input_name << slog::endl; + InferenceEngine::Blob::Ptr input_blob = engine->getRequest()->GetBlob(input_name); + + InferenceEngine::SizeVector blob_size = input_blob->getTensorDesc().getDims(); + const int width = blob_size[3]; + const int height = blob_size[2]; + const int channels = blob_size[1]; + u_int8_t* blob_data = input_blob->buffer().as(); + + cv::Mat resized_image(orig_image); + if (width != orig_image.size().width || height != orig_image.size().height) + { + cv::resize(orig_image, resized_image, cv::Size(width, height)); + } + int batchOffset = batch_index * width * height * channels; + + for (int c = 0; c < channels; c++) + { + for (int h = 0; h < height; h++) + { + for (int w = 0; w < width; w++) + { + blob_data[batchOffset + c * width * height + h * width + w] = + resized_image.at(h, w)[c] * scale_factor; + } + } + } + + slog::debug << "Convert input image to blob: DONE!" << slog::endl; + return true; +} + +bool Models::ObjectDetectionSSDModel::fetchResults(const std::shared_ptr& engine, + std::vector& results, + const float& confidence_thresh, const bool& enable_roi_constraint) +{ + slog::debug << "fetching Infer Resulsts from the given SSD model" << slog::endl; + if (engine == nullptr) + { + slog::err << "Trying to fetch results from Engines." << slog::endl; + return false; + } + + slog::debug << "Fetching Detection Results ..." << slog::endl; + InferenceEngine::InferRequest::Ptr request = engine->getRequest(); + std::string output = getOutputName(); + const float* detections = request->GetBlob(output)->buffer().as(); + + slog::debug << "Analyzing Detection results..." << slog::endl; + auto max_proposal_count = getMaxProposalCount(); + auto object_size = getObjectSize(); + slog::debug << "MaxProprosalCount=" << max_proposal_count << ", ObjectSize=" << object_size << slog::endl; + for (int i = 0; i < max_proposal_count; i++) + { + float image_id = detections[i * object_size + 0]; + if (image_id < 0) + { + // slog::info << "Found objects: " << i << "|" << results.size() << slog::endl; + break; + } + + cv::Rect r; + auto label_num = static_cast(detections[i * object_size + 1]); + std::vector& labels = getLabels(); + auto frame_size = getFrameSize(); + r.x = static_cast(detections[i * object_size + 3] * frame_size.width); + r.y = static_cast(detections[i * object_size + 4] * frame_size.height); + r.width = static_cast(detections[i * object_size + 5] * frame_size.width - r.x); + r.height = static_cast(detections[i * object_size + 6] * frame_size.height - r.y); + + if (enable_roi_constraint) + { + r &= cv::Rect(0, 0, frame_size.width, frame_size.height); + } + + dynamic_vino_lib::ObjectDetectionResult result(r); + std::string label = + label_num < labels.size() ? labels[label_num] : std::string("label #") + std::to_string(label_num); + result.setLabel(label); + float confidence = detections[i * object_size + 2]; + if (confidence <= confidence_thresh /* || r.x == 0 */) + { // why r.x needs to be checked? + continue; + } + result.setConfidence(confidence); + + results.emplace_back(result); + } + + return true; +} + +bool Models::ObjectDetectionSSDModel::updateLayerProperty(const InferenceEngine::CNNNetReader::Ptr net_reader) +{ + slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; + + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + if (input_info_map.size() != 1) + { + slog::warn << "This model seems not SSDNet-like, SSDnet has only one input, but we got " + << std::to_string(input_info_map.size()) << "inputs" << slog::endl; + return false; + } + + InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; + input_info->setPrecision(InferenceEngine::Precision::U8); + addInputInfo("input", input_info_map.begin()->first); + + const InferenceEngine::SizeVector input_dims = input_info->getTensorDesc().getDims(); + setInputHeight(input_dims[2]); + setInputWidth(input_dims[3]); + + slog::info << "Checking OUTPUTs for model " << getModelName() << slog::endl; + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + if (output_info_map.size() != 1) + { + slog::warn << "This model seems not SSDNet-like! We got " << std::to_string(output_info_map.size()) + << "outputs, but SSDnet has only one." << slog::endl; + return false; + } + InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; + addOutputInfo("output", output_info_map.begin()->first); + slog::info << "Checking Object Detection output ... Name=" << output_info_map.begin()->first << slog::endl; + output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); + + /// TODO: double check this part: BEGIN + const InferenceEngine::CNNLayerPtr output_layer = + net_reader->getNetwork().getLayerByName(output_info_map.begin()->first.c_str()); + // output layer should have attribute called num_classes + slog::info << "Checking Object Detection num_classes" << slog::endl; + if (output_layer->params.find("num_classes") == output_layer->params.end()) + { + slog::warn << "This model's output layer (" << output_info_map.begin()->first + << ") should have num_classes integer attribute" << slog::endl; + return false; + } + // class number should be equal to size of label vector + // if network has default "background" class, fake is used + const int num_classes = output_layer->GetParamAsInt("num_classes"); +#if 0 + slog::info << "Checking Object Detection output ... num_classes=" << num_classes << slog::endl; + if (getLabels().size() != num_classes) { + if (getLabels().size() == (num_classes - 1)) { + getLabels().insert(getLabels().begin(), "fake"); + } else { + getLabels().clear(); + } + } +#endif + /// TODO: double check this part: END + + // last dimension of output layer should be 7 + const InferenceEngine::SizeVector output_dims = output_data_ptr->getTensorDesc().getDims(); + setMaxProposalCount(static_cast(output_dims[2])); + slog::info << "max proposal count is: " << getMaxProposalCount() << slog::endl; + + auto object_size = static_cast(output_dims[3]); + if (object_size != 7) + { + slog::warn << "This model is NOT SSDNet-like, whose output data for each detected object" + << "should have 7 dimensions, but was " << std::to_string(object_size) << slog::endl; + return false; + } + setObjectSize(object_size); + + if (output_dims.size() != 4) + { + slog::warn << "This model is not SSDNet-like, output dimensions shoulld be 4, but was" + << std::to_string(output_dims.size()) << slog::endl; + return false; + } + + printAttribute(); + slog::info << "This model is SSDNet-like, Layer Property updated!" << slog::endl; + return true; +} diff --git a/dynamic_vino_lib/src/models/object_detection_yolov2voc_model.cpp b/dynamic_vino_lib/src/models/object_detection_yolov2voc_model.cpp new file mode 100644 index 00000000..ba86a868 --- /dev/null +++ b/dynamic_vino_lib/src/models/object_detection_yolov2voc_model.cpp @@ -0,0 +1,358 @@ +/* + * Copyright (c) 2018 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @brief a header file with declaration of ObjectDetectionYolov2vocModel class + * @file object_detection_model.cpp + */ +#include +#include "dynamic_vino_lib/models/object_detection_yolov2voc_model.h" +#include "dynamic_vino_lib/slog.h" +// Validated Object Detection Network +Models::ObjectDetectionYolov2Model::ObjectDetectionYolov2Model(const std::string& model_loc, int max_batch_size) + : ObjectDetectionModel(model_loc, max_batch_size) +{ +} + +bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const InferenceEngine::CNNNetReader::Ptr net_reader) +{ + slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; + + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + if (input_info_map.size() != 1) + { + slog::warn << "This model seems not Yolo-like, which has only one input, but we got " + << std::to_string(input_info_map.size()) << "inputs" << slog::endl; + return false; + } + + InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; + input_info->setPrecision(InferenceEngine::Precision::FP32); + input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); + input_info_ = input_info; + addInputInfo("input", input_info_map.begin()->first); + + // set output property + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + if (output_info_map.size() != 1) + { + slog::warn << "This model seems not Yolo-like! We got " << std::to_string(output_info_map.size()) + << "outputs, but SSDnet has only one." << slog::endl; + return false; + } + InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; + output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); + addOutputInfo("output", output_info_map.begin()->first); + slog::info << "Checking Object Detection output ... Name=" << output_info_map.begin()->first << slog::endl; + + const InferenceEngine::CNNLayerPtr output_layer = + net_reader->getNetwork().getLayerByName(output_info_map.begin()->first.c_str()); + // output layer should have attribute called num_classes + slog::info << "Checking Object Detection num_classes" << slog::endl; + if (output_layer == nullptr || output_layer->params.find("classes") == output_layer->params.end()) + { + slog::warn << "This model's output layer (" << output_info_map.begin()->first + << ") should have num_classes integer attribute" << slog::endl; + return false; + } + // class number should be equal to size of label vector + // if network has default "background" class, fake is used + const int num_classes = output_layer->GetParamAsInt("classes"); + slog::info << "Checking Object Detection output ... num_classes=" << num_classes << slog::endl; + if (getLabels().size() != num_classes) + { + if (getLabels().size() == (num_classes - 1)) + { + getLabels().insert(getLabels().begin(), "fake"); + } + else + { + getLabels().clear(); + } + } + + // last dimension of output layer should be 7 + const InferenceEngine::SizeVector output_dims = output_data_ptr->getTensorDesc().getDims(); + setMaxProposalCount(static_cast(output_dims[2])); + slog::info << "max proposal count is: " << getMaxProposalCount() << slog::endl; + + auto object_size = static_cast(output_dims[3]); + if (object_size != 33) + { + slog::warn << "This model is NOT Yolo-like, whose output data for each detected object" + << "should have 7 dimensions, but was " << std::to_string(object_size) << slog::endl; + return false; + } + setObjectSize(object_size); + + if (output_dims.size() != 2) + { + slog::warn << "This model is not Yolo-like, output dimensions shoulld be 2, but was" + << std::to_string(output_dims.size()) << slog::endl; + return false; + } + + printAttribute(); + slog::info << "This model is Yolo-like, Layer Property updated!" << slog::endl; + return true; +} + +const std::string Models::ObjectDetectionYolov2Model::getModelCategory() const +{ + return "Object Detection Yolo v2"; +} + +bool Models::ObjectDetectionYolov2Model::enqueue(const std::shared_ptr& engine, const cv::Mat& frame, + const cv::Rect& input_frame_loc) +{ + setFrameSize(frame.cols, frame.rows); + + if (!matToBlob(frame, input_frame_loc, 1, 0, engine)) + { + return false; + } + return true; +} + +bool Models::ObjectDetectionYolov2Model::matToBlob(const cv::Mat& orig_image, const cv::Rect&, float scale_factor, + int batch_index, const std::shared_ptr& engine) +{ + if (engine == nullptr) + { + slog::err << "A frame is trying to be enqueued in a NULL Engine." << slog::endl; + return false; + } + + std::string input_name = getInputName(); + InferenceEngine::Blob::Ptr input_blob = engine->getRequest()->GetBlob(input_name); + + InferenceEngine::SizeVector blob_size = input_blob->getTensorDesc().getDims(); + const int width = blob_size[3]; + const int height = blob_size[2]; + const int channels = blob_size[1]; + float* blob_data = input_blob->buffer().as(); + + int dx = 0; + int dy = 0; + int srcw = 0; + int srch = 0; + + int IH = height; + int IW = width; + + cv::Mat image = orig_image.clone(); + cv::cvtColor(image, image, cv::COLOR_BGR2RGB); + + image.convertTo(image, CV_32F, 1.0 / 255.0, 0); + srcw = image.size().width; + srch = image.size().height; + + cv::Mat resizedImg(IH, IW, CV_32FC3); + resizedImg = cv::Scalar(0.5, 0.5, 0.5); + int imw = image.size().width; + int imh = image.size().height; + float resize_ratio = static_cast(IH) / static_cast(std::max(imw, imh)); + cv::resize(image, image, cv::Size(imw * resize_ratio, imh * resize_ratio)); + + int new_w = imw; + int new_h = imh; + if ((static_cast(IW) / imw) < (static_cast(IH) / imh)) + { + new_w = IW; + new_h = (imh * IW) / imw; + } + else + { + new_h = IH; + new_w = (imw * IW) / imh; + } + dx = (IW - new_w) / 2; + dy = (IH - new_h) / 2; + + imh = image.size().height; + imw = image.size().width; + + for (int row = 0; row < imh; row++) + { + for (int col = 0; col < imw; col++) + { + for (int ch = 0; ch < 3; ch++) + { + resizedImg.at(dy + row, dx + col)[ch] = image.at(row, col)[ch]; + } + } + } + + for (int c = 0; c < channels; c++) + { + for (int h = 0; h < height; h++) + { + for (int w = 0; w < width; w++) + { + blob_data[c * width * height + h * width + w] = resizedImg.at(h, w)[c]; + } + } + } + + setFrameSize(srcw, srch); + return true; +} + +bool Models::ObjectDetectionYolov2Model::fetchResults(const std::shared_ptr& engine, + std::vector& results, + const float& confidence_thresh, const bool& enable_roi_constraint) +{ + try + { + if (engine == nullptr) + { + slog::err << "Trying to fetch results from Engines." << slog::endl; + return false; + } + + InferenceEngine::InferRequest::Ptr request = engine->getRequest(); + + std::string output = getOutputName(); + std::vector& labels = getLabels(); + const float* detections = request->GetBlob(output) + ->buffer() + .as::value_type*>(); + InferenceEngine::CNNLayerPtr layer = getNetReader()->getNetwork().getLayerByName(output.c_str()); + int input_height = input_info_->getTensorDesc().getDims()[2]; + int input_width = input_info_->getTensorDesc().getDims()[3]; + + // --------------------------- Validating output parameters -------------------------------- + if (layer != nullptr && layer->type != "RegionYolo") + { + throw std::runtime_error("Invalid output type: " + layer->type + ". RegionYolo expected"); + } + // --------------------------- Extracting layer parameters -------------------------------- + const int num = layer->GetParamAsInt("num"); + const int coords = layer->GetParamAsInt("coords"); + const int classes = layer->GetParamAsInt("classes"); + auto blob = request->GetBlob(output); + const int out_blob_h = static_cast(blob->getTensorDesc().getDims()[2]); + ; + + std::vector anchors = { 0.572730, 0.677385, 1.874460, 2.062530, 3.338430, + 5.474340, 7.882820, 3.527780, 9.770520, 9.168280 }; + auto side = out_blob_h; + + auto side_square = side * side; + // --------------------------- Parsing YOLO Region output ------------------------------------- + std::vector raw_results; + for (int i = 0; i < side_square; ++i) + { + int row = i / side; + int col = i % side; + + for (int n = 0; n < num; ++n) + { + int obj_index = getEntryIndex(side, coords, classes, n * side * side + i, coords); + int box_index = getEntryIndex(side, coords, classes, n * side * side + i, 0); + + float scale = detections[obj_index]; + + if (scale < confidence_thresh) + { + continue; + } + + float x = (col + detections[box_index + 0 * side_square]) / side * input_width; + float y = (row + detections[box_index + 1 * side_square]) / side * input_height; + float height = std::exp(detections[box_index + 3 * side_square]) * anchors[2 * n + 1] / side * input_height; + float width = std::exp(detections[box_index + 2 * side_square]) * anchors[2 * n] / side * input_width; + + for (int j = 0; j < classes; ++j) + { + int class_index = getEntryIndex(side, coords, classes, n * side_square + i, coords + 1 + j); + + float prob = scale * detections[class_index]; + if (prob < confidence_thresh) + { + continue; + } + + float x_min = x - width / 2; + float y_min = y - height / 2; + + auto frame_size = getFrameSize(); + float x_min_resized = x_min / input_width * frame_size.width; + float y_min_resized = y_min / input_height * frame_size.height; + float width_resized = width / input_width * frame_size.width; + float height_resized = height / input_height * frame_size.height; + + cv::Rect r(x_min_resized, y_min_resized, width_resized, height_resized); + Result result(r); + // result.label_ = j; + std::string label = j < labels.size() ? labels[j] : std::string("label #") + std::to_string(j); + result.setLabel(label); + + result.setConfidence(prob); + raw_results.emplace_back(result); + } + } + } + + std::sort(raw_results.begin(), raw_results.end()); + for (unsigned int i = 0; i < raw_results.size(); ++i) + { + if (raw_results[i].getConfidence() == 0) + { + continue; + } + for (unsigned int j = i + 1; j < raw_results.size(); ++j) + { + auto iou = + dynamic_vino_lib::ObjectDetection::calcIoU(raw_results[i].getLocation(), raw_results[j].getLocation()); + if (iou >= 0.45) + { + raw_results[j].setConfidence(0); + } + } + } + + for (auto& raw_result : raw_results) + { + if (raw_result.getConfidence() < confidence_thresh) + { + continue; + } + + results.push_back(raw_result); + } + + raw_results.clear(); + + return true; + } + catch (const std::exception& error) + { + slog::err << error.what() << slog::endl; + return false; + } + catch (...) + { + slog::err << "Unknown/internal exception happened." << slog::endl; + return false; + } +} + +int Models::ObjectDetectionYolov2Model::getEntryIndex(int side, int lcoords, int lclasses, int location, int entry) +{ + int n = location / (side * side); + int loc = location % (side * side); + return n * side * side * (lcoords + lclasses + 1) + entry * side * side + loc; +} diff --git a/dynamic_vino_lib/src/models/object_segmentation_model.cpp b/dynamic_vino_lib/src/models/object_segmentation_model.cpp index 2feb29c7..343fcd69 100644 --- a/dynamic_vino_lib/src/models/object_segmentation_model.cpp +++ b/dynamic_vino_lib/src/models/object_segmentation_model.cpp @@ -18,59 +18,197 @@ */ #include #include "dynamic_vino_lib/models/object_segmentation_model.h" +#include "dynamic_vino_lib/inferences/object_segmentation.h" #include "dynamic_vino_lib/slog.h" // Validated Object Detection Network -Models::ObjectSegmentationModel::ObjectSegmentationModel( - const std::string & model_loc, - int input_num, int output_num, - int max_batch_size) -: BaseModel(model_loc, input_num, output_num, max_batch_size) +Models::ObjectSegmentationModel::ObjectSegmentationModel(const std::string& model_loc, int max_batch_size) + : BaseModel(model_loc, max_batch_size) { } -void Models::ObjectSegmentationModel::setLayerProperty( - InferenceEngine::CNNNetReader::Ptr net_reader) +bool Models::ObjectSegmentationModel::enqueue(const std::shared_ptr& engine, const cv::Mat& frame, + const cv::Rect& input_frame_loc) { - // set input property - InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); - auto inputInfoItem = *input_info_map.begin(); - inputInfoItem.second->setPrecision(InferenceEngine::Precision::U8); - auto network = net_reader->getNetwork(); - try { - network.addOutput(std::string("detection_output"), 0); - } catch (std::exception & error) { - throw std::logic_error(getModelName() + "is failed when adding detection_output laryer."); + if (engine == nullptr) + { + slog::err << "A frame is trying to be enqueued in a NULL Engine." << slog::endl; + return false; } - network.setBatchSize(1); - slog::info << "Batch size is " << std::to_string(net_reader->getNetwork().getBatchSize()) << - slog::endl; - InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); - for (auto & item : output_info_map) { - item.second->setPrecision(InferenceEngine::Precision::FP32); + + for (const auto& inputInfoItem : input_info_) + { + // Fill first input tensor with images. First b channel, then g and r channels + slog::debug << "first tensor" << inputInfoItem.second->getTensorDesc().getDims().size() << slog::endl; + if (inputInfoItem.second->getTensorDesc().getDims().size() == 4) + { + matToBlob(frame, input_frame_loc, 1.0, 0, engine); + } + + // Fill second input tensor with image info + if (inputInfoItem.second->getTensorDesc().getDims().size() == 2) + { + InferenceEngine::Blob::Ptr input = engine->getRequest()->GetBlob(inputInfoItem.first); + auto data = input->buffer().as::value_type*>(); + data[0] = static_cast(frame.rows); // height + data[1] = static_cast(frame.cols); // width + data[2] = 1; + } } - auto output_ptr = output_info_map.begin(); - input_ = input_info_map.begin()->first; - detection_output_ = output_ptr->first; - mask_output_ = (++output_ptr)->first; + return true; } -void Models::ObjectSegmentationModel::checkLayerProperty( - const InferenceEngine::CNNNetReader::Ptr & net_reader) +bool Models::ObjectSegmentationModel::matToBlob(const cv::Mat& orig_image, const cv::Rect&, float scale_factor, + int batch_index, const std::shared_ptr& engine) { - const InferenceEngine::CNNLayerPtr output_layer = - net_reader->getNetwork().getLayerByName("detection_output"); - const int num_classes = output_layer->GetParamAsInt("num_classes"); - slog::info << "Checking Object Segmentation output ... num_classes=" << num_classes << slog::endl; - if (getLabels().size() != num_classes) { - if (getLabels().size() == (num_classes - 1)) { - getLabels().insert(getLabels().begin(), "fake"); - } else { - getLabels().clear(); - } + (void)scale_factor; + (void)batch_index; + + if (engine == nullptr) + { + slog::err << "A frame is trying to be enqueued in a NULL Engine." << slog::endl; + return false; + } + + size_t channels = orig_image.channels(); + size_t height = orig_image.size().height; + size_t width = orig_image.size().width; + + size_t strideH = orig_image.step.buf[0]; + size_t strideW = orig_image.step.buf[1]; + + bool is_dense = strideW == channels && strideH == channels * width; + + if (!is_dense) + { + slog::err << "Doesn't support conversion from not dense cv::Mat." << slog::endl; + return false; } + + InferenceEngine::TensorDesc tDesc(InferenceEngine::Precision::U8, { 1, channels, height, width }, + InferenceEngine::Layout::NHWC); + + auto shared_blob = InferenceEngine::make_shared_blob(tDesc, orig_image.data); + engine->getRequest()->SetBlob(getInputName(), shared_blob); + + return true; } -const std::string Models::ObjectSegmentationModel::getModelName() const +const std::string Models::ObjectSegmentationModel::getModelCategory() const { return "Object Segmentation"; } + +bool Models::ObjectSegmentationModel::updateLayerProperty(const InferenceEngine::CNNNetReader::Ptr net_reader) +{ + slog::info << "Checking INPUTS for Model" << getModelName() << slog::endl; + + auto network = net_reader->getNetwork(); + input_info_ = InferenceEngine::InputsDataMap(network.getInputsInfo()); + + InferenceEngine::ICNNNetwork::InputShapes inputShapes = network.getInputShapes(); + slog::debug << "input size" << inputShapes.size() << slog::endl; + if (inputShapes.size() != 1) + { + // throw std::runtime_error("Demo supports topologies only with 1 input"); + slog::warn << "This inference sample should have only one input, but we got" << std::to_string(inputShapes.size()) + << "inputs" << slog::endl; + return false; + } + + InferenceEngine::SizeVector& in_size_vector = inputShapes.begin()->second; + slog::debug << "channel size" << in_size_vector[1] << "dimensional" << in_size_vector.size() << slog::endl; + if (in_size_vector.size() != 4 || in_size_vector[1] != 3) + { + // throw std::runtime_error("3-channel 4-dimensional model's input is expected"); + slog::warn << "3-channel 4-dimensional model's input is expected, but we got " << std::to_string(in_size_vector[1]) + << " channels and " << std::to_string(in_size_vector.size()) << " dimensions." << slog::endl; + return false; + } + in_size_vector[0] = 1; + network.reshape(inputShapes); + + InferenceEngine::InputInfo& inputInfo = *network.getInputsInfo().begin()->second; + inputInfo.getPreProcess().setResizeAlgorithm(InferenceEngine::ResizeAlgorithm::RESIZE_BILINEAR); + inputInfo.setLayout(InferenceEngine::Layout::NHWC); + inputInfo.setPrecision(InferenceEngine::Precision::U8); + + // InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; + // addInputInfo("input", input_info_map.begin()->first.c_str()); + addInputInfo("input", inputShapes.begin()->first); + + InferenceEngine::OutputsDataMap outputsDataMap = network.getOutputsInfo(); + if (outputsDataMap.size() != 1) + { + // throw std::runtime_error("Demo supports topologies only with 1 output"); + slog::warn << "This inference sample should have only one output, but we got" + << std::to_string(outputsDataMap.size()) << "outputs" << slog::endl; + return false; + } + + InferenceEngine::Data& data = *outputsDataMap.begin()->second; + data.setPrecision(InferenceEngine::Precision::FP32); + + const InferenceEngine::SizeVector& outSizeVector = data.getTensorDesc().getDims(); + int outChannels, outHeight, outWidth; + slog::debug << "output size vector " << outSizeVector.size() << slog::endl; + switch (outSizeVector.size()) + { + case 3: + outChannels = 0; + outHeight = outSizeVector[1]; + outWidth = outSizeVector[2]; + break; + case 4: + outChannels = outSizeVector[1]; + outHeight = outSizeVector[2]; + outWidth = outSizeVector[3]; + break; + default: + throw std::runtime_error("Unexpected output blob shape. Only 4D and 3D output blobs are" + "supported."); + } + if (outHeight == 0 || outWidth == 0) + { + slog::err << "output_height or output_width is not set, please check the MaskOutput Info " + << "is set correctly." << slog::endl; + // throw std::runtime_error("output_height or output_width is not set, please check the MaskOutputInfo"); + return false; + } + + slog::debug << "output width " << outWidth << slog::endl; + slog::debug << "output hEIGHT " << outHeight << slog::endl; + slog::debug << "output CHANNALS " << outChannels << slog::endl; + addOutputInfo("masks", (outputsDataMap.begin()++)->first); + addOutputInfo("detection", outputsDataMap.begin()->first); + + // const InferenceEngine::CNNLayerPtr output_layer = + // network.getLayerByName(outputsDataMap.begin()->first.c_str()); + const InferenceEngine::CNNLayerPtr output_layer = network.getLayerByName(getOutputName("detection").c_str()); +// const int num_classes = output_layer->GetParamAsInt("num_classes"); +// slog::info << "Checking Object Segmentation output ... num_classes=" << num_classes << slog::endl; + +#if 0 + if (getLabels().size() != num_classes) + { + if (getLabels().size() == (num_classes - 1)) + { + getLabels().insert(getLabels().begin(), "fake"); + } + else + { + getLabels().clear(); + } + } +#endif + /* + const InferenceEngine::SizeVector output_dims = data.getTensorDesc().getDims(); + setMaxProposalCount(static_cast(output_dims[2])); + slog::info << "max proposal count is: " << getMaxProposalCount() << slog::endl; + auto object_size = static_cast(output_dims[3]); + setObjectSize(object_size); + + slog::debug << "model size" << output_dims.size() << slog::endl;*/ + printAttribute(); + slog::info << "This model is SSDNet-like, Layer Property updated!" << slog::endl; + return true; +} diff --git a/dynamic_vino_lib/src/models/person_attribs_detection_model.cpp b/dynamic_vino_lib/src/models/person_attribs_detection_model.cpp new file mode 100644 index 00000000..586878a8 --- /dev/null +++ b/dynamic_vino_lib/src/models/person_attribs_detection_model.cpp @@ -0,0 +1,66 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with declaration of PersonAttribsDetectionModel class + * @file person_attribs_detection_model.cpp + */ +#include +#include "dynamic_vino_lib/models/person_attribs_detection_model.h" +#include "dynamic_vino_lib/slog.h" +// Validated Person Attributes Detection Network +Models::PersonAttribsDetectionModel::PersonAttribsDetectionModel(const std::string& model_loc, int max_batch_size) + : BaseModel(model_loc, max_batch_size) +{ +} + +bool Models::PersonAttribsDetectionModel::updateLayerProperty(InferenceEngine::CNNNetReader::Ptr net_reader) +{ + slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + if (input_info_map.size() != 1) + { + throw std::logic_error("Person Attribs topology should have only one input"); + } + InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; + input_info->setPrecision(InferenceEngine::Precision::U8); + input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); + addInputInfo("input", input_info_map.begin()->first); + + slog::info << "Checking OUTPUTs for model " << getModelName() << slog::endl; + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + if (output_info_map.size() != 3) + { + throw std::logic_error("Person Attribs Network expects networks having 3 output"); + } + input_ = input_info_map.begin()->first; + output_ = output_info_map.begin()->first; + + auto output_iter = output_info_map.begin(); + InferenceEngine::DataPtr attribute_output_ptr = (output_iter++)->second; + InferenceEngine::DataPtr top_output_ptr = (output_iter++)->second; + InferenceEngine::DataPtr bottom_output_ptr = (output_iter++)->second; + + addOutputInfo("attributes_output_", attribute_output_ptr->getName()); + // output_gender_ = gender_output_ptr->name; + addOutputInfo("top_output_", top_output_ptr->getName()); + addOutputInfo("bottom_output_", bottom_output_ptr->getName()); + printAttribute(); + return true; +} + +const std::string Models::PersonAttribsDetectionModel::getModelCategory() const +{ + return "Person Attributes Detection"; +} diff --git a/dynamic_vino_lib/src/models/person_reidentification_model.cpp b/dynamic_vino_lib/src/models/person_reidentification_model.cpp index 7367e212..e0b911ce 100644 --- a/dynamic_vino_lib/src/models/person_reidentification_model.cpp +++ b/dynamic_vino_lib/src/models/person_reidentification_model.cpp @@ -20,31 +20,32 @@ #include "dynamic_vino_lib/models/person_reidentification_model.h" #include "dynamic_vino_lib/slog.h" // Validated Object Detection Network -Models::PersonReidentificationModel::PersonReidentificationModel( - const std::string & model_loc, int input_num, int output_num, int max_batch_size) -: BaseModel(model_loc, input_num, output_num, max_batch_size) {} +Models::PersonReidentificationModel::PersonReidentificationModel(const std::string& model_loc, int max_batch_size) + : BaseModel(model_loc, max_batch_size) +{ +} -void Models::PersonReidentificationModel::setLayerProperty( - InferenceEngine::CNNNetReader::Ptr net_reader) +bool Models::PersonReidentificationModel::updateLayerProperty(InferenceEngine::CNNNetReader::Ptr netreader) { - // set input property - InferenceEngine::InputsDataMap input_info_map( - net_reader->getNetwork().getInputsInfo()); + slog::info << "Checking Inputs for Model" << getModelName() << slog::endl; + + auto network = netreader->getNetwork(); + + InferenceEngine::InputsDataMap input_info_map(network.getInputsInfo()); + InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; input_info->setPrecision(InferenceEngine::Precision::U8); input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); // set output property - InferenceEngine::OutputsDataMap output_info_map( - net_reader->getNetwork().getOutputsInfo()); + InferenceEngine::OutputsDataMap output_info_map(network.getOutputsInfo()); // set input and output layer name input_ = input_info_map.begin()->first; output_ = output_info_map.begin()->first; -} -void Models::PersonReidentificationModel::checkLayerProperty( - const InferenceEngine::CNNNetReader::Ptr & net_reader) {} + return true; +} -const std::string Models::PersonReidentificationModel::getModelName() const +const std::string Models::PersonReidentificationModel::getModelCategory() const { return "Person Reidentification"; } diff --git a/dynamic_vino_lib/src/models/vehicle_attribs_detection_model.cpp b/dynamic_vino_lib/src/models/vehicle_attribs_detection_model.cpp new file mode 100644 index 00000000..825738d4 --- /dev/null +++ b/dynamic_vino_lib/src/models/vehicle_attribs_detection_model.cpp @@ -0,0 +1,66 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a realization file with declaration of VehicleAttribsDetectionModel class + * @file vehicle_attribs_detection_model.cpp + */ +#include +#include "dynamic_vino_lib/models/vehicle_attribs_detection_model.h" +#include "dynamic_vino_lib/slog.h" +// Validated Vehicle Attributes Detection Network +Models::VehicleAttribsDetectionModel::VehicleAttribsDetectionModel(const std::string& model_loc, int max_batch_size) + : BaseModel(model_loc, max_batch_size) +{ +} + +bool Models::VehicleAttribsDetectionModel::updateLayerProperty(InferenceEngine::CNNNetReader::Ptr net_reader) +{ + slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; + // set input property + InferenceEngine::InputsDataMap input_info_map(net_reader->getNetwork().getInputsInfo()); + if (input_info_map.size() != 1) + { + throw std::logic_error("Vehicle Attribs topology should have only one input"); + } + InferenceEngine::OutputsDataMap output_info_map(net_reader->getNetwork().getOutputsInfo()); + if (output_info_map.size() != 2) + { + throw std::logic_error("Vehicle Attribs Network expects networks having two outputs"); + } + + InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; + input_info->setPrecision(InferenceEngine::Precision::U8); + input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); + + // set input and output layer name + input_ = input_info_map.begin()->first; + auto output_iter = output_info_map.begin(); + // color_output_ = (output_iter++)->second->name; + // type_output_ = (output_iter++)->second->name; + InferenceEngine::DataPtr color_output_ptr = (output_iter++)->second; + InferenceEngine::DataPtr type_output_ptr = (output_iter++)->second; + + addOutputInfo("color_output_", color_output_ptr->getName()); + // output_gender_ = gender_output_ptr->name; + addOutputInfo("type_output_", type_output_ptr->getName()); + + printAttribute(); + return true; +} + +const std::string Models::VehicleAttribsDetectionModel::getModelCategory() const +{ + return "Vehicle Attributes Detection"; +} diff --git a/dynamic_vino_lib/src/outputs/base_output.cpp b/dynamic_vino_lib/src/outputs/base_output.cpp index a3248c8d..e3d333ec 100644 --- a/dynamic_vino_lib/src/outputs/base_output.cpp +++ b/dynamic_vino_lib/src/outputs/base_output.cpp @@ -19,11 +19,14 @@ void Outputs::BaseOutput::setPipeline(Pipeline* const pipeline) { - //pipeline->printPipeline(); + // pipeline->printPipeline(); pipeline_ = pipeline; } -Pipeline* Outputs::BaseOutput::getPipeline() const { return pipeline_; } +Pipeline* Outputs::BaseOutput::getPipeline() const +{ + return pipeline_; +} cv::Mat Outputs::BaseOutput::getFrame() const { diff --git a/dynamic_vino_lib/src/outputs/image_window_output.cpp b/dynamic_vino_lib/src/outputs/image_window_output.cpp index 3a6f2df1..e1286a50 100644 --- a/dynamic_vino_lib/src/outputs/image_window_output.cpp +++ b/dynamic_vino_lib/src/outputs/image_window_output.cpp @@ -23,12 +23,16 @@ #include #include #include +#include +#include + +#include "dynamic_vino_lib/outputs/image_window_output.h" #include "dynamic_vino_lib/pipeline.h" -Outputs::ImageWindowOutput::ImageWindowOutput(const std::string& window_name, - int focal_length) - : window_name_(window_name), focal_length_(focal_length) +Outputs::ImageWindowOutput::ImageWindowOutput(const std::string& output_name, int focal_length) + : BaseOutput(output_name), focal_length_(focal_length) { + cv::namedWindow(output_name_, cv::WINDOW_AUTOSIZE); } void Outputs::ImageWindowOutput::feedFrame(const cv::Mat& frame) @@ -48,92 +52,94 @@ void Outputs::ImageWindowOutput::feedFrame(const cv::Mat& frame) } } -void Outputs::ImageWindowOutput::accept( - const std::vector& results) +unsigned Outputs::ImageWindowOutput::findOutput(const cv::Rect& result_rect) { - // std::cout<<"call face"<& results) +{ + for (unsigned i = 0; i < results.size(); i++) { - // throw std::logic_error("size is not equal!"); - slog::err << "the size of Face Detection and Output Vector is not equal!" - << slog::endl; - return; + cv::Rect result_rect = results[i].getLocation(); + unsigned target_index = findOutput(result_rect); + outputs_[target_index].rect = result_rect; + outputs_[target_index].desc += ("[" + results[i].getLicense() + "]"); } +} +void Outputs::ImageWindowOutput::accept(const std::vector& results) +{ for (unsigned i = 0; i < results.size(); i++) { - // outputs_[i].desc.str(""); - outputs_[i].rect = results[i].getLocation(); + cv::Rect result_rect = results[i].getLocation(); + unsigned target_index = findOutput(result_rect); + outputs_[target_index].rect = result_rect; + outputs_[target_index].desc += ("[" + results[i].getColor() + "," + results[i].getType() + "]"); + } +} +void Outputs::ImageWindowOutput::accept(const std::vector& results) +{ + for (unsigned i = 0; i < results.size(); i++) + { + cv::Rect result_rect = results[i].getLocation(); + unsigned target_index = findOutput(result_rect); + outputs_[target_index].rect = result_rect; auto fd_conf = results[i].getConfidence(); if (fd_conf >= 0) { std::ostringstream ostream; ostream << "[" << std::fixed << std::setprecision(3) << fd_conf << "]"; - outputs_[i].desc += ostream.str(); + outputs_[target_index].desc += ostream.str(); } } } -void Outputs::ImageWindowOutput::accept( - const std::vector& results) +void Outputs::ImageWindowOutput::accept(const std::vector& results) { - if (outputs_.size() == 0) - { - initOutputs(results.size()); - } - if (outputs_.size() != results.size()) - { - // throw std::logic_error("size is not equal!"); - slog::err << "the size of Emotion Detection and Output Vector is not equal!" - << slog::endl; - return; - } for (unsigned i = 0; i < results.size(); i++) { + cv::Rect result_rect = results[i].getLocation(); + unsigned target_index = findOutput(result_rect); + outputs_[target_index].rect = result_rect; std::ostringstream ostream; ostream << "[" << results[i].getLabel() << "]"; - outputs_[i].desc += ostream.str(); + outputs_[target_index].desc += ostream.str(); } } -void Outputs::ImageWindowOutput::accept( - const std::vector& results) +void Outputs::ImageWindowOutput::accept(const std::vector& results) { - if (outputs_.size() == 0) - { - initOutputs(results.size()); - } - if (outputs_.size() != results.size()) - { - // throw std::logic_error("size is not equal!"); - slog::err - << "the size of AgeGender Detection and Output Vector is not equal!" - << slog::endl; - return; - } for (unsigned i = 0; i < results.size(); i++) { + cv::Rect result_rect = results[i].getLocation(); + unsigned target_index = findOutput(result_rect); + outputs_[target_index].rect = result_rect; std::ostringstream ostream; - // auto age = results[i].getAge(); - ostream << "[Y" << std::fixed << std::setprecision(0) << results[i].getAge() - << "]"; - outputs_[i].desc += ostream.str(); + ostream << "[Y" << std::fixed << std::setprecision(0) << results[i].getAge() << "]"; + outputs_[target_index].desc += ostream.str(); auto male_prob = results[i].getMaleProbability(); if (male_prob < 0.5) { - outputs_[i].scalar = cv::Scalar(0, 0, 255); + outputs_[target_index].scalar = cv::Scalar(0, 0, 255); } } } -cv::Point Outputs::ImageWindowOutput::calcAxis(cv::Mat r, double cx, double cy, - double cz, cv::Point cp) +cv::Point Outputs::ImageWindowOutput::calcAxis(cv::Mat r, double cx, double cy, double cz, cv::Point cp) { cv::Mat Axis(3, 1, CV_32F); Axis.at(0) = cx; @@ -143,48 +149,31 @@ cv::Point Outputs::ImageWindowOutput::calcAxis(cv::Mat r, double cx, double cy, o.at(2) = camera_matrix_.at(0); Axis = r * Axis + o; cv::Point point; - point.x = static_cast( - (Axis.at(0) / Axis.at(2) * camera_matrix_.at(0)) + - cp.x); - point.y = static_cast( - (Axis.at(1) / Axis.at(2) * camera_matrix_.at(4)) + - cp.y); + point.x = static_cast((Axis.at(0) / Axis.at(2) * camera_matrix_.at(0)) + cp.x); + point.y = static_cast((Axis.at(1) / Axis.at(2) * camera_matrix_.at(4)) + cp.y); return point; } -cv::Mat Outputs::ImageWindowOutput::getRotationTransform(double yaw, - double pitch, - double roll) +cv::Mat Outputs::ImageWindowOutput::getRotationTransform(double yaw, double pitch, double roll) { pitch *= CV_PI / 180.0; yaw *= CV_PI / 180.0; roll *= CV_PI / 180.0; - cv::Matx33f Rx(1, 0, 0, 0, cos(pitch), -sin(pitch), 0, sin(pitch), - cos(pitch)); + cv::Matx33f Rx(1, 0, 0, 0, cos(pitch), -sin(pitch), 0, sin(pitch), cos(pitch)); cv::Matx33f Ry(cos(yaw), 0, -sin(yaw), 0, 1, 0, sin(yaw), 0, cos(yaw)); cv::Matx33f Rz(cos(roll), -sin(roll), 0, sin(roll), cos(roll), 0, 0, 0, 1); auto r = cv::Mat(Rz * Ry * Rx); return r; } -void Outputs::ImageWindowOutput::accept( - const std::vector& results) +void Outputs::ImageWindowOutput::accept(const std::vector& results) { - if (outputs_.size() == 0) - { - initOutputs(results.size()); - } - if (outputs_.size() != results.size()) - { - // throw std::logic_error("size is not equal!"); - slog::err - << "the size of HeadPose Detection and Output Vector is not equal!" - << slog::endl; - return; - } for (unsigned i = 0; i < results.size(); i++) { auto result = results[i]; + cv::Rect result_rect = result.getLocation(); + unsigned target_index = findOutput(result_rect); + outputs_[target_index].rect = result_rect; double yaw = result.getAngleY(); double pitch = result.getAngleP(); double roll = result.getAngleR(); @@ -192,51 +181,37 @@ void Outputs::ImageWindowOutput::accept( feedFrame(frame_); cv::Mat r = getRotationTransform(yaw, pitch, roll); cv::Rect location = result.getLocation(); - auto cp = cv::Point(location.x + location.width / 2, - location.y + location.height / 2); - outputs_[i].hp_cp = cp; - outputs_[i].hp_x = calcAxis(r, scale, 0, 0, cp); - outputs_[i].hp_y = calcAxis(r, 0, -scale, 0, cp); - outputs_[i].hp_ze = calcAxis(r, 0, 0, -scale, cp); - outputs_[i].hp_zs = calcAxis(r, 0, 0, scale, cp); + auto cp = cv::Point(location.x + location.width / 2, location.y + location.height / 2); + outputs_[target_index].hp_cp = cp; + outputs_[target_index].hp_x = calcAxis(r, scale, 0, 0, cp); + outputs_[target_index].hp_y = calcAxis(r, 0, -scale, 0, cp); + outputs_[target_index].hp_ze = calcAxis(r, 0, 0, -scale, cp); + outputs_[target_index].hp_zs = calcAxis(r, 0, 0, scale, cp); } } -void Outputs::ImageWindowOutput::accept( - const std::vector& results) +void Outputs::ImageWindowOutput::accept(const std::vector& results) { - // std::cout<<"call"<= 0) { std::ostringstream ostream; ostream << "[" << std::fixed << std::setprecision(3) << fd_conf << "]"; - outputs_[i].desc += ostream.str(); + outputs_[target_index].desc += ostream.str(); } auto label = results[i].getLabel(); - outputs_[i].desc += "[" + label + "]"; - // std::cout<<"out:" << label < & results) +void Outputs::ImageWindowOutput::mergeMask(const std::vector& results) { + /* std::map class_color; for (unsigned i = 0; i < results.size(); i++) { std::string class_label = results[i].getLabel(); @@ -263,22 +238,31 @@ void Outputs::ImageWindowOutput::mergeMask( } cv::addWeighted(colored_mask, alpha, roi_img, 1.0f - alpha, 0.0f, roi_img); } + */ + const float alpha = 0.5f; + cv::Mat roi_img = frame_; + cv::Mat colored_mask = results[0].getMask(); + cv::resize(colored_mask, colored_mask, cv::Size(frame_.size().width, frame_.size().height)); + cv::addWeighted(colored_mask, alpha, roi_img, 1.0f - alpha, 0.0f, roi_img); } -void Outputs::ImageWindowOutput::accept( - const std::vector & results) +void Outputs::ImageWindowOutput::accept(const std::vector& results) { - if (outputs_.size() == 0) { + if (outputs_.size() == 0) + { initOutputs(results.size()); } - if (outputs_.size() != results.size()) { + if (outputs_.size() != results.size()) + { slog::err << "the size of Object Segmentation and Output Vector is not equal!" << slog::endl; return; } - for (unsigned i = 0; i < results.size(); i++) { + for (unsigned i = 0; i < results.size(); i++) + { outputs_[i].rect = results[i].getLocation(); auto fd_conf = results[i].getConfidence(); - if (fd_conf >= 0) { + if (fd_conf >= 0) + { std::ostringstream ostream; ostream << "[" << std::fixed << std::setprecision(3) << fd_conf << "]"; outputs_[i].desc += ostream.str(); @@ -289,25 +273,34 @@ void Outputs::ImageWindowOutput::accept( mergeMask(results); } -unsigned Outputs::ImageWindowOutput::findOutput( - const cv::Rect & result_rect) +void Outputs::ImageWindowOutput::accept(const std::vector& results) { - for (unsigned i = 0; i < outputs_.size(); i++) { - if (outputs_[i].rect == result_rect) { - return i; + for (unsigned i = 0; i < results.size(); i++) + { + cv::Rect result_rect = results[i].getLocation(); + unsigned target_index = findOutput(result_rect); + if (results[i].getMaleProbability() < 0.5) + { + outputs_[target_index].scalar = cv::Scalar(0, 0, 255); + } + else + { + outputs_[target_index].scalar = cv::Scalar(0, 255, 0); } + outputs_[target_index].pa_top.x = results[i].getTopLocation().x * result_rect.width + result_rect.x; + outputs_[target_index].pa_top.y = results[i].getTopLocation().y * result_rect.height + result_rect.y; + outputs_[target_index].pa_bottom.x = results[i].getBottomLocation().x * result_rect.width + result_rect.x; + outputs_[target_index].pa_bottom.y = results[i].getBottomLocation().y * result_rect.height + result_rect.y; + + outputs_[target_index].rect = result_rect; + outputs_[target_index].desc += "[" + results[i].getAttributes() + "]"; } - OutputData output; - output.desc = ""; - output.scalar = cv::Scalar(255, 0, 0); - outputs_.push_back(output); - return outputs_.size() - 1; } -void Outputs::ImageWindowOutput::accept( - const std::vector & results) +void Outputs::ImageWindowOutput::accept(const std::vector& results) { - for (unsigned i = 0; i < results.size(); i++) { + for (unsigned i = 0; i < results.size(); i++) + { cv::Rect result_rect = results[i].getLocation(); unsigned target_index = findOutput(result_rect); outputs_[target_index].rect = result_rect; @@ -315,6 +308,31 @@ void Outputs::ImageWindowOutput::accept( } } +void Outputs::ImageWindowOutput::accept(const std::vector& results) +{ + for (unsigned i = 0; i < results.size(); i++) + { + cv::Rect result_rect = results[i].getLocation(); + unsigned target_index = findOutput(result_rect); + outputs_[target_index].rect = result_rect; + outputs_[target_index].desc += "[" + results[i].getFaceID() + "]"; + } +} + +void Outputs::ImageWindowOutput::accept(const std::vector& results) +{ + for (unsigned i = 0; i < results.size(); i++) + { + cv::Rect result_rect = results[i].getLocation(); + unsigned target_index = findOutput(result_rect); + std::vector landmark_points = results[i].getLandmarks(); + for (int j = 0; j < landmark_points.size(); j++) + { + outputs_[target_index].landmarks.push_back(landmark_points[j]); + } + } +} + void Outputs::ImageWindowOutput::decorateFrame() { if (getPipeline()->getParameters()->isGetFps()) @@ -322,41 +340,43 @@ void Outputs::ImageWindowOutput::decorateFrame() int fps = getPipeline()->getFPS(); std::stringstream ss; ss << "FPS: " << fps; - cv::putText(frame_, ss.str(), cv::Point2f(0, 65), cv::FONT_HERSHEY_TRIPLEX, - 0.5, cv::Scalar(255, 0, 0)); + cv::putText(frame_, ss.str(), cv::Point2f(0, 65), cv::FONT_HERSHEY_TRIPLEX, 0.5, cv::Scalar(255, 0, 0)); } for (auto o : outputs_) { auto new_y = std::max(15, o.rect.y - 15); - cv::putText(frame_, o.desc, cv::Point2f(o.rect.x, new_y), - cv::FONT_HERSHEY_COMPLEX_SMALL, 0.8, o.scalar); + cv::putText(frame_, o.desc, cv::Point2f(o.rect.x, new_y), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.8, o.scalar); cv::rectangle(frame_, o.rect, o.scalar, 1); - cv::line(frame_, o.hp_cp, o.hp_x, cv::Scalar(0, 0, 255), 2); - cv::line(frame_, o.hp_cp, o.hp_y, cv::Scalar(0, 255, 0), 2); - cv::line(frame_, o.hp_zs, o.hp_ze, cv::Scalar(255, 0, 0), 2); - cv::circle(frame_, o.hp_ze, 3, cv::Scalar(255, 0, 0), 2); + if (o.hp_cp != o.hp_x) + { + cv::line(frame_, o.hp_cp, o.hp_x, cv::Scalar(0, 0, 255), 2); + } + if (o.hp_cp != o.hp_y) + { + cv::line(frame_, o.hp_cp, o.hp_y, cv::Scalar(0, 255, 0), 2); + } + if (o.hp_zs != o.hp_ze) + { + cv::line(frame_, o.hp_zs, o.hp_ze, cv::Scalar(255, 0, 0), 2); + cv::circle(frame_, o.hp_ze, 3, cv::Scalar(255, 0, 0), 2); + } + + for (int i = 0; i < o.landmarks.size(); i++) + { + cv::circle(frame_, o.landmarks[i], 3, cv::Scalar(255, 0, 0), 2); + } } outputs_.clear(); } void Outputs::ImageWindowOutput::handleOutput() { - cv::namedWindow(window_name_, cv::WINDOW_AUTOSIZE); - decorateFrame(); - cv::imshow(window_name_, frame_); - if( cv::waitKey(1) > 0) + if (frame_.cols == 0 || frame_.rows == 0) { - ros::shutdown(); - } -} - -void Outputs::ImageWindowOutput::initOutputs(unsigned size) -{ - outputs_.resize(size); - for (unsigned i = 0; i < size; i++) - { - outputs_[i].desc = ""; - outputs_[i].scalar = cv::Scalar(255, 0, 0); + return; } + decorateFrame(); + cv::imshow(output_name_, frame_); + cv::waitKey(1); } diff --git a/dynamic_vino_lib/src/outputs/ros_service_output.cpp b/dynamic_vino_lib/src/outputs/ros_service_output.cpp index f05c21d7..414b8c8d 100644 --- a/dynamic_vino_lib/src/outputs/ros_service_output.cpp +++ b/dynamic_vino_lib/src/outputs/ros_service_output.cpp @@ -23,107 +23,113 @@ #include "dynamic_vino_lib/outputs/ros_service_output.h" #include "cv_bridge/cv_bridge.h" #include -// Outputs::RosServiceOutput::RosServiceOutput() - -void Outputs::RosServiceOutput::setServiceResponse( - boost::shared_ptr response) +void Outputs::RosServiceOutput::setServiceResponse(boost::shared_ptr response) { - if (object_msg_ptr_ != nullptr && object_msg_ptr_->objects_vector.size() > 0) { + if (detected_objects_topic_ != nullptr && detected_objects_topic_->objects_vector.size() > 0) + { object_msgs::ObjectsInBoxes objs; - objs.objects_vector = object_msg_ptr_->objects_vector; + objs.objects_vector = detected_objects_topic_->objects_vector; response->objects.push_back(objs); - } else if (faces_msg_ptr_ != nullptr && faces_msg_ptr_ ->objects_vector.size() > 0) { - object_msgs::ObjectsInBoxes objs; - objs.objects_vector = faces_msg_ptr_->objects_vector; + } + else if (faces_topic_ != nullptr && faces_topic_->objects_vector.size() > 0) + { + object_msgs::ObjectsInBoxes objs; + objs.objects_vector = faces_topic_->objects_vector; response->objects.push_back(objs); } } -void Outputs::RosServiceOutput::setResponseForFace( - boost::shared_ptr response) +void Outputs::RosServiceOutput::setResponseForFace(boost::shared_ptr response) { - if (faces_msg_ptr_ != nullptr && faces_msg_ptr_->objects_vector.size() > 0) { - object_msgs::ObjectsInBoxes objs; - objs.objects_vector = faces_msg_ptr_->objects_vector; + if (faces_topic_ != nullptr && faces_topic_->objects_vector.size() > 0) + { + object_msgs::ObjectsInBoxes objs; + objs.objects_vector = faces_topic_->objects_vector; response->objects.push_back(objs); } } -void Outputs::RosServiceOutput::setServiceResponse( - boost::shared_ptr response) +void Outputs::RosServiceOutput::setServiceResponse(boost::shared_ptr response) { - if (age_gender_msg_ptr_ != nullptr) { - response->age_gender.objects = age_gender_msg_ptr_->objects; + if (age_gender_topic_ != nullptr) + { + response->age_gender.objects = age_gender_topic_->objects; } } -void Outputs::RosServiceOutput::setServiceResponse( - boost::shared_ptr response) +void Outputs::RosServiceOutput::setServiceResponse(boost::shared_ptr response) { - if (emotions_msg_ptr_ != nullptr) { - response->emotion.emotions = emotions_msg_ptr_->emotions; + if (emotions_topic_ != nullptr) + { + response->emotion.emotions = emotions_topic_->emotions; } } -void Outputs::RosServiceOutput::setServiceResponse( - boost::shared_ptr response) +void Outputs::RosServiceOutput::setServiceResponse(boost::shared_ptr response) { - if (headpose_msg_ptr_ != nullptr) { - response->headpose.headposes = headpose_msg_ptr_->headposes; + if (headpose_topic_ != nullptr) + { + response->headpose.headposes = headpose_topic_->headposes; } } -void Outputs::RosServiceOutput::setServiceResponse( - boost::shared_ptr response) +void Outputs::RosServiceOutput::setServiceResponse(boost::shared_ptr response) +{ + slog::info << "in ObjectsInMasks service::Response ..."; + if (segmented_objects_topic_ != nullptr) { - slog::info << "in ObjectsInMasks service::Response ..."; - if (segmented_object_msg_ptr_ != nullptr) { - response->segmentation.objects_vector = segmented_object_msg_ptr_->objects_vector; - } + response->segmentation.objects_vector = segmented_objects_topic_->objects_vector; } +} void Outputs::RosServiceOutput::setServiceResponse( - boost::shared_ptr response) + boost::shared_ptr response) +{ + slog::info << "in Reidentification service::Response ..."; + if (person_reid_topic_ != nullptr) { - slog::info << "in Reidentification service::Response ..."; - if (person_reid_msg_ptr_ != nullptr) { - response->reidentification.reidentified_vector = person_reid_msg_ptr_->reidentified_vector; - } + response->reidentification.reidentified_vector = person_reid_msg_ptr_->reidentified_vector; } +} -void Outputs::RosServiceOutput::setServiceResponse( - boost::shared_ptr response) +void Outputs::RosServiceOutput::setServiceResponse(boost::shared_ptr response) { slog::info << "in People::Response ..."; - if (faces_msg_ptr_ != nullptr) { + if (faces_topic_ != nullptr) + { slog::info << "[FACES],"; - response->persons.faces = faces_msg_ptr_->objects_vector; - } else if (object_msg_ptr_ != nullptr) { + response->persons.faces = faces_topic_->objects_vector; + } + else if (detected_objects_topic_ != nullptr) + { slog::info << "[FACES(objects)],"; - response->persons.faces = object_msg_ptr_->objects_vector; + response->persons.faces = detected_objects_topic_->objects_vector; } - if (age_gender_msg_ptr_ != nullptr) { + if (age_gender_topic_ != nullptr) + { slog::info << "[AGE_GENDER],"; - response->persons.agegenders = age_gender_msg_ptr_->objects; + response->persons.agegenders = age_gender_topic_->objects; } - if (emotions_msg_ptr_ != nullptr) { + if (emotions_topic_ != nullptr) + { slog::info << "[EMOTION],"; - response->persons.emotions = emotions_msg_ptr_->emotions; + response->persons.emotions = emotions_topic_->emotions; } - if (headpose_msg_ptr_ != nullptr) { + if (headpose_topic_ != nullptr) + { slog::info << "[HEADPOSE],"; - response->persons.headposes = headpose_msg_ptr_->headposes; + response->persons.headposes = headpose_topic_->headposes; } slog::info << "DONE!" << slog::endl; } void Outputs::RosServiceOutput::clearData() { - faces_msg_ptr_ = nullptr; - object_msg_ptr_ = nullptr; - age_gender_msg_ptr_ = nullptr; - emotions_msg_ptr_ = nullptr; - headpose_msg_ptr_ = nullptr; - segmented_object_msg_ptr_ = nullptr; - person_reid_msg_ptr_ = nullptr; + faces_topic_ = nullptr; + detected_objects_topic_ = nullptr; + age_gender_topic_ = nullptr; + emotions_topic_ = nullptr; + headpose_topic_ = nullptr; + segmented_object_topic_ = nullptr; + person_reid_topic_ = nullptr; } diff --git a/dynamic_vino_lib/src/outputs/ros_topic_output.cpp b/dynamic_vino_lib/src/outputs/ros_topic_output.cpp index 3dab5174..e62b8bc9 100644 --- a/dynamic_vino_lib/src/outputs/ros_topic_output.cpp +++ b/dynamic_vino_lib/src/outputs/ros_topic_output.cpp @@ -25,41 +25,124 @@ #include #include -Outputs::RosTopicOutput::RosTopicOutput() +Outputs::RosTopicOutput::RosTopicOutput(std::string pipeline_name) : pipeline_name_(pipeline_name) { - pub_face_ = - nh_.advertise("/openvino_toolkit/faces", 16); - pub_emotion_ = nh_.advertise( - "/openvino_toolkit/emotions", 16); - pub_age_gender_ = nh_.advertise( - "/openvino_toolkit/age_genders", 16); - pub_headpose_ = nh_.advertise( - "/openvino_toolkit/headposes", 16); - pub_object_ = nh_.advertise( - "/openvino_toolkit/detected_objects", 16); + pub_face_ = nh_.advertise("/openvino_toolkit/" + pipeline_name_ + "/faces", 16); + pub_emotion_ = nh_.advertise("/openvino_toolkit/" + pipeline_name_ + "/emotions", 16); + pub_age_gender_ = + nh_.advertise("/openvino_toolkit/" + pipeline_name_ + "/age_genders", 16); + pub_headpose_ = nh_.advertise("/openvino_toolkit/" + pipeline_name_ + "/headposes", 16); + pub_object_ = + nh_.advertise("/openvino_toolkit/" + pipeline_name_ + "/detected_objects", 16); pub_person_reid_ = nh_.advertise( - "/openvino_toolkit/reidentified_persons", 16); - pub_segmented_object_ = nh_.advertise( - "/openvino_toolkit/segmented_obejcts", 16); - - emotions_msg_ptr_ = NULL; - faces_msg_ptr_ = NULL; - age_gender_msg_ptr_ = NULL; - headpose_msg_ptr_ = NULL; - object_msg_ptr_ = NULL; + "/openvino_toolkit/" + pipeline_name_ + "/reidentified_persons", 16); + pub_segmented_object_ = + nh_.advertise("/openvino_toolkit/" + pipeline_name_ + "/segmented_obejcts", 16); + pub_face_reid_ = nh_.advertise( + "/openvino_toolkit/" + pipeline_name_ + "/reidentified_faces", 16); + pub_person_attribs_ = nh_.advertise( + "/openvino_toolkit/" + pipeline_name_ + "/person_attributes", 16); + pub_license_plate_ = nh_.advertise( + "/openvino_toolkit/" + pipeline_name_ + "/detected_license_plates", 16); + pub_vehicle_attribs_ = nh_.advertise( + "/openvino_toolkit/" + pipeline_name_ + "/detected_vehicles_attribs", 16); + pub_landmarks_ = nh_.advertise( + "/openvino_toolkit/" + pipeline_name_ + "/detected_landmarks", 16); + + emotions_topic_ = NULL; + faces_topic_ = NULL; + age_gender_topic_ = NULL; + headpose_topic_ = NULL; + detected_objects_topic_ = NULL; person_reid_msg_ptr_ = NULL; - segmented_object_msg_ptr_ = NULL; + segmented_objects_topic_ = NULL; + face_reid_topic_ = NULL; + person_attribs_topic_ = NULL; + license_plate_topic_ = NULL; + vehicle_attribs_topic_ = NULL; + landmarks_topic_ = NULL; +} + +void Outputs::RosTopicOutput::feedFrame(const cv::Mat& frame) +{ + frame_ = frame.clone(); +} + +void Outputs::RosTopicOutput::accept(const std::vector& results) +{ + vehicle_attribs_topic_ = std::make_shared(); + people_msgs::VehicleAttribs attribs; + for (auto& r : results) + { + // slog::info << ">"; + auto loc = r.getLocation(); + attribs.roi.x_offset = loc.x; + attribs.roi.y_offset = loc.y; + attribs.roi.width = loc.width; + attribs.roi.height = loc.height; + attribs.type = r.getType(); + attribs.color = r.getColor(); + vehicle_attribs_topic_->vehicles.push_back(attribs); + } +} + +void Outputs::RosTopicOutput::accept(const std::vector& results) +{ + license_plate_topic_ = std::make_shared(); + people_msgs::LicensePlate plate; + for (auto& r : results) + { + // slog::info << ">"; + auto loc = r.getLocation(); + plate.roi.x_offset = loc.x; + plate.roi.y_offset = loc.y; + plate.roi.width = loc.width; + plate.roi.height = loc.height; + plate.license = r.getLicense(); + license_plate_topic_->licenses.push_back(plate); + } } -void Outputs::RosTopicOutput::feedFrame(const cv::Mat& frame) {} +void Outputs::RosTopicOutput::accept(const std::vector& results) +{ + person_attribs_topic_ = std::make_shared(); + people_msgs::PersonAttribute person_attrib; + for (auto& r : results) + { + // slog::info << ">"; + auto loc = r.getLocation(); + person_attrib.roi.x_offset = loc.x; + person_attrib.roi.y_offset = loc.y; + person_attrib.roi.width = loc.width; + person_attrib.roi.height = loc.height; + person_attrib.attribute = r.getAttributes(); + person_attribs_topic_->attributes.push_back(person_attrib); + } +} +void Outputs::RosTopicOutput::accept(const std::vector& results) +{ + face_reid_topic_ = std::make_shared(); + people_msgs::Reidentification face; + for (auto& r : results) + { + // slog::info << ">"; + auto loc = r.getLocation(); + face.roi.x_offset = loc.x; + face.roi.y_offset = loc.y; + face.roi.width = loc.width; + face.roi.height = loc.height; + face.identity = r.getFaceID(); + face_reid_topic_->reidentified_vector.push_back(face); + } +} -void Outputs::RosTopicOutput::accept( - const std::vector & results) +void Outputs::RosTopicOutput::accept(const std::vector& results) { person_reid_msg_ptr_ = std::make_shared(); people_msgs::Reidentification person; - for (auto & r : results) { + for (auto& r : results) + { // slog::info << ">"; auto loc = r.getLocation(); person.roi.x_offset = loc.x; @@ -71,12 +154,12 @@ void Outputs::RosTopicOutput::accept( } } -void Outputs::RosTopicOutput::accept( - const std::vector & results) +void Outputs::RosTopicOutput::accept(const std::vector& results) { - segmented_object_msg_ptr_ = std::make_shared(); + segmented_objects_topic_ = std::make_shared(); people_msgs::ObjectInMask object; - for (auto & r : results) { + for (auto& r : results) + { // slog::info << ">"; auto loc = r.getLocation(); object.roi.x_offset = loc.x; @@ -86,20 +169,20 @@ void Outputs::RosTopicOutput::accept( object.object_name = r.getLabel(); object.probability = r.getConfidence(); cv::Mat mask = r.getMask(); - for (int h = 0; h < mask.size().height; ++h) { - for (int w = 0; w < mask.size().width; ++w) { + for (int h = 0; h < mask.size().height; ++h) + { + for (int w = 0; w < mask.size().width; ++w) + { object.mask_array.push_back(mask.at(h, w)); } } - segmented_object_msg_ptr_->objects_vector.push_back(object); + segmented_objects_topic_->objects_vector.push_back(object); } } - -void Outputs::RosTopicOutput::accept( - const std::vector& results) +void Outputs::RosTopicOutput::accept(const std::vector& results) { - faces_msg_ptr_ = std::make_shared(); + faces_topic_ = std::make_shared(); object_msgs::ObjectInBox face; for (auto r : results) @@ -111,14 +194,13 @@ void Outputs::RosTopicOutput::accept( face.roi.height = loc.height; face.object.object_name = r.getLabel(); face.object.probability = r.getConfidence(); - faces_msg_ptr_->objects_vector.push_back(face); + faces_topic_->objects_vector.push_back(face); } } -void Outputs::RosTopicOutput::accept( - const std::vector& results) +void Outputs::RosTopicOutput::accept(const std::vector& results) { - emotions_msg_ptr_ = std::make_shared(); + emotions_topic_ = std::make_shared(); people_msgs::Emotion emotion; for (auto r : results) @@ -129,14 +211,13 @@ void Outputs::RosTopicOutput::accept( emotion.roi.width = loc.width; emotion.roi.height = loc.height; emotion.emotion = r.getLabel(); - emotions_msg_ptr_->emotions.push_back(emotion); + emotions_topic_->emotions.push_back(emotion); } } -void Outputs::RosTopicOutput::accept( - const std::vector& results) +void Outputs::RosTopicOutput::accept(const std::vector& results) { - age_gender_msg_ptr_ = std::make_shared(); + age_gender_topic_ = std::make_shared(); people_msgs::AgeGender ag; for (auto r : results) @@ -158,14 +239,13 @@ void Outputs::RosTopicOutput::accept( ag.gender = "Female"; ag.gender_confidence = 1.0 - male_prob; } - age_gender_msg_ptr_->objects.push_back(ag); + age_gender_topic_->objects.push_back(ag); } } -void Outputs::RosTopicOutput::accept( - const std::vector& results) +void Outputs::RosTopicOutput::accept(const std::vector& results) { - headpose_msg_ptr_ = std::make_shared(); + headpose_topic_ = std::make_shared(); people_msgs::HeadPose hp; for (auto r : results) @@ -178,14 +258,13 @@ void Outputs::RosTopicOutput::accept( hp.yaw = r.getAngleY(); hp.pitch = r.getAngleP(); hp.roll = r.getAngleR(); - headpose_msg_ptr_->headposes.push_back(hp); + headpose_topic_->headposes.push_back(hp); } } -void Outputs::RosTopicOutput::accept( - const std::vector& results) +void Outputs::RosTopicOutput::accept(const std::vector& results) { - object_msg_ptr_ = std::make_shared(); + detected_objects_topic_ = std::make_shared(); object_msgs::ObjectInBox hp; for (auto r : results) @@ -197,84 +276,149 @@ void Outputs::RosTopicOutput::accept( hp.roi.height = loc.height; hp.object.object_name = r.getLabel(); hp.object.probability = r.getConfidence(); - object_msg_ptr_->objects_vector.push_back(hp); + detected_objects_topic_->objects_vector.push_back(hp); + } +} + +void Outputs::RosTopicOutput::accept(const std::vector& results) +{ + landmarks_topic_ = std::make_shared(); + people_msgs::msg::Landmark landmark; + for (auto& r : results) + { + // slog::info << ">"; + auto loc = r.getLocation(); + landmark.roi.x_offset = loc.x; + landmark.roi.y_offset = loc.y; + landmark.roi.width = loc.width; + landmark.roi.height = loc.height; + std::vector landmark_points = r.getLandmarks(); + for (auto pt : landmark_points) + { + geometry_msgs::msg::Point point; + point.x = pt.x; + point.y = pt.y; + landmark.landmark_points.push_back(point); + } + landmarks_topic_->landmarks.push_back(landmark); } } void Outputs::RosTopicOutput::handleOutput() { - std_msgs::Header header = getHeader(); - if (person_reid_msg_ptr_ != nullptr) { + // std_msgs::Header header = getHeader(); + auto header = getPipeline()->getInputDevice()->getLockedHeader(); + if (vehicle_attribs_topic_ != nullptr) + { + people_msgs::VehicleAttribsStamped vehicle_attribs_msg; + vehicle_attribs_msg.header = header; + vehicle_attribs_msg.vehicles.swap(vehicle_attribs_topic_->vehicles); + pub_vehicle_attribs_.publish(vehicle_attribs_msg); + vehicle_attribs_topic_ = nullptr; + } + if (license_plate_topic_ != nullptr) + { + people_msgs::LicensePlateStamped license_plate_msg; + license_plate_msg.header = header; + license_plate_msg.licenses.swap(license_plate_topic_->licenses); + pub_license_plate_.publish(license_plate_msg); + license_plate_topic_ = nullptr; + } + if (person_attribs_topic_ != nullptr) + { + people_msgs::PersonAttributeStamped person_attribute_msg; + person_attribute_msg.header = header; + person_attribute_msg.attributes.swap(person_attribs_topic_->attributes); + pub_person_attribs_.publish(person_attribute_msg); + person_attribs_topic_ = nullptr; + } + if (person_reid_msg_ptr_ != nullptr) + { people_msgs::ReidentificationStamped person_reid_msg; person_reid_msg.header = header; person_reid_msg.reidentified_vector.swap(person_reid_msg_ptr_->reidentified_vector); pub_person_reid_.publish(person_reid_msg); person_reid_msg_ptr_ = nullptr; } - if (segmented_object_msg_ptr_ != nullptr) { + if (segmented_objects_topic_ != nullptr) + { // slog::info << "publishing faces outputs." << slog::endl; people_msgs::ObjectsInMasks segmented_msg; segmented_msg.header = header; - segmented_msg.objects_vector.swap(segmented_object_msg_ptr_->objects_vector); + segmented_msg.objects_vector.swap(segmented_objects_topic_->objects_vector); pub_segmented_object_.publish(segmented_msg); - segmented_object_msg_ptr_ = nullptr; + segmented_objects_topic_ = nullptr; } - if (faces_msg_ptr_ != NULL) + if (faces_topic_ != NULL) { object_msgs::ObjectsInBoxes faces_msg; faces_msg.header = header; - faces_msg.objects_vector.swap(faces_msg_ptr_->objects_vector); + faces_msg.objects_vector.swap(faces_topic_->objects_vector); pub_face_.publish(faces_msg); - faces_msg_ptr_ = nullptr; + faces_topic_ = nullptr; } - if (emotions_msg_ptr_ != nullptr) + if (emotions_topic_ != nullptr) { people_msgs::EmotionsStamped emotions_msg; emotions_msg.header = header; - emotions_msg.emotions.swap(emotions_msg_ptr_->emotions); + emotions_msg.emotions.swap(emotions_topic_->emotions); pub_emotion_.publish(emotions_msg); - emotions_msg_ptr_ = nullptr; + emotions_topic_ = nullptr; } - if (age_gender_msg_ptr_ != nullptr) + if (age_gender_topic_ != nullptr) { people_msgs::AgeGenderStamped age_gender_msg; age_gender_msg.header = header; - age_gender_msg.objects.swap(age_gender_msg_ptr_->objects); + age_gender_msg.objects.swap(age_gender_topic_->objects); pub_age_gender_.publish(age_gender_msg); - age_gender_msg_ptr_ = nullptr; + age_gender_topic_ = nullptr; } - if (headpose_msg_ptr_ != nullptr) + if (headpose_topic_ != nullptr) { people_msgs::HeadPoseStamped headpose_msg; headpose_msg.header = header; - headpose_msg.headposes.swap(headpose_msg_ptr_->headposes); + headpose_msg.headposes.swap(headpose_topic_->headposes); pub_headpose_.publish(headpose_msg); - headpose_msg_ptr_ = nullptr; + headpose_topic_ = nullptr; } - if (object_msg_ptr_ != nullptr) + if (detected_objects_topic_ != nullptr) { object_msgs::ObjectsInBoxes object_msg; object_msg.header = header; - object_msg.objects_vector.swap(object_msg_ptr_->objects_vector); + object_msg.objects_vector.swap(detected_objects_topic_->objects_vector); pub_object_.publish(object_msg); - object_msg_ptr_ = nullptr; + detected_objects_topic_ = nullptr; } - if (object_msg_ptr_ != nullptr) + if (detected_objects_topic_ != nullptr) { object_msgs::ObjectsInBoxes object_msg; object_msg.header = header; - object_msg.objects_vector.swap(object_msg_ptr_->objects_vector); + object_msg.objects_vector.swap(detected_objects_topic_->objects_vector); pub_object_.publish(object_msg); - object_msg_ptr_ = nullptr; + detected_objects_topic_ = nullptr; + } + if (face_reid_topic_ != nullptr) + { + people_msgs::ReidentificationStamped face_reid_msg; + face_reid_msg.header = header; + face_reid_msg.reidentified_vector.swap(face_reid_topic_->reidentified_vector); + + pub_face_reid_.publish(face_reid_msg); + face_reid_topic_ = nullptr; } } +#if 0 // deprecated +/** + * Don't use this inferface to create new time stamp, it'd better use camera/topic + * time stamp. + */ std_msgs::Header Outputs::RosTopicOutput::getHeader() { std_msgs::Header header; @@ -287,3 +431,4 @@ std_msgs::Header Outputs::RosTopicOutput::getHeader() header.stamp.nsec = ns % 1000000000; return header; } +#endif // depreated diff --git a/dynamic_vino_lib/src/outputs/rviz_output.cpp b/dynamic_vino_lib/src/outputs/rviz_output.cpp index 64f99b92..e587a989 100644 --- a/dynamic_vino_lib/src/outputs/rviz_output.cpp +++ b/dynamic_vino_lib/src/outputs/rviz_output.cpp @@ -25,40 +25,58 @@ #include "dynamic_vino_lib/pipeline.h" #include "dynamic_vino_lib/outputs/rviz_output.h" -Outputs::RvizOutput::RvizOutput() +Outputs::RvizOutput::RvizOutput(std::string pipeline_name) : BaseOutput(pipeline_name) { image_topic_ = nullptr; - pub_image_ = nh_.advertise("/openvino_toolkit/images", 16); - image_window_output_ = std::make_shared("WindowForRviz", 950); + pub_image_ = nh_.advertise("/openvino_toolkit/" + pipeline_name + "/images", 16); + image_window_output_ = std::make_shared(pipeline_name, 950); } -void Outputs::RvizOutput::feedFrame(const cv::Mat & frame) +void Outputs::RvizOutput::feedFrame(const cv::Mat& frame) { image_window_output_->feedFrame(frame); } -void Outputs::RvizOutput::accept(const std::vector & results) +void Outputs::RvizOutput::accept(const std::vector& results) +{ + image_window_output_->accept(results); +} +void Outputs::RvizOutput::accept(const std::vector& results) +{ + image_window_output_->accept(results); +} + +void Outputs::RvizOutput::accept(const std::vector& results) +{ + image_window_output_->accept(results); +} + +void Outputs::RvizOutput::accept(const std::vector& results) { image_window_output_->accept(results); } -void Outputs::RvizOutput::accept( - const std::vector & results) +void Outputs::RvizOutput::accept(const std::vector& results) { image_window_output_->accept(results); } -void Outputs::RvizOutput::accept(const std::vector & results) +void Outputs::RvizOutput::accept(const std::vector& results) { image_window_output_->accept(results); } -void Outputs::RvizOutput::accept(const std::vector & results) +void Outputs::RvizOutput::accept(const std::vector& results) { image_window_output_->accept(results); } -void Outputs::RvizOutput::accept(const std::vector & results) +void Outputs::RvizOutput::accept(const std::vector& results) +{ + image_window_output_->accept(results); +} + +void Outputs::RvizOutput::accept(const std::vector& results) { image_window_output_->accept(results); } @@ -67,34 +85,19 @@ void Outputs::RvizOutput::accept(const std::vectoraccept(results); } -void Outputs::RvizOutput::accept(const std::vector & results) +void Outputs::RvizOutput::accept(const std::vector& results) { image_window_output_->accept(results); } - void Outputs::RvizOutput::handleOutput() { image_window_output_->setPipeline(getPipeline()); image_window_output_->decorateFrame(); cv::Mat frame = image_window_output_->getFrame(); - std_msgs::Header header = getHeader(); - std::shared_ptr cv_ptr = - std::make_shared(header, "bgr8", frame); - sensor_msgs::Image image_msg; + std_msgs::Header header = getPipeline()->getInputDevice()->getLockedHeader(); + std::shared_ptr cv_ptr = std::make_shared(header, "bgr8", frame); + sensor_msgs::Image image_msg; image_topic_ = cv_ptr->toImageMsg(); -// image_topic_ = std::make_shared(image_msg); pub_image_.publish(image_topic_); -} - std::shared_ptr image_topic_; -std_msgs::Header Outputs::RvizOutput::getHeader() -{ - std_msgs::Header header; - header.frame_id = getPipeline()->getInputDevice()->getFrameID(); - - std::chrono::high_resolution_clock::time_point tp = std::chrono::high_resolution_clock::now(); - int64 ns = tp.time_since_epoch().count(); - header.stamp.sec = ns / 1000000000; - header.stamp.nsec = ns % 1000000000; - return header; } diff --git a/dynamic_vino_lib/src/pipeline.cpp b/dynamic_vino_lib/src/pipeline.cpp index 4da9fea4..8f4a1a1f 100644 --- a/dynamic_vino_lib/src/pipeline.cpp +++ b/dynamic_vino_lib/src/pipeline.cpp @@ -22,57 +22,57 @@ #include #include #include +#include #include #include "dynamic_vino_lib/pipeline.h" Pipeline::Pipeline(const std::string& name) { - params_ = std::make_shared(name); + if (!name.empty()) + { + params_ = std::make_shared(name); + } counter_ = 0; } -bool Pipeline::add(const std::string& name, - std::shared_ptr input_device) +bool Pipeline::add(const std::string& name, std::shared_ptr input_device) { + if (name.empty()) + { + slog::err << "Item name can't be empty!" << slog::endl; + return false; + } + slog::info << "Adding Input Device into Pipeline: " << name << slog::endl; input_device_name_ = name; input_device_ = input_device; - next_.insert({"", name}); + + addConnect("", name); return true; } -bool Pipeline::add(const std::string& parent, const std::string& name, - std::shared_ptr output) +bool Pipeline::add(const std::string& parent, const std::string& name, std::shared_ptr output) { - if (parent.empty()) + if (parent.empty() || name.empty() || !isLegalConnect(parent, name) || output == nullptr) { - slog::err << "output device have no parent!" << slog::endl; + slog::err << "ARGuments ERROR when adding output instance!" << slog::endl; return false; } - if (name_to_detection_map_.find(parent) == name_to_detection_map_.end()) + if (add(name, output)) { - slog::err << "parent detection does not exists!" << slog::endl; - return false; - } + addConnect(parent, name); - if (output_names_.find(name) != output_names_.end()) - { - return add(parent, name); + return true; } - output_names_.insert(name); - name_to_output_map_[name] = output; - next_.insert({parent, name}); - - /**< Add pipeline instance to Output instance >**/ - output->setPipeline(this); - return true; + return false; } bool Pipeline::add(const std::string& parent, const std::string& name) { - if (isLegalConnect(parent, name)) { + if (isLegalConnect(parent, name)) + { addConnect(parent, name); return true; } @@ -80,53 +80,78 @@ bool Pipeline::add(const std::string& parent, const std::string& name) return false; } -bool Pipeline::add(const std::string& name, - std::shared_ptr output) { - if (name.empty()) { +bool Pipeline::add(const std::string& name, std::shared_ptr output) +{ + if (name.empty()) + { slog::err << "Item name can't be empty!" << slog::endl; return false; } - + std::map>::iterator it = name_to_output_map_.find(name); - if (it != name_to_output_map_.end()) { - slog::warn << "inferance instance for [" << name << - "] already exists, update it with new instance." << slog::endl; + if (it != name_to_output_map_.end()) + { + slog::warn << "inferance instance for [" << name << "] already exists, update it with new instance." << slog::endl; } name_to_output_map_[name] = output; output_names_.insert(name); /**< Add pipeline instance to Output instance >**/ output->setPipeline(this); - + return true; } +void Pipeline::addConnect(const std::string& parent, const std::string& name) +{ + std::pair::iterator, std::multimap::iterator> ret; + ret = next_.equal_range(parent); + + for (std::multimap::iterator it = ret.first; it != ret.second; ++it) + { + if (it->second == name) + { + slog::warn << "The connect [" << parent << "<-->" << name << "] already exists." << slog::endl; + return; + } + } + slog::info << "Adding connection into pipeline:[" << parent << "<-->" << name << "]" << slog::endl; + next_.insert({ parent, name }); +} + bool Pipeline::add(const std::string& parent, const std::string& name, std::shared_ptr inference) { - if (name_to_detection_map_.find(parent) == name_to_detection_map_.end() && - input_device_name_ != parent) + if (parent.empty() || name.empty() || !isLegalConnect(parent, name)) { - slog::err << "parent device/detection does not exists!" << slog::endl; + slog::err << "ARGuments ERROR when adding inference instance!" << slog::endl; return false; } - next_.insert({parent, name}); - name_to_detection_map_[name] = inference; - ++total_inference_; - return true; + + if (add(name, inference)) + { + addConnect(parent, name); + return true; + } + + return false; } -bool Pipeline::add(const std::string& name, - std::shared_ptr inference) { - if (name.empty()) { +bool Pipeline::add(const std::string& name, std::shared_ptr inference) +{ + if (name.empty()) + { slog::err << "Item name can't be empty!" << slog::endl; return false; } - - std::map>::iterator it = name_to_detection_map_.find(name); - if (it != name_to_detection_map_.end()) { - slog::warn << "inferance instance for [" << name << - "] already exists, update it with new instance." << slog::endl; - } else { + + std::map>::iterator it = + name_to_detection_map_.find(name); + if (it != name_to_detection_map_.end()) + { + slog::warn << "inferance instance for [" << name << "] already exists, update it with new instance." << slog::endl; + } + else + { ++total_inference_; } name_to_detection_map_[name] = inference; @@ -134,52 +159,36 @@ bool Pipeline::add(const std::string& name, return true; } -int Pipeline::getCatagoryOrder(const std::string name) -{ - int order = kCatagoryOrder_Unknown; - if (name == input_device_name_) { - order = kCatagoryOrder_Input; - } else if (name_to_detection_map_.find(name) != name_to_detection_map_.end()) { - order = kCatagoryOrder_Inference; - } else if (name_to_output_map_.find(name) != name_to_output_map_.end()) { - order = kCatagoryOrder_Output; - } - - return order; -} - - bool Pipeline::isLegalConnect(const std::string parent, const std::string child) { int parent_order = getCatagoryOrder(parent); int child_order = getCatagoryOrder(child); - slog::info << "Checking connection into pipeline:[" << parent << "(" << parent_order << ")" << - "<-->" << child << "(" << child_order << ")" << - "]" << slog::endl; + slog::info << "Checking connection into pipeline:[" << parent << "(" << parent_order << ")" + << "<-->" << child << "(" << child_order << ")" + << "]" << slog::endl; return (parent_order != kCatagoryOrder_Unknown) && (child_order != kCatagoryOrder_Unknown) && (parent_order <= child_order); } -void Pipeline::addConnect(const std::string & parent, const std::string & name) +int Pipeline::getCatagoryOrder(const std::string name) { - std::pair::iterator, - std::multimap::iterator> - ret; - ret = next_.equal_range(parent); - - for (std::multimap::iterator it = ret.first; it != ret.second; ++it) { - if (it->second == name) { - slog::warn << "The connect [" << parent << "<-->" << name << "] already exists." << - slog::endl; - return; - } + int order = kCatagoryOrder_Unknown; + if (name == input_device_name_) + { + order = kCatagoryOrder_Input; + } + else if (name_to_detection_map_.find(name) != name_to_detection_map_.end()) + { + order = kCatagoryOrder_Inference; + } + else if (name_to_output_map_.find(name) != name_to_output_map_.end()) + { + order = kCatagoryOrder_Output; } - slog::info << "Adding connection into pipeline:[" << parent << "<-->" << name << "]" << - slog::endl; - next_.insert({parent, name}); -} + return order; +} void Pipeline::runOnce() { @@ -188,35 +197,40 @@ void Pipeline::runOnce() if (!input_device_->read(&frame_)) { // throw std::logic_error("Failed to get frame from cv::VideoCapture"); - slog::warn << "Failed to get frame from input_device." << slog::endl; - return; + // slog::warn << "Failed to get frame from input_device." << slog::endl; + return; // do nothing if now frame read out } - countFPS(); width_ = frame_.cols; height_ = frame_.rows; - for (auto& pair : name_to_output_map_) - { - pair.second->feedFrame(frame_); - } + slog::debug << "DEBUG: in Pipeline run process..." << slog::endl; // auto t0 = std::chrono::high_resolution_clock::now(); - for (auto pos = next_.equal_range(input_device_name_); - pos.first != pos.second; ++pos.first) + for (auto pos = next_.equal_range(input_device_name_); pos.first != pos.second; ++pos.first) { std::string detection_name = pos.first->second; auto detection_ptr = name_to_detection_map_[detection_name]; - detection_ptr->enqueue(frame_, - cv::Rect(width_ / 2, height_ / 2, width_, height_)); + detection_ptr->enqueue(frame_, cv::Rect(width_ / 2, height_ / 2, width_, height_)); increaseInferenceCounter(); detection_ptr->submitRequest(); } + + for (auto& pair : name_to_output_map_) + { + pair.second->feedFrame(frame_); + } + countFPS(); + + slog::debug << "DEBUG: align inference process, waiting until all inferences done!" << slog::endl; std::unique_lock lock(counter_mutex_); - cv_.wait(lock, [this]() { return this->counter_ == 0; }); + cv_.wait(lock, [self = this]() { return self->counter_ == 0; }); + // auto t1 = std::chrono::high_resolution_clock::now(); // typedef std::chrono::duration> ms; + slog::debug << "DEBUG: in Pipeline run process...handleOutput" << slog::endl; for (auto& pair : name_to_output_map_) { + // slog::info << "Handling Output ..." << pair.first << slog::endl; pair.second->handleOutput(); } } @@ -225,8 +239,7 @@ void Pipeline::printPipeline() { for (auto& current_node : next_) { - printf("%s --> %s\n", current_node.first.c_str(), - current_node.second.c_str()); + printf("%s --> %s\n", current_node.first.c_str(), current_node.second.c_str()); } } @@ -235,28 +248,30 @@ void Pipeline::setCallback() for (auto& pair : name_to_detection_map_) { std::string detection_name = pair.first; + std::function callb; - callb = [detection_name, this]() - { + callb = [detection_name, this]() { this->callback(detection_name); return; }; pair.second->getEngine()->getRequest()->SetCompletionCallback(callb); } } + void Pipeline::callback(const std::string& detection_name) { + // slog::info<<"Hello callback ----> " << detection_name <fetchResults(); // set output - for (auto pos = next_.equal_range(detection_name); pos.first != pos.second; - ++pos.first) + for (auto pos = next_.equal_range(detection_name); pos.first != pos.second; ++pos.first) { std::string next_name = pos.first->second; + + std::string filter_conditions = findFilterConditions(detection_name, next_name); // if next is output, then print if (output_names_.find(next_name) != output_names_.end()) { - // name_to_output_map_[next_name]->accept(*detection_ptr->getResult()); detection_ptr->observeOutput(name_to_output_map_[next_name]); } else @@ -265,19 +280,21 @@ void Pipeline::callback(const std::string& detection_name) if (detection_ptr_iter != name_to_detection_map_.end()) { auto next_detection_ptr = detection_ptr_iter->second; - for (int i = 0; i < detection_ptr->getResultsLength(); ++i) + size_t batch_size = next_detection_ptr->getMaxBatchSize(); + std::vector next_rois = detection_ptr->getFilteredROIs(filter_conditions); + for (size_t i = 0; i < next_rois.size(); i++) { - const dynamic_vino_lib::Result* prev_result = - detection_ptr->getLocationResult(i); - auto clippedRect = - prev_result->getLocation() & cv::Rect(0, 0, width_, height_); + auto roi = next_rois[i]; + auto clippedRect = roi & cv::Rect(0, 0, width_, height_); cv::Mat next_input = frame_(clippedRect); - next_detection_ptr->enqueue(next_input, prev_result->getLocation()); - } - if (detection_ptr->getResultsLength() > 0) - { - increaseInferenceCounter(); - next_detection_ptr->submitRequest(); + next_detection_ptr->enqueue(next_input, roi); + if ((i + 1) == next_rois.size() || (i + 1) % batch_size == 0) + { + increaseInferenceCounter(); + next_detection_ptr->submitRequest(); + auto request = next_detection_ptr->getEngine()->getRequest(); + request->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY); + } } } } @@ -293,11 +310,13 @@ void Pipeline::initInferenceCounter() counter_ = 0; cv_.notify_all(); } + void Pipeline::increaseInferenceCounter() { std::lock_guard lk(counter_mutex_); ++counter_; } + void Pipeline::decreaseInferenceCounter() { std::lock_guard lk(counter_mutex_); @@ -306,22 +325,14 @@ void Pipeline::decreaseInferenceCounter() void Pipeline::countFPS() { - static int fps = 0; - - static auto t_start = std::chrono::high_resolution_clock::now(); - static int frame_cnt = 0; - - frame_cnt++; - + frame_cnt_++; auto t_end = std::chrono::high_resolution_clock::now(); typedef std::chrono::duration> ms; - ms secondDetection = std::chrono::duration_cast(t_end - t_start); - - if (secondDetection.count() > 1000) { - fps = frame_cnt; - setFPS(fps); - frame_cnt = 0; - t_start = t_end; + ms secondDetection = std::chrono::duration_cast(t_end - t_start_); + if (secondDetection.count() > 1000) + { + setFPS(frame_cnt_); + frame_cnt_ = 0; + t_start_ = t_end; } - -} +} diff --git a/dynamic_vino_lib/src/pipeline_manager.cpp b/dynamic_vino_lib/src/pipeline_manager.cpp index 0e066d8f..3cac858b 100644 --- a/dynamic_vino_lib/src/pipeline_manager.cpp +++ b/dynamic_vino_lib/src/pipeline_manager.cpp @@ -22,13 +22,16 @@ #include #include #include - #include -#include "dynamic_vino_lib/factory.h" #include "dynamic_vino_lib/inferences/age_gender_detection.h" #include "dynamic_vino_lib/inferences/emotions_detection.h" #include "dynamic_vino_lib/inferences/face_detection.h" #include "dynamic_vino_lib/inferences/head_pose_detection.h" +#include "dynamic_vino_lib/inferences/face_reidentification.h" +#include "dynamic_vino_lib/inferences/person_attribs_detection.h" +#include "dynamic_vino_lib/inferences/vehicle_attribs_detection.h" +#include "dynamic_vino_lib/inferences/license_plate_detection.h" +#include "dynamic_vino_lib/inferences/landmarks_detection.h" #include "dynamic_vino_lib/inputs/image_input.h" #include "dynamic_vino_lib/inputs/realsense_camera.h" #include "dynamic_vino_lib/inputs/realsense_camera_topic.h" @@ -38,6 +41,13 @@ #include "dynamic_vino_lib/models/emotion_detection_model.h" #include "dynamic_vino_lib/models/face_detection_model.h" #include "dynamic_vino_lib/models/head_pose_detection_model.h" +#include "dynamic_vino_lib/models/object_detection_ssd_model.h" +#include "dynamic_vino_lib/models/object_detection_yolov2voc_model.h" +#include "dynamic_vino_lib/models/face_reidentification_model.h" +#include "dynamic_vino_lib/models/person_attribs_detection_model.h" +#include "dynamic_vino_lib/models/vehicle_attribs_detection_model.h" +#include "dynamic_vino_lib/models/license_plate_detection_model.h" +#include "dynamic_vino_lib/models/landmarks_detection_model.h" #include "dynamic_vino_lib/outputs/image_window_output.h" #include "dynamic_vino_lib/outputs/ros_topic_output.h" #include "dynamic_vino_lib/outputs/rviz_output.h" @@ -45,85 +55,116 @@ #include "dynamic_vino_lib/pipeline.h" #include "dynamic_vino_lib/pipeline_manager.h" #include "dynamic_vino_lib/pipeline_params.h" +#include "dynamic_vino_lib/services/pipeline_processing_server.h" +#include "dynamic_vino_lib/engines/engine_manager.h" -std::shared_ptr PipelineManager::createPipeline( - const Params::ParamManager::PipelineParams& params) { - if (params.name == "") { +std::shared_ptr PipelineManager::createPipeline(const Params::ParamManager::PipelineRawData& params) +{ + if (params.name == "") + { throw std::logic_error("The name of pipeline won't be empty!"); } - PipelineData data; std::shared_ptr pipeline = std::make_shared(params.name); pipeline->getParameters()->update(params); - auto inputs = parseInputDevice(params); - if (inputs.size() != 1) { - slog::err << "currently one pipeline only supports ONE input." - << slog::endl; + PipelineData data; + data.pipeline = pipeline; + data.params = params; + data.state = PipelineState_ThreadNotCreated; + + auto inputs = parseInputDevice(data); + if (inputs.size() != 1) + { + slog::err << "currently one pipeline only supports ONE input." << slog::endl; return nullptr; } - for (auto it = inputs.begin(); it != inputs.end(); ++it) { + for (auto it = inputs.begin(); it != inputs.end(); ++it) + { pipeline->add(it->first, it->second); auto node = it->second->getHandler(); - if(node != nullptr) { - data.spin_nodes.emplace_back(node); + if (node != nullptr) + { + data.spin_nodes.emplace_back(node); } } - auto outputs = parseOutput(params); - for (auto it = outputs.begin(); it != outputs.end(); ++it) { + auto outputs = parseOutput(data); + for (auto it = outputs.begin(); it != outputs.end(); ++it) + { pipeline->add(it->first, it->second); } auto infers = parseInference(params); - for (auto it = infers.begin(); it != infers.end(); ++it) { + for (auto it = infers.begin(); it != infers.end(); ++it) + { pipeline->add(it->first, it->second); } slog::info << "Updating connections ..." << slog::endl; - for (auto it = params.connects.begin(); it != params.connects.end(); ++it) { + for (auto it = params.connects.begin(); it != params.connects.end(); ++it) + { pipeline->add(it->first, it->second); } - data.pipeline = pipeline; - data.params = params; - data.state = PipelineState_ThreadNotCreated; - pipelines_.insert({params.name, data}); + pipelines_.insert({ params.name, data }); pipeline->setCallback(); slog::info << "One Pipeline Created!" << slog::endl; pipeline->printPipeline(); return pipeline; - } std::map> -PipelineManager::parseInputDevice( - const Params::ParamManager::PipelineParams& params) { +PipelineManager::parseInputDevice(const PipelineData& pdata) +{ std::map> inputs; - for (auto& name : params.inputs) { + for (auto& name : pdata.params.inputs) + { slog::info << "Parsing InputDvice: " << name << slog::endl; std::shared_ptr device = nullptr; - if (name == kInputType_RealSenseCamera) { + if (name == kInputType_RealSenseCamera) + { device = std::make_shared(); - } else if (name == kInputType_StandardCamera) { + } + else if (name == kInputType_StandardCamera) + { device = std::make_shared(); - } else if (name == kInputType_CameraTopic) { - device = std::make_shared(); - std::cout <<"register yaml"<(params.input_meta); + } + // else if (name == kInputType_IpCamera) + // { + // if (pdata.params.input_meta != "") + // { + // device = std::make_shared(pdata.params.input_meta); + // } + // } + // else if (name == kInputType_CameraTopic || name == kInputType_ImageTopic) + // { + // device = std::make_shared(pdata.parent_node); + // } + else if (name == kInputType_Video) + { + if (pdata.params.input_meta != "") + { + device = std::make_shared(pdata.params.input_meta); } - } else if (name == kInputType_Image) { - if (params.input_meta != "") { - device = std::make_shared(params.input_meta); + } + else if (name == kInputType_Image) + { + if (pdata.params.input_meta != "") + { + device = std::make_shared(pdata.params.input_meta); } } + else + { + slog::err << "Invalid input device name: " << name << slog::endl; + } - if (device != nullptr) { + if (device != nullptr) + { device->initialize(); - inputs.insert({name, device}); + inputs.insert({ name, device }); slog::info << " ... Adding one Input device: " << name << slog::endl; } } @@ -131,24 +172,36 @@ PipelineManager::parseInputDevice( return inputs; } -std::map> -PipelineManager::parseOutput( - const Params::ParamManager::PipelineParams& params) { +std::map> PipelineManager::parseOutput(const PipelineData& pdata) +{ std::map> outputs; - for (auto& name : params.outputs) { + for (auto& name : pdata.params.outputs) + { slog::info << "Parsing Output: " << name << slog::endl; std::shared_ptr object = nullptr; - if (name == kOutputTpye_RosTopic) { - object = std::make_shared(); - } else if (name == kOutputTpye_ImageWindow) { - object = std::make_shared("Results"); - } else if (name == kOutputTpye_RViz) { - object = std::make_shared(); - } else if (name == kOutputTpye_RosService) { - object = std::make_shared(); - } - if (object != nullptr) { - outputs.insert({name, object}); + if (name == kOutputTpye_RosTopic) + { + // object = std::make_shared(pdata.params.name, pdata.parent_node); + } + else if (name == kOutputTpye_ImageWindow) + { + object = std::make_shared(pdata.params.name); + } + else if (name == kOutputTpye_RViz) + { + // object = std::make_shared(pdata.params.name, pdata.parent_node); + } + else if (name == kOutputTpye_RosService) + { + object = std::make_shared(pdata.params.name); + } + else + { + slog::err << "Invalid output name: " << name << slog::endl; + } + if (object != nullptr) + { + outputs.insert({ name, object }); slog::info << " ... Adding one Output: " << name << slog::endl; } } @@ -157,56 +210,74 @@ PipelineManager::parseOutput( } std::map> -PipelineManager::parseInference( - const Params::ParamManager::PipelineParams& params) { - /**< update plugins for devices >**/ - auto pcommon = Params::ParamManager::getInstance().getCommon(); - std::string FLAGS_l = pcommon.custom_cpu_library; - std::string FLAGS_c = pcommon.custom_cldnn_library; - bool FLAGS_pc = pcommon.enable_performance_count; - - std::map> - inferences; - for (auto& infer : params.infers) { - if (infer.name.empty() || infer.model.empty()) { +PipelineManager::parseInference(const Params::ParamManager::PipelineRawData& params) +{ + std::map> inferences; + for (auto& infer : params.infers) + { + if (infer.name.empty() || infer.model.empty()) + { continue; } slog::info << "Parsing Inference: " << infer.name << slog::endl; std::shared_ptr object = nullptr; - if (plugins_for_devices_.find(infer.engine) == plugins_for_devices_.end()) { - plugins_for_devices_[infer.engine] = - *Factory::makePluginByName(infer.engine, FLAGS_l, FLAGS_c, FLAGS_pc); - } - if (infer.name == kInferTpye_FaceDetection) { + if (infer.name == kInferTpye_FaceDetection) + { object = createFaceDetection(infer); - - } else if (infer.name == kInferTpye_AgeGenderRecognition) { + } + else if (infer.name == kInferTpye_AgeGenderRecognition) + { object = createAgeGenderRecognition(infer); - - } else if (infer.name == kInferTpye_EmotionRecognition) { + } + else if (infer.name == kInferTpye_EmotionRecognition) + { object = createEmotionRecognition(infer); - - } else if (infer.name == kInferTpye_HeadPoseEstimation) { + } + else if (infer.name == kInferTpye_HeadPoseEstimation) + { object = createHeadPoseEstimation(infer); - - } else if (infer.name == kInferTpye_ObjectDetection) { + } + else if (infer.name == kInferTpye_ObjectDetection) + { object = createObjectDetection(infer); - } - else if (infer.name == kInferTpye_ObjectSegmentation) { + else if (infer.name == kInferTpye_ObjectSegmentation) + { object = createObjectSegmentation(infer); } - else if (infer.name == kInferTpye_PersonReidentification) { + else if (infer.name == kInferTpye_PersonReidentification) + { object = createPersonReidentification(infer); - } - else { + } + else if (infer.name == kInferTpye_FaceReidentification) + { + object = createFaceReidentification(infer); + } + else if (infer.name == kInferTpye_PersonAttribsDetection) + { + object = createPersonAttribsDetection(infer); + } + else if (infer.name == kInferTpye_LandmarksDetection) + { + object = createLandmarksDetection(infer); + } + else if (infer.name == kInferTpye_VehicleAttribsDetection) + { + object = createVehicleAttribsDetection(infer); + } + else if (infer.name == kInferTpye_LicensePlateDetection) + { + object = createLicensePlateDetection(infer); + } + else + { slog::err << "Invalid inference name: " << infer.name << slog::endl; } - - if (object != nullptr) { - inferences.insert({infer.name, object}); + if (object != nullptr) + { + inferences.insert({ infer.name, object }); slog::info << " ... Adding one Inference: " << infer.name << slog::endl; } } @@ -215,30 +286,17 @@ PipelineManager::parseInference( } std::shared_ptr -PipelineManager::createFaceDetection( - const Params::ParamManager::InferenceParams& infer) { - // TODO: add batch size in param_manager - auto face_detection_model = - std::make_shared(infer.model, 1, 1, 1); - face_detection_model->modelInit(); - auto face_detection_engine = std::make_shared( - plugins_for_devices_[infer.engine], face_detection_model); - auto face_inference_ptr = std::make_shared( - 0.5); // TODO: add output_threshold in param_manager - face_inference_ptr->loadNetwork(face_detection_model); - face_inference_ptr->loadEngine(face_detection_engine); - - return face_inference_ptr; +PipelineManager::createFaceDetection(const Params::ParamManager::InferenceRawData& infer) +{ + return createObjectDetection(infer); } std::shared_ptr -PipelineManager::createAgeGenderRecognition( - const Params::ParamManager::InferenceParams& param) { - auto model = - std::make_shared(param.model, 1, 2, 16); +PipelineManager::createAgeGenderRecognition(const Params::ParamManager::InferenceRawData& param) +{ + auto model = std::make_shared(param.model, param.batch); model->modelInit(); - auto engine = std::make_shared( - plugins_for_devices_[param.engine], model); + auto engine = engine_manager_.createEngine(param.engine, model); auto infer = std::make_shared(); infer->loadNetwork(model); infer->loadEngine(engine); @@ -247,13 +305,11 @@ PipelineManager::createAgeGenderRecognition( } std::shared_ptr -PipelineManager::createEmotionRecognition( - const Params::ParamManager::InferenceParams& param) { - auto model = - std::make_shared(param.model, 1, 1, 16); +PipelineManager::createEmotionRecognition(const Params::ParamManager::InferenceRawData& param) +{ + auto model = std::make_shared(param.model, param.batch); model->modelInit(); - auto engine = std::make_shared( - plugins_for_devices_[param.engine], model); + auto engine = engine_manager_.createEngine(param.engine, model); auto infer = std::make_shared(); infer->loadNetwork(model); infer->loadEngine(engine); @@ -262,13 +318,11 @@ PipelineManager::createEmotionRecognition( } std::shared_ptr -PipelineManager::createHeadPoseEstimation( - const Params::ParamManager::InferenceParams& param) { - auto model = - std::make_shared(param.model, 1, 3, 16); +PipelineManager::createHeadPoseEstimation(const Params::ParamManager::InferenceRawData& param) +{ + auto model = std::make_shared(param.model, param.batch); model->modelInit(); - auto engine = std::make_shared( - plugins_for_devices_[param.engine], model); + auto engine = engine_manager_.createEngine(param.engine, model); auto infer = std::make_shared(); infer->loadNetwork(model); infer->loadEngine(engine); @@ -277,16 +331,26 @@ PipelineManager::createHeadPoseEstimation( } std::shared_ptr -PipelineManager::createObjectDetection( -const Params::ParamManager::InferenceParams & infer) +PipelineManager::createObjectDetection(const Params::ParamManager::InferenceRawData& infer) { - auto object_detection_model = - std::make_shared(infer.model, 1, 1, 1); + std::shared_ptr object_detection_model; + std::shared_ptr object_inference_ptr; + slog::debug << "for test in createObjectDetection()" << slog::endl; + if (infer.model_type == kInferTpye_ObjectDetectionTypeSSD) + { + object_detection_model = std::make_shared(infer.model, infer.batch); + } + if (infer.model_type == kInferTpye_ObjectDetectionTypeYolov2voc) + { + object_detection_model = std::make_shared(infer.model, infer.batch); + } + + slog::debug << "for test in createObjectDetection(), Created SSDModel" << slog::endl; + object_inference_ptr = std::make_shared( + infer.enable_roi_constraint, infer.confidence_threshold); // To-do theshold configuration + slog::debug << "for test in createObjectDetection(), before modelInit()" << slog::endl; object_detection_model->modelInit(); - auto object_detection_engine = std::make_shared( - plugins_for_devices_[infer.engine], object_detection_model); - auto object_inference_ptr = std::make_shared( - infer.enable_roi_constraint, infer.confidence_threshold); // To-do theshold configuration + auto object_detection_engine = engine_manager_.createEngine(infer.engine, object_detection_model); object_inference_ptr->loadNetwork(object_detection_model); object_inference_ptr->loadEngine(object_detection_engine); @@ -294,70 +358,136 @@ const Params::ParamManager::InferenceParams & infer) } std::shared_ptr -PipelineManager::createObjectSegmentation(const Params::ParamManager::InferenceParams & infer) +PipelineManager::createObjectSegmentation(const Params::ParamManager::InferenceRawData& infer) { - auto obejct_segmentation_model = - std::make_shared(infer.model, 1, 2, 1); - obejct_segmentation_model->modelInit(); - auto obejct_segmentation_engine = std::make_shared( - plugins_for_devices_[infer.engine], obejct_segmentation_model); - auto segmentation_inference_ptr = std::make_shared(0.5); - segmentation_inference_ptr->loadNetwork(obejct_segmentation_model); - segmentation_inference_ptr->loadEngine(obejct_segmentation_engine); + auto model = std::make_shared(infer.model, infer.batch); + model->modelInit(); + slog::info << "Segmentation model initialized." << slog::endl; + auto engine = engine_manager_.createEngine(infer.engine, model); + slog::info << "Segmentation Engine initialized." << slog::endl; + auto segmentation_inference_ptr = std::make_shared(infer.confidence_threshold); + slog::info << "Segmentation Inference instanced." << slog::endl; + segmentation_inference_ptr->loadNetwork(model); + segmentation_inference_ptr->loadEngine(engine); return segmentation_inference_ptr; } std::shared_ptr -PipelineManager::createPersonReidentification( - const Params::ParamManager::InferenceParams & infer) +PipelineManager::createPersonReidentification(const Params::ParamManager::InferenceRawData& infer) { - auto person_reidentification_model = - std::make_shared(infer.model, 1, 1, infer.batch); + std::shared_ptr person_reidentification_model; + std::shared_ptr reidentification_inference_ptr; + slog::debug << "for test in createPersonReidentification()" << slog::endl; + person_reidentification_model = std::make_shared(infer.model, infer.batch); person_reidentification_model->modelInit(); - auto person_reidentification_engine = std::make_shared( - plugins_for_devices_[infer.engine], person_reidentification_model); - auto reidentification_inference_ptr = - std::make_shared(infer.confidence_threshold); + slog::info << "Reidentification model initialized" << slog::endl; + auto person_reidentification_engine = engine_manager_.createEngine(infer.engine, person_reidentification_model); + reidentification_inference_ptr = + std::make_shared(infer.confidence_threshold); reidentification_inference_ptr->loadNetwork(person_reidentification_model); reidentification_inference_ptr->loadEngine(person_reidentification_engine); return reidentification_inference_ptr; } -void PipelineManager::threadPipeline(const char* name) { +std::shared_ptr +PipelineManager::createVehicleAttribsDetection(const Params::ParamManager::InferenceRawData& infer) +{ + auto model = std::make_shared(infer.model, infer.batch); + model->modelInit(); + auto engine = engine_manager_.createEngine(infer.engine, model); + auto vehicle_attribs_ptr = std::make_shared(); + vehicle_attribs_ptr->loadNetwork(model); + vehicle_attribs_ptr->loadEngine(engine); + + return vehicle_attribs_ptr; +} + +std::shared_ptr +PipelineManager::createLicensePlateDetection(const Params::ParamManager::InferenceRawData& infer) +{ + auto model = std::make_shared(infer.model, infer.batch); + model->modelInit(); + auto engine = engine_manager_.createEngine(infer.engine, model); + auto license_plate_ptr = std::make_shared(); + license_plate_ptr->loadNetwork(model); + license_plate_ptr->loadEngine(engine); + + return license_plate_ptr; +} + +std::shared_ptr +PipelineManager::createPersonAttribsDetection(const Params::ParamManager::InferenceRawData& infer) +{ + auto model = std::make_shared(infer.model, infer.batch); + slog::debug << "for test in createPersonAttributesDetection()" << slog::endl; + model->modelInit(); + auto engine = engine_manager_.createEngine(infer.engine, model); + auto attribs_inference_ptr = std::make_shared(infer.confidence_threshold); + attribs_inference_ptr->loadNetwork(model); + attribs_inference_ptr->loadEngine(engine); + + return attribs_inference_ptr; +} + +void PipelineManager::threadPipeline(const char* name) +{ PipelineData& p = pipelines_[name]; - while (p.state == PipelineState_ThreadRunning && p.pipeline != nullptr && ros::ok()) { - for (auto& node : p.spin_nodes) { + while (p.state != PipelineState_ThreadStopped && p.pipeline != nullptr && ros::ok()) + { + if (p.state != PipelineState_ThreadPasued) + { ros::spinOnce(); + p.pipeline->runOnce(); } - p.pipeline->runOnce(); std::this_thread::sleep_for(std::chrono::milliseconds(1)); } } -void PipelineManager::runAll() { - for (auto it = pipelines_.begin(); it != pipelines_.end(); ++it) { - if(it->second.state != PipelineState_ThreadRunning) { + +void PipelineManager::runAll() +{ + for (auto it = pipelines_.begin(); it != pipelines_.end(); ++it) + { + if (it->second.state != PipelineState_ThreadRunning) + { it->second.state = PipelineState_ThreadRunning; } - if(it->second.thread == nullptr) { - it->second.thread = std::make_shared(&PipelineManager::threadPipeline, this, it->second.params.name.c_str()); + if (it->second.thread == nullptr) + { + it->second.thread = + std::make_shared(&PipelineManager::threadPipeline, this, it->second.params.name.c_str()); } } } void PipelineManager::stopAll() { - for (auto it = pipelines_.begin(); it != pipelines_.end(); ++it) { - if (it->second.state == PipelineState_ThreadRunning) { + for (auto it = pipelines_.begin(); it != pipelines_.end(); ++it) + { + if (it->second.state == PipelineState_ThreadRunning) + { it->second.state = PipelineState_ThreadStopped; } } } -void PipelineManager::joinAll() { - for (auto it = pipelines_.begin(); it != pipelines_.end(); ++it) { - if(it->second.thread != nullptr && it->second.state == PipelineState_ThreadRunning) { +void PipelineManager::runService() +{ + auto node = + std::make_shared>("pipeline_service"); + ros::spin(); // hold the thread waiting for pipeline service +} + +void PipelineManager::joinAll() +{ + auto service_thread = std::make_shared(&PipelineManager::runService, this); + service_thread->join(); // pipeline service + + for (auto it = pipelines_.begin(); it != pipelines_.end(); ++it) + { + if (it->second.thread != nullptr && it->second.state == PipelineState_ThreadRunning) + { it->second.thread->join(); } } diff --git a/dynamic_vino_lib/src/pipeline_params.cpp b/dynamic_vino_lib/src/pipeline_params.cpp index 947688a5..18cb1835 100644 --- a/dynamic_vino_lib/src/pipeline_params.cpp +++ b/dynamic_vino_lib/src/pipeline_params.cpp @@ -26,16 +26,17 @@ #include #include "dynamic_vino_lib/pipeline_params.h" -PipelineParams::PipelineParams(const std::string& name) { params_.name = name; } +PipelineParams::PipelineParams(const std::string& name) +{ + params_.name = name; +} -PipelineParams::PipelineParams( - const Params::ParamManager::PipelineParams& params) +PipelineParams::PipelineParams(const Params::ParamManager::PipelineRawData& params) { params_ = params; } -PipelineParams& PipelineParams::operator=( - const Params::ParamManager::PipelineParams& params) +PipelineParams& PipelineParams::operator=(const Params::ParamManager::PipelineRawData& params) { params_.name = params.name; params_.infers = params.infers; @@ -46,8 +47,7 @@ PipelineParams& PipelineParams::operator=( return *this; } -Params::ParamManager::PipelineParams PipelineParams::getPipeline( - const std::string& name) +Params::ParamManager::PipelineRawData PipelineParams::getPipeline(const std::string& name) { return Params::ParamManager::getInstance().getPipeline(name); } @@ -60,15 +60,14 @@ void PipelineParams::update() } } -void PipelineParams::update( - const Params::ParamManager::PipelineParams& params) { +void PipelineParams::update(const Params::ParamManager::PipelineRawData& params) +{ params_ = params; } bool PipelineParams::isOutputTo(std::string& output) { - if (std::find(params_.outputs.begin(), params_.outputs.end(), output) != - params_.outputs.end()) + if (std::find(params_.outputs.begin(), params_.outputs.end(), output) != params_.outputs.end()) { return true; } @@ -79,6 +78,21 @@ bool PipelineParams::isOutputTo(std::string& output) bool PipelineParams::isGetFps() { /**< Only "Image" input can't computing FPS >**/ - return std::find(params_.inputs.begin(), params_.inputs.end(), - kInputType_Image) == params_.inputs.end(); + if (params_.inputs.size() == 0) + { + return false; + } + return std::find(params_.inputs.begin(), params_.inputs.end(), kInputType_Image) == params_.inputs.end(); +} + +std::string PipelineParams::findFilterConditions(const std::string& input, const std::string& output) +{ + for (auto filter : params_.filters) + { + if (!input.compare(filter.input) && !output.compare(filter.output)) + { + return filter.filter_conditions; + } + } + return ""; } diff --git a/dynamic_vino_lib/src/services/frame_processing_server.cpp b/dynamic_vino_lib/src/services/frame_processing_server.cpp index baaaae67..da7bf107 100644 --- a/dynamic_vino_lib/src/services/frame_processing_server.cpp +++ b/dynamic_vino_lib/src/services/frame_processing_server.cpp @@ -12,19 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "dynamic_vino_lib/services/frame_processing_server.h" -#include -#include -#include -#include #include #include +#include #include #include #include #include #include +#include "dynamic_vino_lib/services/frame_processing_server.h" + #include "dynamic_vino_lib/pipeline_manager.h" #include "dynamic_vino_lib/pipeline.h" #include "dynamic_vino_lib/inputs/base_input.h" @@ -33,57 +31,60 @@ namespace vino_service { -template -FrameProcessingServer::FrameProcessingServer( - const std::string & service_name, - const std::string & config_path): - service_name_(service_name), - config_path_(config_path) +template +FrameProcessingServer::FrameProcessingServer(const std::string& service_name, const std::string& config_path) + : service_name_(service_name), config_path_(config_path) { - nh_=std::make_shared(); - initService(config_path); + nh_ = std::make_shared(service_name_); + initService(); } -template -void FrameProcessingServer::initService( - const std::string & config_path) +template +void FrameProcessingServer::initService() { - Params::ParamManager::getInstance().parse(config_path); + std::cout << "!!!!" << config_path_ << std::endl; + Params::ParamManager::getInstance().parse(config_path_); Params::ParamManager::getInstance().print(); - + auto pcommon = Params::ParamManager::getInstance().getCommon(); auto pipelines = Params::ParamManager::getInstance().getPipelines(); - if (pipelines.size() != 1) { + if (pipelines.size() != 1) + { throw std::logic_error("1 and only 1 pipeline can be set to FrameProcessServer!"); } - for (auto & p : pipelines) { + for (auto& p : pipelines) + { PipelineManager::getInstance().createPipeline(p); } - - ros::ServiceServer srv = nh_->advertiseService >("/openvino_toolkit/service",std::bind(&FrameProcessingServer::cbService,this,std::placeholders::_1)); - service_ = std::make_shared(srv); -} - + ros::ServiceServer srv = nh_->advertiseService >( + "/openvino_toolkit/service", std::bind(&FrameProcessingServer::cbService, this, std::placeholders::_1)); + service_ = std::make_shared(srv); +} -template -bool FrameProcessingServer::cbService( - ros::ServiceEvent& event) +template +bool FrameProcessingServer::cbService(ros::ServiceEvent& event) { boost::shared_ptr res = boost::make_shared(); - std::map pipelines_ = - PipelineManager::getInstance().getPipelines(); - for (auto it = pipelines_.begin(); it != pipelines_.end(); ++it) { - PipelineManager::PipelineData & p = pipelines_[it->second.params.name.c_str()]; + std::map pipelines_ = PipelineManager::getInstance().getPipelines(); + for (auto it = pipelines_.begin(); it != pipelines_.end(); ++it) + { + PipelineManager::PipelineData& p = pipelines_[it->second.params.name.c_str()]; auto input = p.pipeline->getInputDevice(); + + Input::Config config; + config.path = event.getRequest().image_path; + input->config(config); p.pipeline->runOnce(); auto output_handle = p.pipeline->getOutputHandle(); - for (auto & pair : output_handle) { - if (!pair.first.compare(kOutputTpye_RosService)) { + for (auto& pair : output_handle) + { + if (!pair.first.compare(kOutputTpye_RosService)) + { pair.second->setServiceResponse(res); event.getResponse() = *res; pair.second->clearData(); @@ -95,7 +96,42 @@ bool FrameProcessingServer::cbService( return false; } +template <> +bool FrameProcessingServer::cbService(ros::ServiceEvent& event) +{ + boost::shared_ptr res = boost::make_shared(); + std::map pipelines_ = PipelineManager::getInstance().getPipelines(); + for (auto it = pipelines_.begin(); it != pipelines_.end(); ++it) + { + PipelineManager::PipelineData& p = pipelines_[it->second.params.name.c_str()]; + auto input = p.pipeline->getInputDevice(); + + Input::Config config; + + //TODO (Corsair-cxs) It's unfinished, I only do it for build success. + config.path = event.getRequest().image_paths.front(); + input->config(config); + + p.pipeline->runOnce(); + auto output_handle = p.pipeline->getOutputHandle(); + + for (auto& pair : output_handle) + { + if (!pair.first.compare(kOutputTpye_RosService)) + { + pair.second->setServiceResponse(res); + event.getResponse() = *res; + pair.second->clearData(); + return true; // TODO(weizhi) , return directly, suppose only 1 pipeline dealing with 1 request. + } + } + } + slog::info << "[FrameProcessingServer] Callback finished!" << slog::endl; + return false; +} +//TODO It is not advisable to use object_msgs like DetectObject or ClassifyObject as a param for FrameProcessingServer. template class FrameProcessingServer; + template class FrameProcessingServer; template class FrameProcessingServer; template class FrameProcessingServer; diff --git a/dynamic_vino_lib/src/services/pipeline_processing_server.cpp b/dynamic_vino_lib/src/services/pipeline_processing_server.cpp new file mode 100644 index 00000000..97d04322 --- /dev/null +++ b/dynamic_vino_lib/src/services/pipeline_processing_server.cpp @@ -0,0 +1,118 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dynamic_vino_lib/services/pipeline_processing_server.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dynamic_vino_lib/pipeline_manager.h" +#include "dynamic_vino_lib/pipeline.h" +#include "dynamic_vino_lib/inputs/base_input.h" +#include "dynamic_vino_lib/inputs/image_input.h" +#include "dynamic_vino_lib/slog.h" + +namespace vino_service +{ +template +PipelineProcessingServer::PipelineProcessingServer(const std::string& service_name) : service_name_(service_name) +{ + nh_ = std::make_shared(); + pipelines_ = PipelineManager::getInstance().getPipelinesPtr(); + initPipelineService(); +} + +template +void PipelineProcessingServer::initPipelineService() +{ + ros::ServiceServer srv = nh_->advertiseService< + ros::ServiceEvent >( + "/openvino_toolkit/pipeline/service", + std::bind(&PipelineProcessingServer::cbService, this, std::placeholders::_1)); + service_ = std::make_shared(srv); +} + +template +void PipelineProcessingServer::setResponse(ros::ServiceEvent& event) +{ + for (auto it = pipelines_->begin(); it != pipelines_->end(); ++it) + { + pipeline_srv_msgs::Pipelines pipeline_msg; + pipeline_msg.name = it->first; + pipeline_msg.running_status = std::to_string(it->second.state); + + auto connection_map = it->second.pipeline->getPipelineDetail(); + for (auto& current_pipe : connection_map) + { + pipeline_srv_msgs::Pipeline connection; + connection.input = current_pipe.first.c_str(); + connection.output = current_pipe.second.c_str(); + pipeline_msg.connections.push_back(connection); + } + event.getResponse().pipelines.push_back(pipeline_msg); + } +} + +template +void PipelineProcessingServer::setPipelineByRequest(std::string pipeline_name, PipelineManager::PipelineState state) +{ + for (auto it = pipelines_->begin(); it != pipelines_->end(); ++it) + { + if (pipeline_name == it->first) + { + it->second.state = state; + } + } +} + +template +bool PipelineProcessingServer::cbService(ros::ServiceEvent& event) +{ + std::string req_cmd = event.getRequest().pipeline_request.cmd; + std::string req_val = event.getRequest().pipeline_request.value; + slog::info << "[PipelineProcessingServer] Pipeline Service get request cmd: " << req_cmd << " val:" << req_val + << slog::endl; + + PipelineManager::PipelineState state; + if (req_cmd != "GET_PIPELINE") // not only get pipeline but also set pipeline by request + { + if (req_cmd == "STOP_PIPELINE") + state = PipelineManager::PipelineState_ThreadStopped; + else if (req_cmd == "RUN_PIPELINE") + state = PipelineManager::PipelineState_ThreadRunning; + else if (req_cmd == "PAUSE_PIPELINE") + state = PipelineManager::PipelineState_ThreadPasued; + + setPipelineByRequest(req_val, state); + } + else // fill in pipeline status into service response + { + setResponse(event); + } + + return true; +} +template class PipelineProcessingServer; +} // namespace vino_service diff --git a/dynamic_vino_lib/src/services/test.cpp b/dynamic_vino_lib/src/services/test.cpp new file mode 100644 index 00000000..e69de29b diff --git a/people_msgs/CMakeLists.txt b/people_msgs/CMakeLists.txt index fbf650ad..195f82e8 100644 --- a/people_msgs/CMakeLists.txt +++ b/people_msgs/CMakeLists.txt @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 2.8.3) +cmake_minimum_required(VERSION 3.0.2) project(people_msgs) @@ -35,6 +35,14 @@ add_message_files(DIRECTORY msg FILES ObjectsInMasks.msg Reidentification.msg ReidentificationStamped.msg + PersonAttribute.msg + PersonAttributeStamped.msg + LicensePlate.msg + LicensePlateStamped.msg + VehicleAttribs.msg + VehicleAttribsStamped.msg + Landmark.msg + LandmarkStamped.msg ) add_service_files(FILES diff --git a/people_msgs/msg/Landmark.msg b/people_msgs/msg/Landmark.msg new file mode 100644 index 00000000..48513b54 --- /dev/null +++ b/people_msgs/msg/Landmark.msg @@ -0,0 +1,17 @@ +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This message can represent the detected ROI and it's landmarks +geometry_msgs/Point[] landmark_points # target ID +sensor_msgs/RegionOfInterest roi # region of interest \ No newline at end of file diff --git a/people_msgs/msg/LandmarkStamped.msg b/people_msgs/msg/LandmarkStamped.msg new file mode 100644 index 00000000..bc796313 --- /dev/null +++ b/people_msgs/msg/LandmarkStamped.msg @@ -0,0 +1,17 @@ +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This message can represent the detected ROIs and their landmarks +std_msgs/Header header # timestamp in header is the time the sensor captured the raw data +Landmark[] landmarks # landmarks vector for different ROIs \ No newline at end of file diff --git a/people_msgs/msg/LicensePlate.msg b/people_msgs/msg/LicensePlate.msg new file mode 100644 index 00000000..6ba97f8c --- /dev/null +++ b/people_msgs/msg/LicensePlate.msg @@ -0,0 +1,16 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +string license # The license for the detected vehicle in roi +sensor_msgs/RegionOfInterest roi # region of interest for a vehicle license \ No newline at end of file diff --git a/people_msgs/msg/LicensePlateStamped.msg b/people_msgs/msg/LicensePlateStamped.msg new file mode 100644 index 00000000..fa4fbc75 --- /dev/null +++ b/people_msgs/msg/LicensePlateStamped.msg @@ -0,0 +1,16 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +std_msgs/Header header +people_msgs/LicensePlate[] licenses \ No newline at end of file diff --git a/people_msgs/msg/PersonAttribute.msg b/people_msgs/msg/PersonAttribute.msg new file mode 100644 index 00000000..12dd2793 --- /dev/null +++ b/people_msgs/msg/PersonAttribute.msg @@ -0,0 +1,17 @@ +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This message can represent a peron's attributes +string attribute # person attributes +sensor_msgs/RegionOfInterest roi # region of interest \ No newline at end of file diff --git a/people_msgs/msg/PersonAttributeStamped.msg b/people_msgs/msg/PersonAttributeStamped.msg new file mode 100644 index 00000000..0fb6dfcd --- /dev/null +++ b/people_msgs/msg/PersonAttributeStamped.msg @@ -0,0 +1,17 @@ +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This message can represent identified persons' attributes +std_msgs/Header header # timestamp in header is the time the sensor captured the raw data +PersonAttribute[] attributes # ObjectInBox array \ No newline at end of file diff --git a/people_msgs/msg/VehicleAttribs.msg b/people_msgs/msg/VehicleAttribs.msg new file mode 100644 index 00000000..39b49696 --- /dev/null +++ b/people_msgs/msg/VehicleAttribs.msg @@ -0,0 +1,17 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +string type # type: car, van, truck, bus +string color # color: blue, gray, yellow, green, black, red, white +sensor_msgs/RegionOfInterest roi # region of interest for a vehicle diff --git a/people_msgs/msg/VehicleAttribsStamped.msg b/people_msgs/msg/VehicleAttribsStamped.msg new file mode 100644 index 00000000..3cdcd47e --- /dev/null +++ b/people_msgs/msg/VehicleAttribsStamped.msg @@ -0,0 +1,16 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +std_msgs/Header header +people_msgs/VehicleAttribs[] vehicles diff --git a/pipeline_srv_msgs/CMakeLists.txt b/pipeline_srv_msgs/CMakeLists.txt new file mode 100644 index 00000000..6144488e --- /dev/null +++ b/pipeline_srv_msgs/CMakeLists.txt @@ -0,0 +1,88 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.0.2) + +project(pipeline_srv_msgs) + +find_package(catkin REQUIRED COMPONENTS + std_msgs + sensor_msgs + geometry_msgs + message_generation +) + +add_message_files(DIRECTORY msg FILES + PipelineRequest.msg + Pipeline.msg + Pipelines.msg +) + +add_service_files(FILES + PipelineSrv.srv +) +generate_messages(DEPENDENCIES + std_msgs + sensor_msgs + geometry_msgs + object_msgs +) + +catkin_package( + CATKIN_DEPENDS + std_msgs + sensor_msgs + geometry_msgs + message_runtime +) + +# Flags +if(UNIX OR APPLE) + # Linker flags. + if( ${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Intel") + # GCC specific flags. ICC is compatible with them. + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -z noexecstack -z relro -z now") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -z noexecstack -z relro -z now") + elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") + # In Clang, -z flags are not compatible, they need to be passed to linker via -Wl. + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now") + endif() + + # Compiler flags. + if( ${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") + # GCC specific flags. + if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9 OR CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 4.9) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIE -fstack-protector-strong") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIE -fstack-protector") + endif() + elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") + # Clang is compatbile with some of the flags. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIE -fstack-protector") + elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL "Intel" ) + # Same as above, with exception that ICC compilation crashes with -fPIE option, even + # though it uses -pie linker option that require -fPIE during compilation. Checksec + # shows that it generates correct PIE anyway if only -pie is provided. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstack-protector") + endif() + + # Generic flags. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fno-operator-names -Wformat -Wformat-security -Wall") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + # Dot not forward c++11 flag to GPU beucause it is not supported + set( CUDA_PROPAGATE_HOST_FLAGS OFF ) + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -D_FORTIFY_SOURCE=2") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pie") +endif() diff --git a/pipeline_srv_msgs/msg/Pipeline.msg b/pipeline_srv_msgs/msg/Pipeline.msg new file mode 100644 index 00000000..80f63a18 --- /dev/null +++ b/pipeline_srv_msgs/msg/Pipeline.msg @@ -0,0 +1,16 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +string input # From +string output # To diff --git a/pipeline_srv_msgs/msg/PipelineRequest.msg b/pipeline_srv_msgs/msg/PipelineRequest.msg new file mode 100644 index 00000000..565ea4fc --- /dev/null +++ b/pipeline_srv_msgs/msg/PipelineRequest.msg @@ -0,0 +1,17 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +std_msgs/Header header # Header +string cmd # Name of a request action +string value # value of an action \ No newline at end of file diff --git a/pipeline_srv_msgs/msg/Pipelines.msg b/pipeline_srv_msgs/msg/Pipelines.msg new file mode 100644 index 00000000..b2928b38 --- /dev/null +++ b/pipeline_srv_msgs/msg/Pipelines.msg @@ -0,0 +1,18 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +std_msgs/Header header # Header +string name # Name of pipeline +Pipeline[] connections # connection map of a pipeline +string running_status # Pipeline running state \ No newline at end of file diff --git a/pipeline_srv_msgs/package.xml b/pipeline_srv_msgs/package.xml new file mode 100644 index 00000000..6d174e93 --- /dev/null +++ b/pipeline_srv_msgs/package.xml @@ -0,0 +1,39 @@ + + + + + + pipeline_srv_msgs + 0.3.0 + A package containing people message definitions. + Yang Lu + Weizhi Liu + Chao Li + Hongkun Chen + Weizhi Liu + Weizhi Li + Apache License 2.0 + + catkin + + std_msgs + sensor_msgs + geometry_msgs + message_generation + + std_msgs + sensor_msgs + geometry_msgs + message_runtime + diff --git a/pipeline_srv_msgs/srv/PipelineSrv.srv b/pipeline_srv_msgs/srv/PipelineSrv.srv new file mode 100644 index 00000000..f3e3f952 --- /dev/null +++ b/pipeline_srv_msgs/srv/PipelineSrv.srv @@ -0,0 +1,17 @@ +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PipelineRequest pipeline_request # request content of pipeline service +--- +Pipelines[] pipelines # return all pipeline status diff --git a/sample/CMakeLists.txt b/sample/CMakeLists.txt index f91efb5c..82a9b857 100644 --- a/sample/CMakeLists.txt +++ b/sample/CMakeLists.txt @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 2.8.3) +cmake_minimum_required(VERSION 3.0.2) project(dynamic_vino_sample) @@ -66,7 +66,7 @@ include_directories ( ${InferenceEngine_INCLUDE_DIRS}/../build/samples/thirdparty/gflags/include ${InferenceEngine_INCLUDE_DIRS}/../samples ${InferenceEngine_DIR}/../src - ${dynamic_vino_lib_INCLUDE_DIRS} + #${dynamic_vino_lib_INCLUDE_DIRS} ${vino_param_lib_INCLUDE_DIRS} ) @@ -79,21 +79,21 @@ else() return() endif() -if (NOT DEFINED ENV{CPU_EXTENSION_LIB}) - message(FATAL_ERROR "Please set ENV CPU_EXTENSION_LIB with 'export CPU_EXTENSION_LIB='") -endif() -set (CpuExtension_lib $ENV{CPU_EXTENSION_LIB}) -add_library(cpu_extension SHARED IMPORTED) -set_target_properties(cpu_extension PROPERTIES - IMPORTED_LOCATION $ENV{CPU_EXTENSION_LIB}) - -if (NOT DEFINED ENV{GFLAGS_LIB}) - message(FATAL_ERROR "Please set ENV GFLAGS_LIB with 'export GFLAGS_LIB='") -endif() -set (Gflags_lib $ENV{GFLAGS_LIB}) -add_library(gflags STATIC IMPORTED) -set_target_properties(gflags PROPERTIES - IMPORTED_LOCATION $ENV{GFLAGS_LIB}) +#if (NOT DEFINED ENV{CPU_EXTENSION_LIB}) +# message(FATAL_ERROR "Please set ENV CPU_EXTENSION_LIB with 'export CPU_EXTENSION_LIB='") +#endif() +#set (CpuExtension_lib $ENV{CPU_EXTENSION_LIB}) +#add_library(cpu_extension SHARED IMPORTED) +#set_target_properties(cpu_extension PROPERTIES +# IMPORTED_LOCATION $ENV{CPU_EXTENSION_LIB}) + +#if (NOT DEFINED ENV{GFLAGS_LIB}) +# message(FATAL_ERROR "Please set ENV GFLAGS_LIB with 'export GFLAGS_LIB='") +#endif() +#set (Gflags_lib $ENV{GFLAGS_LIB}) +#add_library(gflags STATIC IMPORTED) +#set_target_properties(gflags PROPERTIES +# IMPORTED_LOCATION $ENV{GFLAGS_LIB}) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") @@ -109,7 +109,6 @@ add_dependencies(pipeline_with_params target_link_libraries(pipeline_with_params ${catkin_LIBRARIES} - cpu_extension gflags ${vino_param_lib_LIBRARIES} ${InferenceEngine_LIBRARIES} @@ -128,7 +127,6 @@ add_dependencies(image_people_server target_link_libraries(image_people_server ${catkin_LIBRARIES} - cpu_extension gflags ${vino_param_lib_LIBRARIES} ${InferenceEngine_LIBRARIES} @@ -148,7 +146,6 @@ add_dependencies(image_people_client target_link_libraries(image_people_client ${catkin_LIBRARIES} - cpu_extension gflags ${vino_param_lib_LIBRARIES} ${InferenceEngine_LIBRARIES} @@ -167,7 +164,6 @@ add_dependencies(image_object_server target_link_libraries(image_object_server ${catkin_LIBRARIES} - cpu_extension gflags ${vino_param_lib_LIBRARIES} ${InferenceEngine_LIBRARIES} @@ -187,7 +183,6 @@ add_dependencies(image_object_client target_link_libraries(image_object_client ${catkin_LIBRARIES} - cpu_extension gflags ${vino_param_lib_LIBRARIES} ${InferenceEngine_LIBRARIES} @@ -207,7 +202,6 @@ add_dependencies(image_segmentation_client target_link_libraries(image_segmentation_client ${catkin_LIBRARIES} - cpu_extension gflags ${vino_param_lib_LIBRARIES} ${InferenceEngine_LIBRARIES} @@ -226,7 +220,6 @@ add_dependencies(image_segmentation_server target_link_libraries(image_segmentation_server ${catkin_LIBRARIES} - cpu_extension gflags ${vino_param_lib_LIBRARIES} ${InferenceEngine_LIBRARIES} @@ -246,7 +239,6 @@ add_dependencies(image_reidentification_client target_link_libraries(image_reidentification_client ${catkin_LIBRARIES} - cpu_extension gflags ${vino_param_lib_LIBRARIES} ${InferenceEngine_LIBRARIES} @@ -265,13 +257,29 @@ add_dependencies(image_reidentification_server target_link_libraries(image_reidentification_server ${catkin_LIBRARIES} - cpu_extension gflags ${vino_param_lib_LIBRARIES} ${InferenceEngine_LIBRARIES} ${OpenCV_LIBRARIES} ) +add_executable(pipeline_service_client + src/pipeline_service_client.cpp +) + +add_dependencies(pipeline_service_client + ${${PROJECT_NAME}_EXPORTED_TARGETS} + ${catkin_EXPORTED_TARGETS} + ${dynamic_vino_lib_TARGETS} +) + +target_link_libraries(pipeline_service_client + ${catkin_LIBRARIES} + gflags + ${vino_param_lib_LIBRARIES} + ${InferenceEngine_LIBRARIES} + ${OpenCV_LIBRARIES} +) if(UNIX OR APPLE) # Linker flags. diff --git a/sample/include/sample/utility.hpp b/sample/include/sample/utility.hpp index f29217d0..b1e052e5 100644 --- a/sample/include/sample/utility.hpp +++ b/sample/include/sample/utility.hpp @@ -32,84 +32,73 @@ static const char help_message[] = "Print a usage message."; /// @brief message for images argument -static const char input_choice[] = - "Optional. Input choice (RealSenseCamera, StandardCamera, Video, Image). " \ - "Default value is StandardCamera."; +static const char input_choice[] = "Optional. Input choice (RealSenseCamera, StandardCamera, Video, Image). " + "Default value is StandardCamera."; /// @brief message for model argument -static const char face_detection_model_message[] = - "Required. Path to an .xml file with a trained face detection model."; -static const char age_gender_model_message[] = - "Optional. Path to an .xml file with a trained age gender model."; -static const char head_pose_model_message[] = - "Optional. Path to an .xml file with a trained head pose model."; -static const char emotions_model_message[] = - "Optional. Path to an .xml file with a trained emotions model."; +static const char face_detection_model_message[] = "Required. Path to an .xml file with a trained face detection " + "model."; +static const char age_gender_model_message[] = "Optional. Path to an .xml file with a trained age gender model."; +static const char head_pose_model_message[] = "Optional. Path to an .xml file with a trained head pose model."; +static const char emotions_model_message[] = "Optional. Path to an .xml file with a trained emotions model."; static const char object_model_message[] = "Required. Path to an .xml file with a trained model."; /// @brief message for plugin argument -static const char plugin_message[] = - "Plugin name. For example MKLDNNPlugin. If this parameter is pointed, " \ -"the sample will look for this plugin only."; +static const char plugin_message[] = "Plugin name. For example MKLDNNPlugin. If this parameter is pointed, " + "the sample will look for this plugin only."; /// @brief message for assigning face detection calculation to device -static const char target_device_message[] = - "Specify the target device for Face Detection (CPU, GPU, FPGA, or MYRIAD). " \ -"Sample will look for a suitable plugin for device specified."; +static const char target_device_message[] = "Specify the target device for Face Detection (CPU, GPU, FPGA, or MYRIAD). " + "Sample will look for a suitable plugin for device specified."; /// @brief message for assigning age gender calculation to device static const char target_device_message_ag[] = - "Specify the target device for Age Gender Detection (CPU, GPU, FPGA, or MYRIAD). " \ -"Sample will look for a suitable plugin for device specified."; + "Specify the target device for Age Gender Detection (CPU, GPU, FPGA, or MYRIAD). " + "Sample will look for a suitable plugin for device specified."; /// @brief message for assigning age gender calculation to device static const char target_device_message_hp[] = - "Specify the target device for Head Pose Detection (CPU, GPU, FPGA, or MYRIAD). " \ -"Sample will look for a suitable plugin for device specified."; + "Specify the target device for Head Pose Detection (CPU, GPU, FPGA, or MYRIAD). " + "Sample will look for a suitable plugin for device specified."; static const char target_device_message_em[] = - "Specify the target device for Emotions Detection (CPU, GPU, FPGA, or MYRIAD). " \ -"Sample will look for a suitable plugin for device specified."; + "Specify the target device for Emotions Detection (CPU, GPU, FPGA, or MYRIAD). " + "Sample will look for a suitable plugin for device specified."; /// @brief message for number of simultaneously age gender detections using dynamic batch static const char num_batch_ag_message[] = - "Specify number of maximum simultaneously processed faces for Age Gender Detection" \ + "Specify number of maximum simultaneously processed faces for Age Gender Detection" "(default is 16)."; /// @brief message for number of simultaneously age gender detections using dynamic batch static const char num_batch_hp_message[] = - "Specify number of maximum simultaneously processed faces for Head Pose Detection" \ + "Specify number of maximum simultaneously processed faces for Head Pose Detection" "(default is 16)."; /// @brief message for number of simultaneously age gender detections using dynamic batch static const char num_batch_em_message[] = - "Specify number of maximum simultaneously processed faces for Emotions Detection" \ + "Specify number of maximum simultaneously processed faces for Emotions Detection" "(default is 16)."; /// @brief message for performance counters -static const char - performance_counter_message[] = "Enables per-layer performance report."; +static const char performance_counter_message[] = "Enables per-layer performance report."; /// @brief message for clDNN custom kernels desc -static const char - custom_cldnn_message[] = "Required for clDNN (GPU)-targeted custom kernels."\ -"Absolute path to the xml file with the kernels desc."; +static const char custom_cldnn_message[] = "Required for clDNN (GPU)-targeted custom kernels." + "Absolute path to the xml file with the kernels desc."; /// @brief message for user library argument -static const char custom_cpu_library_message[] = - "Required for MKLDNN (CPU)-targeted custom layers." \ -"Absolute path to a shared library with the kernels impl."; +static const char custom_cpu_library_message[] = "Required for MKLDNN (CPU)-targeted custom layers." + "Absolute path to a shared library with the kernels impl."; /// @brief message for probability threshold argument -static const char - thresh_output_message[] = "Probability threshold for detections."; +static const char thresh_output_message[] = "Probability threshold for detections."; /// @brief message raw output flag static const char raw_output_message[] = "Inference results as raw values."; /// @brief message no wait for keypress after input stream completed -static const char - no_wait_for_keypress_message[] = "No wait for key press in the end."; +static const char no_wait_for_keypress_message[] = "No wait for key press in the end."; /// @brief message no show processed video static const char no_show_processed_video[] = "No show processed video."; @@ -199,13 +188,12 @@ DEFINE_string(i_path, "", input_file_path); /// It is a optional parameter DEFINE_string(config, "", parameter_file_message); -static void showUsageForParam() { +static void showUsageForParam() +{ std::cout << std::endl; std::cout << "vino_param_sample [OPTION]" << std::endl; std::cout << "Options:" << std::endl; std::cout << std::endl; std::cout << " -h " << help_message << std::endl; - std::cout << " -config \"\" " << parameter_file_message - << std::endl; + std::cout << " -config \"\" " << parameter_file_message << std::endl; } - diff --git a/sample/src/image_object_client.cpp b/sample/src/image_object_client.cpp index 1cdddb9a..3b408dbe 100644 --- a/sample/src/image_object_client.cpp +++ b/sample/src/image_object_client.cpp @@ -26,47 +26,45 @@ #include - - - -int main(int argc, char ** argv) +int main(int argc, char** argv) { - ros::init(argc, argv, "image_object_client"); + ros::init(argc, argv, "image_object_client"); ros::NodeHandle n; - if (argc != 2) { + if (argc != 2) + { ROS_INFO("Usage: rosrun dynamic_vino_sample image_object_client "); - //You can find a sample image in /opt/openvino_toolkit/ros_openvino_toolkit/data/images/car.png + // You can find a sample image in /opt/openvino_toolkit/ros_openvino_toolkit/data/images/car.png return -1; } std::string image_path = argv[1]; ros::ServiceClient client = n.serviceClient("/openvino_toolkit/service"); - object_msgs::DetectObject srv; + srv.request.image_paths = image_path; if (client.call(srv)) { - ROS_INFO("Request service success!"); + ROS_INFO("Request service success!"); cv::Mat image = cv::imread(image_path); int width = image.cols; int height = image.rows; - for (unsigned int i = 0; i < srv.response.objects[0].objects_vector.size(); i++) { + for (unsigned int i = 0; i < srv.response.objects[0].objects_vector.size(); i++) + { std::stringstream ss; - ss << srv.response.objects[0].objects_vector[i].object.object_name << ": " << - srv.response.objects[0].objects_vector[i].object.probability * 100 << "%"; - ROS_INFO("%d: object: %s", i, - srv.response.objects[0].objects_vector[i].object.object_name.c_str()); - ROS_INFO( "prob: %f", - srv.response.objects[0].objects_vector[i].object.probability); - ROS_INFO( - "location: (%d, %d, %d, %d)", - srv.response.objects[0].objects_vector[i].roi.x_offset, srv.response.objects[0].objects_vector[i].roi.y_offset, - srv.response.objects[0].objects_vector[i].roi.width, srv.response.objects[0].objects_vector[i].roi.height); + ss << srv.response.objects[0].objects_vector[i].object.object_name << ": " + << srv.response.objects[0].objects_vector[i].object.probability * 100 << "%"; + ROS_INFO("%d: object: %s", i, srv.response.objects[0].objects_vector[i].object.object_name.c_str()); + ROS_INFO("prob: %f", srv.response.objects[0].objects_vector[i].object.probability); + ROS_INFO("location: (%d, %d, %d, %d)", + srv.response.objects[0].objects_vector[i].roi.x_offset, + srv.response.objects[0].objects_vector[i].roi.y_offset, + srv.response.objects[0].objects_vector[i].roi.width, + srv.response.objects[0].objects_vector[i].roi.height); int xmin = srv.response.objects[0].objects_vector[i].roi.x_offset; int ymin = srv.response.objects[0].objects_vector[i].roi.y_offset; @@ -79,24 +77,15 @@ int main(int argc, char ** argv) cv::Point left_top = cv::Point(xmin, ymin); cv::Point right_bottom = cv::Point(xmax, ymax); cv::rectangle(image, left_top, right_bottom, cv::Scalar(0, 255, 0), 1, 8, 0); - cv::rectangle(image, cvPoint(xmin, ymin), cvPoint(xmax, ymin + 20), cv::Scalar(0, 255, 0), - -1); - cv::putText(image, ss.str(), cvPoint(xmin + 5, ymin + 20), cv::FONT_HERSHEY_PLAIN, 1, - cv::Scalar(0, 0, 255), 1); + cv::rectangle(image, cv::Point(xmin, ymin), cv::Point(xmax, ymin + 20), cv::Scalar(0, 255, 0), -1); + cv::putText(image, ss.str(), cv::Point(xmin + 5, ymin + 20), cv::FONT_HERSHEY_PLAIN, 1, cv::Scalar(0, 0, 255), 1); } cv::imshow("image_detection", image); cv::waitKey(0); - - - } else + } + else { - ROS_ERROR("Failed to request service \"openvino_toolkit/service\" "); return -1; + ROS_ERROR("Failed to request service \"openvino_toolkit/service\" "); + return -1; } - - - - } - - - diff --git a/sample/src/image_object_server.cpp b/sample/src/image_object_server.cpp index 25905814..5c62b140 100644 --- a/sample/src/image_object_server.cpp +++ b/sample/src/image_object_server.cpp @@ -42,7 +42,6 @@ #include #include "dynamic_vino_lib/common.h" #include "dynamic_vino_lib/engines/engine.h" -#include "dynamic_vino_lib/factory.h" #include "dynamic_vino_lib/inferences/age_gender_detection.h" #include "dynamic_vino_lib/inferences/base_inference.h" #include "dynamic_vino_lib/inferences/emotions_detection.h" @@ -75,8 +74,9 @@ bool parseAndCheckCommandLine(int argc, char** argv) int main(int argc, char** argv) { ros::init(argc, argv, "image_object_servier"); - - if (!parseAndCheckCommandLine(argc, argv)) return 0; + + if (!parseAndCheckCommandLine(argc, argv)) + return 0; ros::param::param("~param_file", FLAGS_config, "/param/pipeline_people.yaml"); @@ -84,14 +84,12 @@ int main(int argc, char** argv) std::string service_name = "/openvino_toolkit/service"; slog::info << "service name=" << service_name << slog::endl; - // ----- Parsing and validation of input args----------------------- + // ----- Parsing and validation of input args----------------------- + auto node = + std::make_shared>(service_name, FLAGS_config); - auto node = std::make_shared>(service_name, FLAGS_config); - slog::info << "Waiting for service request..." << slog::endl; ros::spin(); slog::info << "--------------End of Excution--------------" << FLAGS_config << slog::endl; - } diff --git a/sample/src/image_people_client.cpp b/sample/src/image_people_client.cpp index 5dfb1b60..85d310a7 100644 --- a/sample/src/image_people_client.cpp +++ b/sample/src/image_people_client.cpp @@ -25,59 +25,54 @@ #include - - -int main(int argc, char ** argv) +int main(int argc, char** argv) { - ros::init(argc, argv, "image_people_client"); + ros::init(argc, argv, "image_people_client"); ros::NodeHandle n; - if (argc != 2) { - ROS_INFO( "Usage: rosrun dynamic_vino_sample image_people_client" - ""); + if (argc != 2) + { + ROS_INFO("Usage: rosrun dynamic_vino_sample image_people_client" + ""); return -1; } - ros::ServiceClient client = n.serviceClient("/openvino_toolkit/service"); + std::string image_path = argv[1]; + people_msgs::PeopleSrv srv; + srv.request.image_path = image_path; if (client.call(srv)) { - ROS_INFO("Request service success!"); + ROS_INFO("Request service success!"); if (srv.response.persons.emotions.size() == 0 && srv.response.persons.agegenders.size() == 0 && srv.response.persons.headposes.size() == 0) { - ROS_INFO( "Get response, but no any person found."); + ROS_INFO("Get response, but no any person found."); return 0; } - ROS_INFO( "Found persons..."); + ROS_INFO("Found persons..."); - for (unsigned int i = 0; i < srv.response.persons.faces.size(); i++) { - ROS_INFO( "%d: object: %s", i, - srv.response.persons.faces[i].object.object_name.c_str()); - ROS_INFO( "prob: %f", - srv.response.persons.faces[i].object.probability); - ROS_INFO( "location: (%d, %d, %d, %d)", - srv.response.persons.faces[i].roi.x_offset, srv.response.persons.faces[i].roi.y_offset, - srv.response.persons.faces[i].roi.width, srv.response.persons.faces[i].roi.height); - ROS_INFO( "Emotions: %s", - srv.response.persons.emotions[i].emotion.c_str()); - ROS_INFO( "Age: %f, Gender: %s", - srv.response.persons.agegenders[i].age, srv.response.persons.agegenders[i].gender.c_str()); - ROS_INFO( "Yaw, Pitch and Roll for head pose is: (%f, %f, %f),", - srv.response.persons.headposes[i].yaw, srv.response.persons.headposes[i].pitch, - srv.response.persons.headposes[i].roll); - } + for (unsigned int i = 0; i < srv.response.persons.faces.size(); i++) + { + ROS_INFO("%d: object: %s", i, srv.response.persons.faces[i].object.object_name.c_str()); + ROS_INFO("prob: %f", srv.response.persons.faces[i].object.probability); + ROS_INFO("location: (%d, %d, %d, %d)", srv.response.persons.faces[i].roi.x_offset, + srv.response.persons.faces[i].roi.y_offset, srv.response.persons.faces[i].roi.width, + srv.response.persons.faces[i].roi.height); + ROS_INFO("Emotions: %s", srv.response.persons.emotions[i].emotion.c_str()); + ROS_INFO("Age: %f, Gender: %s", srv.response.persons.agegenders[i].age, + srv.response.persons.agegenders[i].gender.c_str()); + ROS_INFO("Yaw, Pitch and Roll for head pose is: (%f, %f, %f),", srv.response.persons.headposes[i].yaw, + srv.response.persons.headposes[i].pitch, srv.response.persons.headposes[i].roll); + } } - else + else { - ROS_ERROR("Failed to request service \"/openvino_toolkit/service\" "); - return -1; + ROS_ERROR("Failed to request service \"/openvino_toolkit/service\" "); + return -1; } } - - - diff --git a/sample/src/image_people_server.cpp b/sample/src/image_people_server.cpp index 92520ec5..f32de3ee 100644 --- a/sample/src/image_people_server.cpp +++ b/sample/src/image_people_server.cpp @@ -42,7 +42,6 @@ #include #include "dynamic_vino_lib/common.h" #include "dynamic_vino_lib/engines/engine.h" -#include "dynamic_vino_lib/factory.h" #include "dynamic_vino_lib/inferences/age_gender_detection.h" #include "dynamic_vino_lib/inferences/base_inference.h" #include "dynamic_vino_lib/inferences/emotions_detection.h" @@ -59,7 +58,6 @@ #include "opencv2/opencv.hpp" #include "sample/utility.hpp" - bool parseAndCheckCommandLine(int argc, char** argv) { // -----Parsing and validation of input args--------------------------- @@ -76,9 +74,9 @@ bool parseAndCheckCommandLine(int argc, char** argv) int main(int argc, char** argv) { ros::init(argc, argv, "sample_image_people_client"); - - if (!parseAndCheckCommandLine(argc, argv)) return 0; + if (!parseAndCheckCommandLine(argc, argv)) + return 0; ros::param::param("~param_file", FLAGS_config, "/param/pipeline_people.yaml"); @@ -88,13 +86,11 @@ int main(int argc, char** argv) slog::info << "service name=" << service_name << slog::endl; - auto node = std::make_shared>(service_name, FLAGS_config); - + auto node = std::make_shared>(service_name, FLAGS_config); + slog::info << "Waiting for service request..." << slog::endl; - + ros::spin(); - - slog::info << "--------------End of Excution--------------" << FLAGS_config << slog::endl; + slog::info << "--------------End of Excution--------------" << FLAGS_config << slog::endl; } diff --git a/sample/src/image_reidentification_client.cpp b/sample/src/image_reidentification_client.cpp index 70a94991..ab7a8e31 100644 --- a/sample/src/image_reidentification_client.cpp +++ b/sample/src/image_reidentification_client.cpp @@ -26,46 +26,35 @@ #include - - -int main(int argc, char ** argv) +int main(int argc, char** argv) { - ros::init(argc, argv, "image_reidentification_client"); + ros::init(argc, argv, "image_reidentification_client"); ros::NodeHandle n; ros::ServiceClient client = n.serviceClient("/openvino_toolkit/service"); - people_msgs::ReidentificationSrv srv; if (client.call(srv)) { - ROS_INFO("Request service success!"); + ROS_INFO("Request service success!"); - for (unsigned int i = 0; i < srv.response.reidentification.reidentified_vector.size(); i++) { + for (unsigned int i = 0; i < srv.response.reidentification.reidentified_vector.size(); i++) + { std::stringstream ss; - ss << srv.response.reidentification.reidentified_vector[i].identity ; - ROS_INFO("%d: object: %s", i, - srv.response.reidentification.reidentified_vector[i].identity.c_str()); - - ROS_INFO( - "location: (%d, %d, %d, %d)", - srv.response.reidentification.reidentified_vector[i].roi.x_offset, srv.response.reidentification.reidentified_vector[i].roi.y_offset, - srv.response.reidentification.reidentified_vector[i].roi.width,srv.response.reidentification.reidentified_vector[i].roi.height); + ss << srv.response.reidentification.reidentified_vector[i].identity; + ROS_INFO("%d: object: %s", i, srv.response.reidentification.reidentified_vector[i].identity.c_str()); + + ROS_INFO("location: (%d, %d, %d, %d)", srv.response.reidentification.reidentified_vector[i].roi.x_offset, + srv.response.reidentification.reidentified_vector[i].roi.y_offset, + srv.response.reidentification.reidentified_vector[i].roi.width, + srv.response.reidentification.reidentified_vector[i].roi.height); } - - } - else + else { - ROS_ERROR("Failed to request service \"openvino_toolkit/service\" "); return -1; + ROS_ERROR("Failed to request service \"openvino_toolkit/service\" "); + return -1; } - - - - } - - - \ No newline at end of file diff --git a/sample/src/image_reidentification_server.cpp b/sample/src/image_reidentification_server.cpp index 86293ddb..0eaba1aa 100644 --- a/sample/src/image_reidentification_server.cpp +++ b/sample/src/image_reidentification_server.cpp @@ -42,7 +42,6 @@ #include #include "dynamic_vino_lib/common.h" #include "dynamic_vino_lib/engines/engine.h" -#include "dynamic_vino_lib/factory.h" #include "dynamic_vino_lib/inferences/age_gender_detection.h" #include "dynamic_vino_lib/inferences/base_inference.h" #include "dynamic_vino_lib/inferences/emotions_detection.h" @@ -60,7 +59,6 @@ #include "sample/utility.hpp" #include - bool parseAndCheckCommandLine(int argc, char** argv) { // -----Parsing and validation of input args--------------------------- @@ -77,8 +75,9 @@ bool parseAndCheckCommandLine(int argc, char** argv) int main(int argc, char** argv) { ros::init(argc, argv, "image_segmentation_servier"); - - if (!parseAndCheckCommandLine(argc, argv)) return 0; + + if (!parseAndCheckCommandLine(argc, argv)) + return 0; ros::param::param("~param_file", FLAGS_config, "/param/image_segmentation_server.yaml"); @@ -86,14 +85,12 @@ int main(int argc, char** argv) std::string service_name = "/openvino_toolkit/service"; slog::info << "service name=" << service_name << slog::endl; - // ----- Parsing and validation of input args----------------------- + // ----- Parsing and validation of input args----------------------- + auto node = std::make_shared>(service_name, + FLAGS_config); - auto node = std::make_shared>(service_name, FLAGS_config); - slog::info << "Waiting for reid service request..." << slog::endl; ros::spin(); slog::info << "--------------End of Excution--------------" << FLAGS_config << slog::endl; - } diff --git a/sample/src/image_segmentation_client.cpp b/sample/src/image_segmentation_client.cpp index be98d04a..60b0505b 100644 --- a/sample/src/image_segmentation_client.cpp +++ b/sample/src/image_segmentation_client.cpp @@ -26,47 +26,36 @@ #include - - -int main(int argc, char ** argv) +int main(int argc, char** argv) { - ros::init(argc, argv, "image_segmentation_client"); + ros::init(argc, argv, "image_segmentation_client"); ros::NodeHandle n; ros::ServiceClient client = n.serviceClient("/openvino_toolkit/service"); - people_msgs::ObjectsInMasksSrv srv; if (client.call(srv)) { - ROS_INFO("Request service success!"); - - for (unsigned int i = 0; i < srv.response.segmentation.objects_vector.size(); i++) { + ROS_INFO("Request service success!"); + + for (unsigned int i = 0; i < srv.response.segmentation.objects_vector.size(); i++) + { std::stringstream ss; - ss << srv.response.segmentation.objects_vector[i].object_name << ": " << - srv.response.segmentation.objects_vector[i].probability * 100 << "%"; - ROS_INFO("%d: object: %s", i, - srv.response.segmentation.objects_vector[i].object_name.c_str()); - ROS_INFO( "prob: %f", - srv.response.segmentation.objects_vector[i].probability); - ROS_INFO( - "location: (%d, %d, %d, %d)", - srv.response.segmentation.objects_vector[i].roi.x_offset, srv.response.segmentation.objects_vector[i].roi.y_offset, - srv.response.segmentation.objects_vector[i].roi.width,srv.response.segmentation.objects_vector[i].roi.height); + ss << srv.response.segmentation.objects_vector[i].object_name << ": " + << srv.response.segmentation.objects_vector[i].probability * 100 << "%"; + ROS_INFO("%d: object: %s", i, srv.response.segmentation.objects_vector[i].object_name.c_str()); + ROS_INFO("prob: %f", srv.response.segmentation.objects_vector[i].probability); + ROS_INFO("location: (%d, %d, %d, %d)", srv.response.segmentation.objects_vector[i].roi.x_offset, + srv.response.segmentation.objects_vector[i].roi.y_offset, + srv.response.segmentation.objects_vector[i].roi.width, + srv.response.segmentation.objects_vector[i].roi.height); } - } - else + else { - ROS_ERROR("Failed to request service \"openvino_toolkit/service\" "); return -1; + ROS_ERROR("Failed to request service \"openvino_toolkit/service\" "); + return -1; } - - - - } - - - \ No newline at end of file diff --git a/sample/src/image_segmentation_server.cpp b/sample/src/image_segmentation_server.cpp index 34288a29..7a827609 100644 --- a/sample/src/image_segmentation_server.cpp +++ b/sample/src/image_segmentation_server.cpp @@ -42,7 +42,6 @@ #include #include "dynamic_vino_lib/common.h" #include "dynamic_vino_lib/engines/engine.h" -#include "dynamic_vino_lib/factory.h" #include "dynamic_vino_lib/inferences/age_gender_detection.h" #include "dynamic_vino_lib/inferences/base_inference.h" #include "dynamic_vino_lib/inferences/emotions_detection.h" @@ -59,8 +58,6 @@ #include "opencv2/opencv.hpp" #include "sample/utility.hpp" - - bool parseAndCheckCommandLine(int argc, char** argv) { // -----Parsing and validation of input args--------------------------- @@ -77,8 +74,9 @@ bool parseAndCheckCommandLine(int argc, char** argv) int main(int argc, char** argv) { ros::init(argc, argv, "image_segmentation_servier"); - - if (!parseAndCheckCommandLine(argc, argv)) return 0; + + if (!parseAndCheckCommandLine(argc, argv)) + return 0; ros::param::param("~param_file", FLAGS_config, "/param/image_segmentation_server.yaml"); @@ -86,14 +84,12 @@ int main(int argc, char** argv) std::string service_name = "/openvino_toolkit/service"; slog::info << "service name=" << service_name << slog::endl; - // ----- Parsing and validation of input args----------------------- + // ----- Parsing and validation of input args----------------------- + auto node = + std::make_shared>(service_name, FLAGS_config); - auto node = std::make_shared>(service_name, FLAGS_config); - slog::info << "Waiting for seg service request..." << slog::endl; ros::spin(); slog::info << "--------------End of Excution--------------" << FLAGS_config << slog::endl; - } diff --git a/sample/src/pipeline_service_client.cpp b/sample/src/pipeline_service_client.cpp new file mode 100644 index 00000000..108c68d5 --- /dev/null +++ b/sample/src/pipeline_service_client.cpp @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2018 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** +* \brief A sample for this library. This sample performs face detection, + * emotions detection, age gender detection and head pose estimation. +* \file sample/main.cpp +*/ + +#include +#include +#include "opencv2/opencv.hpp" + +#include +#include + +pipeline_srv_msgs::PipelineSrv getRequestMsg(std::string cmd, std::string value) +{ + pipeline_srv_msgs::PipelineSrv srv; + srv.request.pipeline_request.cmd = cmd; + srv.request.pipeline_request.value = value; + return srv; +} + +bool stopPipelineByName(ros::ServiceClient& client, std::string name) +{ + auto srv = getRequestMsg("STOP_PIPELINE", name); // object_pipeline1 + bool result = client.call(srv) ? true : false; + return result; +} + +bool pausePipelineByName(ros::ServiceClient& client, std::string name) +{ + auto srv = getRequestMsg("PAUSE_PIPELINE", name); // object_pipeline1 + bool result = client.call(srv) ? true : false; + return result; +} + +bool runPipelineByName(ros::ServiceClient& client, std::string name) +{ + auto srv = getRequestMsg("RUN_PIPELINE", name); // object_pipeline1 + bool result = client.call(srv) ? true : false; + return result; +} + +bool request(ros::ServiceClient& client, std::string cmd_name, std::string cmd_value) +{ + auto srv = getRequestMsg(cmd_name, cmd_value); // object_pipeline1 + bool result = client.call(srv) ? true : false; + return result; +} + +bool getAllPipelines(ros::ServiceClient& client, std::vector& pipelines) +{ + auto srv = getRequestMsg("GET_PIPELINE", ""); + bool success = client.call(srv) ? true : false; + if (success) + { + for (auto it = srv.response.pipelines.begin(); it != srv.response.pipelines.end(); ++it) + { + pipelines.push_back(*it); + std::cout << it->name << " status:" << it->running_status << std::endl; + for (auto connect = it->connections.begin(); connect != it->connections.end(); ++connect) + { + printf("%s --> %s\n", connect->input.c_str(), connect->output.c_str()); + } + } + } + return success; +} + +int main(int argc, char** argv) +{ + if (argc != 3) + { + ROS_INFO("Usage: rosrun dynamic_vino_sample pipeline_service_client "); + return -1; + } + std::string cmd_name = argv[1]; + std::string cmd_value = argv[2]; + + ros::init(argc, argv, "pipeline_service_client"); + + ros::NodeHandle n; + + ros::ServiceClient client = n.serviceClient("/openvino_toolkit/pipeline/service"); + + std::vector pipelines; + + auto success = getAllPipelines(client, pipelines) ? true : false; + + if (!success) + { + ROS_ERROR("Failed to request service."); + return -1; + } + + success = request(client, cmd_name, cmd_value); + if (!success) + { + ROS_ERROR("Failed to request service."); + return -1; + } + // stopPipelineByName(client,"object_pipeline1"); +} diff --git a/sample/src/pipeline_with_params.cpp b/sample/src/pipeline_with_params.cpp index 475d6cb3..8bce1606 100644 --- a/sample/src/pipeline_with_params.cpp +++ b/sample/src/pipeline_with_params.cpp @@ -42,12 +42,13 @@ #include #include "dynamic_vino_lib/common.h" #include "dynamic_vino_lib/engines/engine.h" -#include "dynamic_vino_lib/factory.h" #include "dynamic_vino_lib/inferences/age_gender_detection.h" #include "dynamic_vino_lib/inferences/base_inference.h" #include "dynamic_vino_lib/inferences/emotions_detection.h" #include "dynamic_vino_lib/inferences/face_detection.h" #include "dynamic_vino_lib/inferences/head_pose_detection.h" +#include "dynamic_vino_lib/inferences/object_detection_yolov2_voc.h" +#include "dynamic_vino_lib/inferences/object_detection_ssd.h" #include "dynamic_vino_lib/inputs/realsense_camera_topic.h" #include "dynamic_vino_lib/outputs/image_window_output.h" #include "dynamic_vino_lib/outputs/ros_topic_output.h" @@ -81,20 +82,18 @@ bool parseAndCheckCommandLine(int argc, char** argv) int main(int argc, char** argv) { + ros::init(argc, argv, "sample_with_params"); - ros::init(argc, argv, "sample_with_params"); - // register signal SIGINT and signal handler signal(SIGINT, signalHandler); std::string FLAGS_config; - ros::param::param("~param_file", FLAGS_config, "/param/pipeline_people.yaml"); + ros::param::param("~param_file", FLAGS_config, "/param/pipeline_object_oss.yaml"); slog::info << "FLAGS_config=" << FLAGS_config << slog::endl; try { - std::cout << "InferenceEngine: " - << InferenceEngine::GetInferenceEngineVersion() << std::endl; + std::cout << "InferenceEngine: " << InferenceEngine::GetInferenceEngineVersion() << std::endl; // ----- Parsing and validation of input args----------------------- if (!parseAndCheckCommandLine(argc, argv)) @@ -112,15 +111,15 @@ int main(int argc, char** argv) throw std::logic_error("Pipeline parameters should be set!"); } - for (auto & p : pipelines) { + for (auto& p : pipelines) + { PipelineManager::getInstance().createPipeline(p); } - + PipelineManager::getInstance().runAll(); PipelineManager::getInstance().joinAll(); slog::info << "Execution successful" << slog::endl; - } catch (const std::exception& error) { diff --git a/script/environment_setup.sh b/script/environment_setup.sh index bcd0808a..6d9a8049 100755 --- a/script/environment_setup.sh +++ b/script/environment_setup.sh @@ -63,14 +63,13 @@ echo "Set OTHER_DEPENDENCY to $OTHER_DEPENDENCY" # Clean Existing Directories if [ "$CLEAN" == "1" ]; then read -n1 -p "The clean operation will delete some manually created directories, - including ~/code, ~/catkin_ws, /opt/intel, /opt/openvino_toolkit, and OpenVINO tar ball. + including ~/code, /opt/intel, /opt/openvino_toolkit, and OpenVINO tar ball. Do you want to clean existing directories[Y/N]?" answer case $answer in Y|y) echo echo "===================Cleaning...====================================" - echo $ROOT_PASSWD | sudo -S rm -rf ~/code - echo $ROOT_PASSWD | sudo -S rm -rf /opt/intel - rm -rf ~/catkin_ws + #echo $ROOT_PASSWD | sudo -S rm -rf ~/code + #echo $ROOT_PASSWD | sudo -S rm -rf /opt/intel echo $ROOT_PASSWD | sudo -S rm -rf /opt/openvino_toolkit if [[ $system_ver = "16.04" && -L "/usr/lib/x86_64-linux-gnu/libboost_python3.so" ]]; then echo $ROOT_PASSWD | sudo -S rm /usr/lib/x86_64-linux-gnu/libboost_python3.so @@ -84,12 +83,14 @@ fi # Setup ROS from Debian if [ "$ROS_DEBIAN" == "1" ]; then echo "===================Installing ROS from Debian Package...=======================" + echo $ROOT_PASSWD | sudo -S apt-get install -y curl + curl http://repo.ros2.org/repos.key | sudo apt-key add - echo $ROOT_PASSWD | sudo -S sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' - #echo $ROOT_PASSWD | sudo -S apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-key 421C365BD9FF1F717815A3895523BAEEB01FA116 - #For those who cannot access hkp protocal - echo $ROOT_PASSWD | curl http://repo.ros2.org/repos.key | sudo apt-key add - + echo $ROOT_PASSWD | sudo -S apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options http-proxy="$http_proxy" --recv-key F42ED6FBAB17C654 + #echo $ROOT_PASSWD | sudo -S apt-key adv --keyserver 'hkp://pgp.mit.edu:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 + #For those who cannot access hkp protocal echo $ROOT_PASSWD | sudo -S apt-get update - echo $ROOT_PASSWD | sudo -S apt-get install -y ros-$ros_ver-desktop-full #For ubuntu16.04 Ros-melodic + echo $ROOT_PASSWD | sudo -S apt-get install -y ros-$ros_ver-desktop-full if [ ! -f "/etc/ros/rosdep/sources.list.d/20-default.list" ]; then echo $ROOT_PASSWD | sudo -S rosdep init @@ -97,19 +98,19 @@ if [ "$ROS_DEBIAN" == "1" ]; then echo "file already exists, skip..." fi - set +o errexit + set +o errexit rosdep update until [ $? == 0 ] do rosdep update done tail ~/.bashrc | grep "/opt/ros/$ros_ver/setup.bash" - set -o errexit + set -o errexit if [ "$?" == "1" ]; then echo "source /opt/ros/$ros_ver/setup.bash" >> ~/.bashrc else - echo "ros melodic already set, skip..." + echo "ros $ros_ver already set, skip..." fi source ~/.bashrc echo $ROOT_PASSWD | sudo -S apt-get install -y python-rosinstall python-rosinstall-generator python-wstool build-essential @@ -211,7 +212,7 @@ if [ "$DLDT" == "1" ]; then mkdir -p ~/code && cd ~/code git clone https://github.com/opencv/dldt.git cd dldt/inference-engine/ - git checkout 2018_R5 + git checkout 2019_R3.1 git submodule init git submodule update --recursive mkdir build && cd build @@ -228,7 +229,7 @@ if [ "$MODEL_ZOO" == "1" ]; then mkdir -p ~/code && cd ~/code git clone https://github.com/opencv/open_model_zoo.git cd open_model_zoo/demos/ - git checkout 2018_R5 + git checkout 2019_R3.1 mkdir build && cd build cmake -DCMAKE_BUILD_TYPE=Release /opt/openvino_toolkit/dldt/inference-engine make -j8 @@ -240,39 +241,35 @@ fi # Setup LIBREALSENSE if [ "$LIBREALSENSE" == "1" ]; then echo "===================Setting Up LibRealSense...=======================" - echo $ROOT_PASSWD | sudo -S apt-get install -y libssl-dev libusb-1.0-0-dev pkg-config libgtk-3-dev - echo $ROOT_PASSWD | sudo -S apt-get install -y libglfw3-dev libgl1-mesa-dev libglu1-mesa-dev - mkdir -p ~/code && cd ~/code - git clone https://github.com/IntelRealSense/librealsense - cd ~/code/librealsense - git checkout v2.17.1 - mkdir build && cd build - cmake ../ -DBUILD_EXAMPLES=true - echo $ROOT_PASSWD | sudo -S make uninstall - make clean - make - echo $ROOT_PASSWD | sudo -S make install - cd .. - echo $ROOT_PASSWD | sudo -S cp config/99-realsense-libusb.rules /etc/udev/rules.d/ - echo $ROOT_PASSWD | sudo -S udevadm control --reload-rules - udevadm trigger + echo "Install server public key for librealsense" + if [ -n "$http_proxy" ]; then + echo $ROOT_PASSWD | sudo -S apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options http-proxy=$http_proxy --recv-key C8B3A55A6F3EFCDE + else + echo $ROOT_PASSWD | sudo -S apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-key C8B3A55A6F3EFCDE + fi + if ! test "$(grep "http://realsense-hw-public.s3.amazonaws.com/Debian/apt-repo" /etc/apt/sources.list)" + then + echo $ROOT_PASSWD | sudo -S add-apt-repository "deb http://realsense-hw-public.s3.amazonaws.com/Debian/apt-repo bionic main" -u + fi + + echo $ROOT_PASSWD | sudo -S apt update && sudo -S apt-get install -y librealsense2-dkms librealsense2-utils librealsense2-dev echo "==== END install librealsense ====" fi # Setup other dependencies if [ "$OTHER_DEPENDENCY" == "1" ]; then echo "===================Setting UP OTHER_DEPENDENCY DEPENDENCY...=======================" - pip3 install numpy - pip3 install networkx - echo $ROOT_PASSWD | sudo -S apt-get install python3-yaml + echo $ROOT_PASSWD | sudo -S apt-get install -y python3-pip + sudo pip3 install numpy + sudo pip3 install networkx if [ $system_ver = "16.04" ]; then echo $ROOT_PASSWD | sudo -S apt-get install -y --no-install-recommends libboost-all-dev cd /usr/lib/x86_64-linux-gnu - sudo ln -s libboost_python-py35.so libboost_python3.so + echo $ROOT_PASSWD | sudo -S ln -sf libboost_python-py35.so libboost_python3.so elif [ $system_ver = "18.04" ]; then echo $ROOT_PASSWD | sudo -S apt-get install -y --no-install-recommends libboost-all-dev - sudo apt install libboost-python1.62.0 + echo $ROOT_PASSWD | sudo -S apt install -y --no-install-recommends libboost-python1.62.0 fi echo "==== END install other dependencies ====" fi diff --git a/script/environment_setup_binary.sh b/script/environment_setup_binary.sh index d5d6e2cb..afaeedb6 100755 --- a/script/environment_setup_binary.sh +++ b/script/environment_setup_binary.sh @@ -59,13 +59,12 @@ if [ "$CLEAN" == "1" ]; then echo "===================Clean Existing Directories...====================================" read -n1 -p "The clean operation will delete some manually created directories, - including ~/code, ~/catkin_ws, /opt/intel, /opt/openvino_toolkit, and OpenVINO tar ball. + including ~/code, /opt/intel, /opt/openvino_toolkit, and OpenVINO tar ball. Do you want to clean existing directories[Y/N]?" answer case $answer in Y|y) echo echo "===================Cleaning...====================================" echo $ROOT_PASSWD | sudo -S rm -rf ~/code - rm -rf ~/catkin_ws echo $ROOT_PASSWD | sudo -S rm -rf /opt/intel echo $ROOT_PASSWD | sudo -S rm -rf /opt/openvino_toolkit if [[ $system_ver = "16.04" && -L "/usr/lib/x86_64-linux-gnu/libboost_python3.so" ]]; then @@ -80,10 +79,12 @@ fi # Setup ROS from debian if [ "$ROS_DEBIAN" == "1" ]; then echo "===================Installing ROS from Debian Package...=======================" + echo $ROOT_PASSWD | sudo -S apt-get install -y curl + curl http://repo.ros2.org/repos.key | sudo apt-key add - echo $ROOT_PASSWD | sudo -S sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' - #echo $ROOT_PASSWD | sudo -S apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-key 421C365BD9FF1F717815A3895523BAEEB01FA116 - #For those who cannot access hkp protocal - echo $ROOT_PASSWD | curl http://repo.ros2.org/repos.key | sudo apt-key add - + echo $ROOT_PASSWD | sudo -S apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options http-proxy="$http_proxy" --recv-key F42ED6FBAB17C654 + #echo $ROOT_PASSWD | sudo -S apt-key adv --keyserver 'hkp://pgp.mit.edu:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 + #For those who cannot access hkp protocal echo $ROOT_PASSWD | sudo -S apt-get update echo $ROOT_PASSWD | sudo -S apt-get install -y ros-$ros_ver-desktop-full @@ -149,19 +150,19 @@ fi #setup OPENVINO if [ "$OPENVINO" == "1" ]; then cd ~/Downloads - wget -c http://registrationcenter-download.intel.com/akdlm/irc_nas/15078/l_openvino_toolkit_p_2018.5.455.tgz - tar -xvf l_openvino_toolkit_p_2018.5.455.tgz - cd l_openvino_toolkit_p_2018.5.455 - echo $ROOT_PASSWD | sudo -S ./install_cv_sdk_dependencies.sh + wget -c http://registrationcenter-download.intel.com/akdlm/irc_nas/16057/l_openvino_toolkit_p_2019.3.376.tgz + tar -xvf l_openvino_toolkit_p_2019.3.376.tgz + cd l_openvino_toolkit_p_2019.3.376 + echo $ROOT_PASSWD | sudo -S ./install_openvino_dependencies.sh cp $basedir/openvino_silent.cfg . echo $ROOT_PASSWD | sudo -S ./install.sh --silent openvino_silent.cfg set +o errexit - tail ~/.bashrc | grep "computer_vision_sdk/bin/setupvars.sh" + tail ~/.bashrc | grep "openvino/bin/setupvars.sh" set -o errexit if [ "$?" == "1" ]; then - echo "source /opt/intel/computer_vision_sdk/bin/setupvars.sh" >> ~/.bashrc + echo "source /opt/intel/openvino/bin/setupvars.sh" >> ~/.bashrc else echo "openvino already set, skip..." fi @@ -169,7 +170,7 @@ fi #install OPENCL Driver for GPU if [ "$OPENCL" == "1" ]; then - cd /opt/intel/computer_vision_sdk/install_dependencies + cd /opt/intel/openvino/install_dependencies echo $ROOT_PASSWD | sudo -S ./install_NEO_OCL_driver.sh echo "install OPENCL Driver for GPU" fi @@ -177,39 +178,35 @@ fi # Setup LIBREALSENSE if [ "$LIBREALSENSE" == "1" ]; then echo "===================Setting Up LibRealSense...=======================" - echo $ROOT_PASSWD | sudo -S apt-get install -y libssl-dev libusb-1.0-0-dev pkg-config libgtk-3-dev - echo $ROOT_PASSWD | sudo -S apt-get install -y libglfw3-dev libgl1-mesa-dev libglu1-mesa-dev - mkdir -p ~/code && cd ~/code - git clone https://github.com/IntelRealSense/librealsense - cd ~/code/librealsense - git checkout v2.17.1 - mkdir build && cd build - cmake ../ -DBUILD_EXAMPLES=true - echo $ROOT_PASSWD | sudo -S make uninstall - make clean - make - echo $ROOT_PASSWD | sudo -S make install - cd .. - echo $ROOT_PASSWD | sudo -S cp config/99-realsense-libusb.rules /etc/udev/rules.d/ - echo $ROOT_PASSWD | sudo -S udevadm control --reload-rules - udevadm trigger + echo "Install server public key for librealsense" + if [ -n "$http_proxy" ]; then + echo $ROOT_PASSWD | sudo -S apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options http-proxy=$http_proxy --recv-key C8B3A55A6F3EFCDE + else + echo $ROOT_PASSWD | sudo -S apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-key C8B3A55A6F3EFCDE + fi + if ! test "$(grep "http://realsense-hw-public.s3.amazonaws.com/Debian/apt-repo" /etc/apt/sources.list)" + then + echo $ROOT_PASSWD | sudo -S add-apt-repository "deb http://realsense-hw-public.s3.amazonaws.com/Debian/apt-repo bionic main" -u + fi + + echo $ROOT_PASSWD | sudo -S apt update && sudo -S apt-get install -y librealsense2-dkms librealsense2-utils librealsense2-dev echo "==== END install librealsense ====" fi # Setup other dependencies if [ "$OTHER_DEPENDENCY" == "1" ]; then echo "===================Setting UP OTHER_DEPENDENCY DEPENDENCY...=======================" - echo $ROOT_PASSWD | sudo -S apt-get install python3-pip - pip3 install numpy - pip3 install networkx + echo $ROOT_PASSWD | sudo -S apt-get install -y python3-pip + sudo pip3 install numpy + sudo pip3 install networkx if [ $system_ver = "16.04" ]; then echo $ROOT_PASSWD | sudo -S apt-get install -y --no-install-recommends libboost-all-dev cd /usr/lib/x86_64-linux-gnu - sudo ln -s libboost_python-py35.so libboost_python3.so + echo $ROOT_PASSWD | sudo -S ln -sf libboost_python-py35.so libboost_python3.so elif [ $system_ver = "18.04" ]; then echo $ROOT_PASSWD | sudo -S apt-get install -y --no-install-recommends libboost-all-dev - sudo apt install libboost-python1.62.0 + echo $ROOT_PASSWD | sudo -S apt install -y --no-install-recommends libboost-python1.62.0 fi echo "==== END install other dependencies ====" fi diff --git a/script/viewer/pipeTree.py b/script/viewer/pipeTree.py new file mode 100644 index 00000000..d5cd4a36 --- /dev/null +++ b/script/viewer/pipeTree.py @@ -0,0 +1,125 @@ +#!/usr/bin/python + +from __future__ import unicode_literals # at top of module +from __future__ import division, print_function, with_statement +from PyQt5.QtGui import QPainter,QPen,QBrush,QColor +from PyQt5.QtCore import QRect + +class TreeNode(object): + """The basic node of tree structure""" + + def __init__(self, name, parent=None): + super(TreeNode, self).__init__() + self.name = name + self.parent = parent + self.child = {} + + def __repr__(self) : + return 'TreeNode(%s)' % self.name + + + def __contains__(self, item): + return item in self.child + + + def __len__(self): + """return number of children node""" + return len(self.child) + + def __bool__(self, item): + """always return True for exist node""" + return True + + @property + def path(self): + """return path string (from root to current node)""" + if self.parent: + return '%s %s' % (self.parent.path.strip(), self.name) + else: + return self.name + + def get_child(self, name, defval=None): + """get a child node of current node""" + return self.child.get(name, defval) + + def add_child(self, name, obj=None): + """add a child node to current node""" + if obj and not isinstance(obj, TreeNode): + raise ValueError('TreeNode only add another TreeNode obj as child') + if obj is None: + obj = TreeNode(name) + obj.parent = self + self.child[name] = obj + return obj + def del_child(self, name): + """remove a child node from current node""" + if name in self.child: + del self.child[name] + + def find_child(self, path, create=False): + """find child node by path/name, return None if not found""" + # convert path to a list if input is a string + path = path if isinstance(path, list) else path.split() + cur = self + obj = None + for sub in path: + # search + obj = cur.get_child(sub) + if obj is None and create: + # create new node if need + obj = cur.add_child(sub) + # check if search done + if obj is None: + break + cur = obj + return obj + def find_node(self,name): + for name,obj in self.items(): + if name == name: + return obj + return obj.find_node(name) + def items(self): + return self.child.items() + + def dump(self, indent=0): + """dump tree to string""" + tab = ' '*(indent-1) + ' |- ' if indent > 0 else '' + print('%s%s' % (tab, self.name)) + for name, obj in self.items(): + obj.dump(indent+1) + + def dump_graphics(self,convas,name,state,x,y,w,h): + rect_width = 160 + rect_height = 30 + p = QPainter() + if state == '2': + color = QColor(0, 200, 0) + + elif state == '3': + color = QColor(255,255,0) + else: + color = QColor(200, 0, 0) + draw_x = x + (rect_width + 40) + for i,(name,obj) in enumerate(self.items()): + p.begin(convas) + p.setPen(QPen()) + p.setBrush(QBrush(color)) + draw_y = y + i * (rect_height + 10) + rect = QRect(draw_x ,draw_y,rect_width,rect_height) + p.drawRect(rect) + p.drawText(draw_x+2,draw_y + rect_height/2,name) + if self.name != 'root': + p.drawLine(x+rect_width,y+rect_height/2,draw_x,draw_y+rect_height/2) + p.end() + + obj.dump_graphics(convas,name,state,draw_x,draw_y,w,h) + + + def depth(self,depth = 0): + if not self.items(): + return depth + max_depth = 0 + for name,obj in self.items(): + + max_depth = max(max_depth, obj.depth(depth+1)) + return max_depth diff --git a/script/viewer/service.py b/script/viewer/service.py new file mode 100644 index 00000000..a9a475b1 --- /dev/null +++ b/script/viewer/service.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +from pipeline_srv_msgs.srv import * +import rospy +import sys +from pipeTree import TreeNode + +def getMaping(pipeline): + map_dict = dict() + for pipeline in pipeline.connections: + if pipeline.input not in map_dict.keys(): + map_dict[pipeline.input] = list() + map_dict[pipeline.input].append(pipeline.output) + + return map_dict + +def getTreeByMap(parent,input,map): + if input not in map.keys(): + return parent + for output in map[input]: + child = parent.add_child(output) + getTreeByMap(child,output,map) + return parent + + +def getTree(parent,input,pipeline): + map = getMaping(pipeline) + return getTreeByMap(parent,input,map) + +def reqPipelineService(cmd, value): + rospy.wait_for_service('/openvino_toolkit/pipeline/service') + try: + + req = rospy.ServiceProxy('/openvino_toolkit/pipeline/service', PipelineSrv) + req_msg = PipelineSrvRequest() + req_msg.pipeline_request.cmd = cmd + req_msg.pipeline_request.value = value + res = req(req_msg) + return res + except rospy.ServiceException, e: + print "Service call failed: %s"%e + +def usage(): + return "%s "%sys.argv[0] + +if __name__ == "__main__": + if len(sys.argv) == 3: + cmd = str(sys.argv[1]) + value = str(sys.argv[2]) + else: + print usage() + sys.exit(1) + + print("Requesting cmd:%s with value:%s"%(cmd, value)) + response = reqPipelineService(cmd, value) + #print(response) + for pipeline in response.pipelines: + root = getTree(TreeNode(pipeline.name),'',pipeline) + root.dump() + diff --git a/script/viewer/viewer.py b/script/viewer/viewer.py new file mode 100644 index 00000000..93b71918 --- /dev/null +++ b/script/viewer/viewer.py @@ -0,0 +1,115 @@ +import sys +from PyQt5.QtWidgets import QWidget, QDesktopWidget, QApplication,QPushButton,QLabel +from PyQt5.QtGui import QPainter,QPen,QBrush,QColor +from PyQt5.QtCore import QRect +from service import reqPipelineService,getTree +from pipeline_srv_msgs.srv import * +from pipeTree import TreeNode + + + + +class PipelineWidget(object): + def __init__(self,convas,name,x,y): + self.name = name + self.convas = convas + self.name_label = QLabel(self.name,convas) + self.name_label.move(x,y) + self.run_btn = QPushButton('run pipeline ',convas) + self.run_btn.move(x, y+20) + self.run_btn.clicked.connect(lambda:self.onClick(self.run_btn)) + + + + self.pause_btn = QPushButton('pause pipeline',convas) + + self.pause_btn.move(x, y+50) + self.pause_btn.clicked.connect(lambda:self.onClick(self.pause_btn)) + + + self.stop_btn = QPushButton('stop pipeline ',convas) + self.stop_btn.move(x, y+80) + self.stop_btn.clicked.connect(lambda:self.onClick(self.stop_btn)) + + self.refresh() + + def onClick(self, whichbtn): + if whichbtn == self.run_btn: + reqPipelineService("RUN_PIPELINE",self.name) + elif whichbtn == self.pause_btn: + reqPipelineService("PAUSE_PIPELINE",self.name) + elif whichbtn == self.stop_btn: + reqPipelineService("STOP_PIPELINE",self.name) + self.refresh() + + def refresh(self): + response = reqPipelineService("GET_PIPELINE","") + self.convas.response = response + for id,pipeline in enumerate(response.pipelines): + if self.name == pipeline.name: + if pipeline.running_status == '1': + self.run_btn.setEnabled(False) + self.pause_btn.setEnabled(False) + self.stop_btn.setEnabled(False) + elif pipeline.running_status == '2': + self.run_btn.setEnabled(False) + self.pause_btn.setEnabled(True) + self.stop_btn.setEnabled(True) + elif pipeline.running_status == '3': + self.run_btn.setEnabled(True) + self.pause_btn.setEnabled(False) + self.stop_btn.setEnabled(True) + self.convas.update() + + + +class Window(QWidget): + + + def __init__(self,window_width=1000,window_height=800): + super(Window,self).__init__() + self.window_width = window_width + self.window_height = window_height + + self.initWindow() + + self.show() + + def initWindow(self): + #set window param + print('Waiting for pipeling service...') + self.response = reqPipelineService("GET_PIPELINE","") + self.window_height = len(self.response.pipelines) * 200 + self.resize(self.window_width,self.window_height) + self.center() + self.setWindowTitle('Pipeline Viewer') + + for id,pipeline in enumerate(self.response.pipelines): + PipelineWidget(self,pipeline.name,10,id*self.window_height/(len(self.response.pipelines)+1)+30) + + + def paintEvent(self,event): + response = self.response + #response = reqPipelineService("GET_PIPELINE","") + for id,pipeline in enumerate(self.response.pipelines): + pipe_root = getTree(TreeNode('root'),'',pipeline) + pipe_root.dump_graphics(self,pipeline.name,pipeline.running_status,10,id*self.window_height/(len(response.pipelines)+1) + 30,self.window_width,self.window_height) + + + + def center(self): + + + qr = self.frameGeometry() + + cp = QDesktopWidget().availableGeometry().center() + + qr.moveCenter(cp) + self.move(qr.topLeft()) + + +if __name__ == '__main__': + + app = QApplication(sys.argv) + ex = Window() + sys.exit(app.exec_()) \ No newline at end of file diff --git a/vino_launch/CMakeLists.txt b/vino_launch/CMakeLists.txt index b463d711..b6bd5adc 100644 --- a/vino_launch/CMakeLists.txt +++ b/vino_launch/CMakeLists.txt @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 2.8.3) +cmake_minimum_required (VERSION 3.0.2) project(vino_launch) diff --git a/vino_launch/launch/image_object_server_oss.launch b/vino_launch/launch/image_object_server_oss.launch deleted file mode 100644 index a2698e2b..00000000 --- a/vino_launch/launch/image_object_server_oss.launch +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - diff --git a/vino_launch/launch/image_people_server_oss.launch b/vino_launch/launch/image_people_server_oss.launch deleted file mode 100644 index 0d9bcf8b..00000000 --- a/vino_launch/launch/image_people_server_oss.launch +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - diff --git a/vino_launch/launch/image_reidentification_server_oss.launch b/vino_launch/launch/image_reidentification_server_oss.launch deleted file mode 100644 index 3ea12149..00000000 --- a/vino_launch/launch/image_reidentification_server_oss.launch +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - diff --git a/vino_launch/launch/multi_pipeline_service.launch b/vino_launch/launch/multi_pipeline_service.launch new file mode 100644 index 00000000..60e2b180 --- /dev/null +++ b/vino_launch/launch/multi_pipeline_service.launch @@ -0,0 +1,11 @@ + + + + + + + + + + diff --git a/vino_launch/launch/pipeline_reidentification_oss.launch b/vino_launch/launch/pipeline_face_reidentification.launch similarity index 80% rename from vino_launch/launch/pipeline_reidentification_oss.launch rename to vino_launch/launch/pipeline_face_reidentification.launch index eaa393e3..5e98c109 100644 --- a/vino_launch/launch/pipeline_reidentification_oss.launch +++ b/vino_launch/launch/pipeline_face_reidentification.launch @@ -1,11 +1,10 @@ - + - diff --git a/vino_launch/launch/pipeline_object_oss_topic.launch b/vino_launch/launch/pipeline_object_oss_topic.launch deleted file mode 100644 index 5624dfea..00000000 --- a/vino_launch/launch/pipeline_object_oss_topic.launch +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/vino_launch/launch/pipeline_image_oss.launch b/vino_launch/launch/pipeline_people.launch similarity index 97% rename from vino_launch/launch/pipeline_image_oss.launch rename to vino_launch/launch/pipeline_people.launch index 04d43e8f..7f7c016e 100644 --- a/vino_launch/launch/pipeline_image_oss.launch +++ b/vino_launch/launch/pipeline_people.launch @@ -1,5 +1,5 @@ - + - - - - - - - - - - - - - - - diff --git a/vino_launch/launch/pipeline_people_oss.launch b/vino_launch/launch/pipeline_people_oss.launch deleted file mode 100644 index 0d7a2b13..00000000 --- a/vino_launch/launch/pipeline_people_oss.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/vino_launch/launch/pipeline_object_oss.launch b/vino_launch/launch/pipeline_vehicle_detection.launch similarity index 70% rename from vino_launch/launch/pipeline_object_oss.launch rename to vino_launch/launch/pipeline_vehicle_detection.launch index 8f450044..7011a7db 100644 --- a/vino_launch/launch/pipeline_object_oss.launch +++ b/vino_launch/launch/pipeline_vehicle_detection.launch @@ -1,14 +1,13 @@ - + - - + diff --git a/vino_launch/param/image_object_server.yaml b/vino_launch/param/image_object_server.yaml index 2f9e2f27..030cb841 100644 --- a/vino_launch/param/image_object_server.yaml +++ b/vino_launch/param/image_object_server.yaml @@ -3,16 +3,18 @@ Pipelines: inputs: [Image] infers: - name: ObjectDetection - model: /opt/intel/computer_vision_sdk/deployment_tools/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP32/mobilenet-ssd.xml + model: /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP32/mobilenet-ssd.xml engine: CPU label: to/be/set/xxx.labels - batch: 16 + batch: 1 + confidence_threshold: 0.5 + enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame outputs: [RosService] - confidence_threshold: 0.2 connects: - left: Image right: [ObjectDetection] - left: ObjectDetection right: [RosService] - input_path: "/opt/openvino_toolkit/ros_openvino_toolkit/data/images/car.png" -OpenvinoCommon: + input_path: "/home/intel/Pictures/car.png" + +Common: diff --git a/vino_launch/param/image_object_server_oss.yaml b/vino_launch/param/image_object_server_oss.yaml deleted file mode 100644 index 935ea613..00000000 --- a/vino_launch/param/image_object_server_oss.yaml +++ /dev/null @@ -1,18 +0,0 @@ -Pipelines: -- name: object - inputs: [Image] - infers: - - name: ObjectDetection - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP32/mobilenet-ssd.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - outputs: [RosService] - confidence_threshold: 0.2 - connects: - - left: Image - right: [ObjectDetection] - - left: ObjectDetection - right: [RosService] - input_path: "/opt/openvino_toolkit/ros_openvino_toolkit/data/images/car.png" -OpenvinoCommon: diff --git a/vino_launch/param/image_people_server.yaml b/vino_launch/param/image_people_server.yaml index 2705ba63..578ec311 100644 --- a/vino_launch/param/image_people_server.yaml +++ b/vino_launch/param/image_people_server.yaml @@ -3,37 +3,39 @@ Pipelines: inputs: [Image] infers: - name: FaceDetection - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-detection-adas-0001/FP32/face-detection-adas-0001.xml + model: /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/face-detection-adas-0001.xml engine: CPU - label: to/be/set/xxx.labels + label: /to/be/set/xxx.labels batch: 1 + confidence_threshold: 0.5 + enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame - name: AgeGenderRecognition - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.xml + model: /opt/openvino_toolkit/models/age-gender-recognition/output/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.xml engine: CPU label: to/be/set/xxx.labels batch: 16 - name: EmotionRecognition - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.xml + model: /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.xml engine: CPU label: to/be/set/xxx.labels batch: 16 - name: HeadPoseEstimation - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/head-pose-estimation-adas-0001/FP32/head-pose-estimation-adas-0001.xml + model: /opt/openvino_toolkit/models/head-pose-estimation/output/intel/head-pose-estimation-adas-0001/FP32/head-pose-estimation-adas-0001.xml engine: CPU label: to/be/set/xxx.labels batch: 16 - outputs: [RosService] - confidence_threshold: 0.2 + outputs: [RosService, RViz] connects: - left: Image right: [FaceDetection] - left: FaceDetection - right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, RosService] + right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, RosService, RViz] - left: AgeGenderRecognition - right: [RosService] + right: [RosService, RViz] - left: EmotionRecognition - right: [RosService] + right: [RosService, RViz] - left: HeadPoseEstimation - right: [RosService] - input_path: "/opt/openvino_toolkit/ros_openvino_toolkit/data/images/team.jpg" -OpenvinoCommon: + right: [RosService, RViz] + input_path: "~/Pictures/face.jpeg" + +Common: diff --git a/vino_launch/param/image_people_server_oss.yaml b/vino_launch/param/image_people_server_oss.yaml deleted file mode 100644 index 3e0a12ca..00000000 --- a/vino_launch/param/image_people_server_oss.yaml +++ /dev/null @@ -1,39 +0,0 @@ -Pipelines: -- name: people - inputs: [Image] - infers: - - name: FaceDetection - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Transportation/object_detection/face/pruned_mobilenet_reduced_ssd_shared_weights/dldt/face-detection-adas-0001.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 1 - - name: AgeGenderRecognition - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_attributes/age_gender/dldt/age-gender-recognition-retail-0013.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - - name: EmotionRecognition - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_attributes/emotions_recognition/0003/dldt/emotions-recognition-retail-0003.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - - name: HeadPoseEstimation - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Transportation/object_attributes/headpose/vanilla_cnn/dldt/head-pose-estimation-adas-0001.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - outputs: [RosService] - confidence_threshold: 0.2 - connects: - - left: Image - right: [FaceDetection] - - left: FaceDetection - right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, RosService] - - left: AgeGenderRecognition - right: [RosService] - - left: EmotionRecognition - right: [RosService] - - left: HeadPoseEstimation - right: [RosService] - input_path: "/opt/openvino_toolkit/ros_openvino_toolkit/data/images/team.jpg" -OpenvinoCommon: diff --git a/vino_launch/param/image_segmentation_server.yaml b/vino_launch/param/image_segmentation_server.yaml index f0e0c16b..44dcd41c 100644 --- a/vino_launch/param/image_segmentation_server.yaml +++ b/vino_launch/param/image_segmentation_server.yaml @@ -3,7 +3,7 @@ Pipelines: inputs: [Image] infers: - name: ObjectSegmentation - model: /opt/models/mask_rcnn_inception_v2_coco_2018_01_28/output/frozen_inference_graph.xml + model: /opt/openvino_toolkit/models/segmentation/output/FP32/frozen_inference_graph.xml engine: CPU label: to/be/set/xxx.labels batch: 1 diff --git a/vino_launch/param/multi_pipleine_service_object.yaml b/vino_launch/param/multi_pipleine_service_object.yaml new file mode 100644 index 00000000..06d2059d --- /dev/null +++ b/vino_launch/param/multi_pipleine_service_object.yaml @@ -0,0 +1,46 @@ +Pipelines: + +- name: pipe1 + inputs: [StandardCamera] + infers: + - name: ObjectDetection + model: /opt/openvino_toolkit/open_model_zoo/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP32/mobilenet-ssd.xml + engine: CPU + label: to/be/set/xxx.labels + batch: 16 + outputs: [ImageWindow, RosTopic, RViz] + confidence_threshold: 0.5 + connects: + - left: StandardCamera + right: [ObjectDetection] + - left: ObjectDetection + right: [ImageWindow] + - left: ObjectDetection + right: [RosTopic] + - left: ObjectDetection + right: [RViz] + + +- name: pipe2 + inputs: [RealSenseCamera] + infers: + - name: ObjectDetection + model: /opt/openvino_toolkit/open_model_zoo/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP32/mobilenet-ssd.xml + engine: CPU + label: to/be/set/xxx.labels + batch: 16 + outputs: [ImageWindow, RosTopic, RViz] + confidence_threshold: 0.5 + connects: + - left: RealSenseCamera + right: [ObjectDetection] + - left: ObjectDetection + right: [ImageWindow] + - left: ObjectDetection + right: [RosTopic] + - left: ObjectDetection + right: [RViz] + + + +OpenvinoCommon: diff --git a/vino_launch/param/pipeline_face_reidentification.yaml b/vino_launch/param/pipeline_face_reidentification.yaml new file mode 100644 index 00000000..dba0ba56 --- /dev/null +++ b/vino_launch/param/pipeline_face_reidentification.yaml @@ -0,0 +1,36 @@ +Pipelines: +- name: people + inputs: [RealSenseCamera] + infers: + - name: FaceDetection + model: /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/face-detection-adas-0001.xml + engine: CPU + label: /opt/intel/openvino/deployment_tools/tools/model_downloader/Transportation/object_detection/face/pruned_mobilenet_reduced_ssd_shared_weights/dldt/face-detection-adas-0001.labels + batch: 1 + confidence_threshold: 0.5 + enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame + - name: LandmarksDetection + model: /opt/openvino_toolkit/models/landmarks-regression/output/intel/landmarks-regression-retail-0009/FP32/landmarks-regression-retail-0009.xml + engine: CPU + label: to/be/set/xxx.labels + batch: 1 + - name: FaceReidentification + model: /opt/openvino_toolkit/models/face-reidentification/output/intel/face-reidentification-retail-0095/FP32/face-reidentification-retail-0095.xml + engine: CPU + label: to/be/set/xxx.labels + batch: 1 + confidence_threshold: 0.9 + outputs: [ImageWindow, RosTopic, RViz] + connects: + - left: RealSenseCamera + right: [FaceDetection] + - left: FaceDetection + right: [LandmarksDetection, FaceReidentification] + - left: FaceDetection + right: [ImageWindow, RosTopic, RViz] + - left: LandmarksDetection + right: [ImageWindow, RosTopic, RViz] + - left: FaceReidentification + right: [ImageWindow, RosTopic, RViz] + +Common: diff --git a/vino_launch/param/pipeline_image.yaml b/vino_launch/param/pipeline_image.yaml index 95c21b6a..e5a93587 100644 --- a/vino_launch/param/pipeline_image.yaml +++ b/vino_launch/param/pipeline_image.yaml @@ -4,32 +4,33 @@ Pipelines: input_path: /opt/openvino_toolkit/ros_openvino_toolkit/data/images/team.jpg infers: - name: FaceDetection - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-detection-adas-0001/FP32/face-detection-adas-0001.xml + model: /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/face-detection-adas-0001.xml engine: CPU label: to/be/set/xxx.labels batch: 1 + confidence_threshold: 0.5 + enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame - name: AgeGenderRecognition - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.xml + model: /opt/openvino_toolkit/models/age-gender-recognition/output/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.xml engine: CPU label: to/be/set/xxx.labels batch: 16 - name: EmotionRecognition - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.xml + model: /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.xml engine: CPU label: to/be/set/xxx.labels batch: 16 - name: HeadPoseEstimation - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/head-pose-estimation-adas-0001/FP32/head-pose-estimation-adas-0001.xml + model: /opt/openvino_toolkit/models/head-pose-estimation/output/intel/head-pose-estimation-adas-0001/FP32/head-pose-estimation-adas-0001.xml engine: CPU label: to/be/set/xxx.labels batch: 16 outputs: [ImageWindow, RosTopic, RViz] - confidence_threshold: 0.2 connects: - left: Image right: [FaceDetection] - left: FaceDetection - right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, ImageWindow, RosTopic RViz] + right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, ImageWindow, RosTopic, RViz] - left: AgeGenderRecognition right: [ImageWindow, RosTopic, RViz] - left: EmotionRecognition @@ -37,4 +38,4 @@ Pipelines: - left: HeadPoseEstimation right: [ImageWindow, RosTopic, RViz] -OpenvinoCommon: +Common: diff --git a/vino_launch/param/pipeline_image_oss.yaml b/vino_launch/param/pipeline_image_oss.yaml deleted file mode 100644 index 5c87fcb8..00000000 --- a/vino_launch/param/pipeline_image_oss.yaml +++ /dev/null @@ -1,40 +0,0 @@ -Pipelines: -- name: people - inputs: [Image] - input_path: /opt/openvino_toolkit/ros_openvino_toolkit/data/images/team.jpg - infers: - - name: FaceDetection - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Transportation/object_detection/face/pruned_mobilenet_reduced_ssd_shared_weights/dldt/face-detection-adas-0001.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 1 - - name: AgeGenderRecognition - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_attributes/age_gender/dldt/age-gender-recognition-retail-0013.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - - name: EmotionRecognition - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_attributes/emotions_recognition/0003/dldt/emotions-recognition-retail-0003.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - - name: HeadPoseEstimation - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Transportation/object_attributes/headpose/vanilla_cnn/dldt/head-pose-estimation-adas-0001.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - outputs: [ImageWindow, RosTopic,RViz] - confidence_threshold: 0.2 - connects: - - left: Image - right: [FaceDetection] - - left: FaceDetection - right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, ImageWindow, RosTopic, Rviz] - - left: AgeGenderRecognition - right: [ImageWindow, RosTopic,RViz] - - left: EmotionRecognition - right: [ImageWindow, RosTopic,RViz] - - left: HeadPoseEstimation - right: [ImageWindow, RosTopic,RViz] - -OpenvinoCommon: diff --git a/vino_launch/param/pipeline_object.yaml b/vino_launch/param/pipeline_object.yaml index d4f5fffe..b6077d1f 100644 --- a/vino_launch/param/pipeline_object.yaml +++ b/vino_launch/param/pipeline_object.yaml @@ -3,12 +3,13 @@ Pipelines: inputs: [RealSenseCamera] infers: - name: ObjectDetection - model: /opt/intel/computer_vision_sdk/deployment_tools/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP16/mobilenet-ssd.xml - engine: GPU + model: /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP16/mobilenet-ssd.xml + engine: MYRIAD label: to/be/set/xxx.labels - batch: 16 + batch: 1 + confidence_threshold: 0.5 + enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame outputs: [ImageWindow, RosTopic, RViz] - confidence_threshold: 0.5 connects: - left: RealSenseCamera right: [ObjectDetection] diff --git a/vino_launch/param/pipeline_object_oss.yaml b/vino_launch/param/pipeline_object_oss.yaml deleted file mode 100644 index 4fe4ca7a..00000000 --- a/vino_launch/param/pipeline_object_oss.yaml +++ /dev/null @@ -1,22 +0,0 @@ -Pipelines: -- name: object - inputs: [RealSenseCamera] - infers: - - name: ObjectDetection - model: /opt/intel/computer_vision_sdk/deployment_tools/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP16/mobilenet-ssd.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - outputs: [ImageWindow, RosTopic, RViz] - confidence_threshold: 0.5 - connects: - - left: RealSenseCamera - right: [ObjectDetection] - - left: ObjectDetection - right: [ImageWindow] - - left: ObjectDetection - right: [RosTopic] - - left: ObjectDetection - right: [RViz] - -OpenvinoCommon: diff --git a/vino_launch/param/pipeline_object_oss_topic.yaml b/vino_launch/param/pipeline_object_oss_topic.yaml deleted file mode 100644 index a29b85d1..00000000 --- a/vino_launch/param/pipeline_object_oss_topic.yaml +++ /dev/null @@ -1,22 +0,0 @@ -Pipelines: -- name: object - inputs: [RealSenseCameraTopic] - infers: - - name: ObjectDetection - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP32/mobilenet-ssd.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - outputs: [ImageWindow, RosTopic, RViz] - confidence_threshold: 0.5 - connects: - - left: RealSenseCameraTopic - right: [ObjectDetection] - - left: ObjectDetection - right: [ImageWindow] - - left: ObjectDetection - right: [RosTopic] - - left: ObjectDetection - right: [RViz] - -OpenvinoCommon: diff --git a/vino_launch/param/pipeline_object_topic.yaml b/vino_launch/param/pipeline_object_topic.yaml index 8fb74e10..67ec2a90 100644 --- a/vino_launch/param/pipeline_object_topic.yaml +++ b/vino_launch/param/pipeline_object_topic.yaml @@ -3,12 +3,13 @@ Pipelines: inputs: [RealSenseCameraTopic] infers: - name: ObjectDetection - model: /opt/intel/computer_vision_sdk/deployment_tools/model_downloader/object_detection/common/mobilenet-ssd/caffe/output/FP16/mobilenet-ssd.xml + model: /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP16/mobilenet-ssd.xml engine: MYRIAD label: to/be/set/xxx.labels - batch: 16 + batch: 1 + confidence_threshold: 0.5 + enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame outputs: [ImageWindow, RosTopic, RViz] - confidence_threshold: 0.5 connects: - left: RealSenseCameraTopic right: [ObjectDetection] diff --git a/vino_launch/param/pipeline_people.yaml b/vino_launch/param/pipeline_people.yaml new file mode 100644 index 00000000..21d09b54 --- /dev/null +++ b/vino_launch/param/pipeline_people.yaml @@ -0,0 +1,40 @@ +Pipelines: +- name: people + inputs: [RealSenseCamera] + infers: + - name: FaceDetection + model: /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/face-detection-adas-0001.xml + engine: MYRIAD + label: /to/be/set/xxx.labels + batch: 1 + confidence_threshold: 0.5 + enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame + - name: AgeGenderRecognition + model: /opt/openvino_toolkit/models/age-gender-recognition/output/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.xml + engine: CPU + label: to/be/set/xxx.labels + batch: 16 + - name: EmotionRecognition + model: /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.xml + engine: CPU + label: /opt/openvino_toolkit/models/emotions-recognition/output/FP32/Retail/object_attributes/emotions_recognition/0003/dldt/emotions-recognition-retail-0003.labels + batch: 16 + - name: HeadPoseEstimation + model: /opt/openvino_toolkit/models/head-pose-estimation/output/intel/head-pose-estimation-adas-0001/FP32/head-pose-estimation-adas-0001.xml + engine: CPU + label: to/be/set/xxx.labels + batch: 16 + outputs: [ImageWindow, RosTopic, RViz] + connects: + - left: RealSenseCamera + right: [FaceDetection] + - left: FaceDetection + right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, ImageWindow, RosTopic, RViz] + - left: AgeGenderRecognition + right: [ImageWindow, RosTopic, RViz] + - left: EmotionRecognition + right: [ImageWindow, RosTopic, RViz] + - left: HeadPoseEstimation + right: [ImageWindow, RosTopic, RViz] + +Common: diff --git a/vino_launch/param/pipeline_people_myriad.yaml b/vino_launch/param/pipeline_people_myriad.yaml deleted file mode 100644 index 30dc280a..00000000 --- a/vino_launch/param/pipeline_people_myriad.yaml +++ /dev/null @@ -1,39 +0,0 @@ -Pipelines: -- name: people - inputs: [StandardCamera] - infers: - - name: FaceDetection - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-detection-adas-0001/FP16/face-detection-adas-0001.xml - engine: MYRIAD - label: to/be/set/xxx.labels - batch: 1 - - name: AgeGenderRecognition - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013.xml - engine: MYRIAD - label: to/be/set/xxx.labels - batch: 16 - - name: EmotionRecognition - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/emotions-recognition-retail-0003/FP16/emotions-recognition-retail-0003.xml - engine: MYRIAD - label: to/be/set/xxx.labels - batch: 16 - - name: HeadPoseEstimation - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/head-pose-estimation-adas-0001/FP16/head-pose-estimation-adas-0001.xml - engine: MYRIAD - label: to/be/set/xxx.labels - batch: 16 - outputs: [ImageWindow, RosTopic,RViz] - confidence_threshold: 0.2 - connects: - - left: StandardCamera - right: [FaceDetection] - - left: FaceDetection - right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, ImageWindow, RosTopic, RViz] - - left: AgeGenderRecognition - right: [ImageWindow, RosTopic,RViz] - - left: EmotionRecognition - right: [ImageWindow, RosTopic,RViz] - - left: HeadPoseEstimation - right: [ImageWindow, RosTopic,RViz] - -OpenvinoCommon: diff --git a/vino_launch/param/pipeline_people_oss.yaml b/vino_launch/param/pipeline_people_oss.yaml deleted file mode 100644 index dd000e34..00000000 --- a/vino_launch/param/pipeline_people_oss.yaml +++ /dev/null @@ -1,39 +0,0 @@ -Pipelines: -- name: people - inputs: [StandardCamera] - infers: - - name: FaceDetection - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Transportation/object_detection/face/pruned_mobilenet_reduced_ssd_shared_weights/dldt/face-detection-adas-0001.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 1 - - name: AgeGenderRecognition - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_attributes/age_gender/dldt/age-gender-recognition-retail-0013.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - - name: EmotionRecognition - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_attributes/emotions_recognition/0003/dldt/emotions-recognition-retail-0003.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - - name: HeadPoseEstimation - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Transportation/object_attributes/headpose/vanilla_cnn/dldt/head-pose-estimation-adas-0001.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - outputs: [ImageWindow, RosTopic,RViz] - confidence_threshold: 0.2 - connects: - - left: StandardCamera - right: [FaceDetection] - - left: FaceDetection - right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, ImageWindow, RosTopic, Rviz] - - left: AgeGenderRecognition - right: [ImageWindow, RosTopic,RViz] - - left: EmotionRecognition - right: [ImageWindow, RosTopic,RViz] - - left: HeadPoseEstimation - right: [ImageWindow, RosTopic,RViz] - -OpenvinoCommon: diff --git a/vino_launch/param/pipeline_reidentification.yaml b/vino_launch/param/pipeline_reidentification.yaml index bc0db8a0..7f3cbe46 100644 --- a/vino_launch/param/pipeline_reidentification.yaml +++ b/vino_launch/param/pipeline_reidentification.yaml @@ -3,25 +3,25 @@ Pipelines: inputs: [StandardCamera] infers: - name: ObjectDetection - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/person-detection-retail-0013/FP32/person-detection-retail-0013.xml + model: /opt/openvino_toolkit/models/person-detection/output/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml engine: CPU label: to/be/set/xxx.labels batch: 1 confidence_threshold: 0.5 enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame - name: PersonReidentification - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/person-reidentification-retail-0076/FP32/person-reidentification-retail-0076.xml + model: /opt/openvino_toolkit/models/person-reidentification/output/intel/person-reidentification-retail-0076/FP32/person-reidentification-retail-0076.xml engine: CPU label: to/be/set/xxx.labels batch: 1 confidence_threshold: 0.7 - outputs: [ImageWindow, RosTopic, RViz] + outputs: [ImageWindow, RViz, RosTopic] connects: - left: StandardCamera right: [ObjectDetection] - left: ObjectDetection right: [PersonReidentification] - left: PersonReidentification - right: [ImageWindow, RosTopic, RViz] + right: [ImageWindow, RViz, RosTopic] OpenvinoCommon: diff --git a/vino_launch/param/pipeline_reidentification_oss.yaml b/vino_launch/param/pipeline_reidentification_oss.yaml deleted file mode 100644 index 98837f9f..00000000 --- a/vino_launch/param/pipeline_reidentification_oss.yaml +++ /dev/null @@ -1,27 +0,0 @@ -Pipelines: -- name: object - inputs: [StandardCamera] - infers: - - name: ObjectDetection - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_detection/pedestrian/rmnet_ssd/0013/dldt/person-detection-retail-0013.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 1 - confidence_threshold: 0.5 - enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame - - name: PersonReidentification - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_reidentification/pedestrian/rmnet_based/0076/dldt/person-reidentification-retail-0076.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 1 - confidence_threshold: 0.7 - outputs: [ImageWindow, RosTopic, RViz] - connects: - - left: StandardCamera - right: [ObjectDetection] - - left: ObjectDetection - right: [PersonReidentification] - - left: PersonReidentification - right: [ImageWindow, RosTopic, RViz] - -OpenvinoCommon: diff --git a/vino_launch/param/pipeline_reidentification_server_oss.yaml b/vino_launch/param/pipeline_reidentification_server_oss.yaml deleted file mode 100644 index 43f00d13..00000000 --- a/vino_launch/param/pipeline_reidentification_server_oss.yaml +++ /dev/null @@ -1,28 +0,0 @@ -Pipelines: -- name: object - inputs: [Image] - infers: - - name: ObjectDetection - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_detection/pedestrian/rmnet_ssd/0013/dldt/person-detection-retail-0013.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 1 - confidence_threshold: 0.5 - enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame - - name: PersonReidentification - model: /opt/openvino_toolkit/open_model_zoo/model_downloader/Retail/object_reidentification/pedestrian/rmnet_based/0076/dldt/person-reidentification-retail-0076.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 1 - confidence_threshold: 0.7 - outputs: [RosService] - connects: - - left: Image - right: [ObjectDetection] - - left: ObjectDetection - right: [PersonReidentification] - - left: PersonReidentification - right: [RosService] - input_path: "/opt/openvino_toolkit/ros_openvino_toolkit/data/images/team.jpg" - -OpenvinoCommon: \ No newline at end of file diff --git a/vino_launch/param/pipeline_segmentation.yaml b/vino_launch/param/pipeline_segmentation.yaml index e954d8e3..fd4a7851 100644 --- a/vino_launch/param/pipeline_segmentation.yaml +++ b/vino_launch/param/pipeline_segmentation.yaml @@ -3,12 +3,12 @@ Pipelines: inputs: [RealSenseCameraTopic] infers: - name: ObjectSegmentation - model: /opt/models/mask_rcnn_inception_v2_coco_2018_01_28/output/frozen_inference_graph.xml - engine: CPU + model: /opt/openvino_toolkit/models/segmentation/output/FP16/frozen_inference_graph.xml + engine: "HETERO:CPU,GPU,MYRIAD" label: to/be/set/xxx.labels batch: 1 + confidence_threshold: 0.5 outputs: [ImageWindow, RosTopic, RViz] - confidence_threshold: 0.2 connects: - left: RealSenseCameraTopic right: [ObjectSegmentation] @@ -20,4 +20,3 @@ Pipelines: right: [RViz] OpenvinoCommon: - diff --git a/vino_launch/param/pipeline_vehicle_detection.yaml b/vino_launch/param/pipeline_vehicle_detection.yaml new file mode 100644 index 00000000..81c80193 --- /dev/null +++ b/vino_launch/param/pipeline_vehicle_detection.yaml @@ -0,0 +1,35 @@ +Pipelines: +- name: object + inputs: [RealSenseCamera] + infers: + - name: ObjectDetection + model: /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/intel/vehicle-license-plate-detection-barrier-0106/FP32/vehicle-license-plate-detection-barrier-0106.xml + engine: CPU + model_type: SSD + label: /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/Security/object_detection/barrier/0106/dldt/vehicle-license-plate-detection-barrier-0106.labels + batch: 1 + enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame + - name: VehicleAttribsDetection + model: /opt/openvino_toolkit/models/vehicle-attributes-recongnition/output/intel/vehicle-attributes-recognition-barrier-0039/FP32/vehicle-attributes-recognition-barrier-0039.xml + engine: CPU + label: to/be/set/xxx.labels + batch: 1 + - name: LicensePlateDetection + model: /opt/openvino_toolkit/models/license-plate-recognition/output/intel/license-plate-recognition-barrier-0001/FP32/license-plate-recognition-barrier-0001.xml + engine: CPU + label: to/be/set/xxx.labels + batch: 1 + outputs: [ImageWindow, RViz, RosTopic] + connects: + - left: RealSenseCamera + right: [ObjectDetection] + - left: ObjectDetection + right: [{VehicleAttribsDetection: label == vehicle && confidence >= 0.8}, {LicensePlateDetection: label == license && confidence >= 0.8}] + - left: ObjectDetection + right: [ImageWindow, RosTopic, RViz] + - left: VehicleAttribsDetection + right: [ImageWindow, RosTopic, RViz] + - left: LicensePlateDetection + right: [ImageWindow, RosTopic, RViz] + +OpenvinoCommon: \ No newline at end of file diff --git a/vino_launch/param/pipeline_video.yaml b/vino_launch/param/pipeline_video.yaml index bac3f74d..fa632bbb 100644 --- a/vino_launch/param/pipeline_video.yaml +++ b/vino_launch/param/pipeline_video.yaml @@ -1,20 +1,26 @@ Pipelines: - name: segmentation inputs: [Video] - input_path: /home/intel/Videos/2018-09-26-101439.webm + input_path: /home/iei/Videos/object_segmentation.MP4 infers: - name: ObjectSegmentation - model: /opt/models/mask_rcnn_inception_v2_coco_2018_01_28/output/frozen_inference_graph.xml + model: /opt/openvino_toolkit/models/segmentation/output/FP32/frozen_inference_graph.xml + #model: /opt/models/mask_rcnn_inception_v2_coco_2018_01_28/output/frozen_inference_graph.xml engine: CPU label: to/be/set/xxx.labels batch: 1 + confidence_threshold: 0.5 outputs: [ImageWindow, RosTopic, RViz] - confidence_threshold: 0.2 connects: - left: Video right: [ObjectSegmentation] - left: ObjectSegmentation - right: [ImageWindow, RosTopic, RViz] + right: [ImageWindow] + - left: ObjectSegmentation + right: [RosTopic] + - left: ObjectSegmentation + right: [RViz] -OpenvinoCommon: + +OpenvinoCommon: diff --git a/vino_param_lib/CMakeLists.txt b/vino_param_lib/CMakeLists.txt index a0c5c2d8..bcda4545 100644 --- a/vino_param_lib/CMakeLists.txt +++ b/vino_param_lib/CMakeLists.txt @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 2.8.3) +cmake_minimum_required(VERSION 3.0.2) project(vino_param_lib) find_package(catkin REQUIRED COMPONENTS diff --git a/vino_param_lib/include/vino_param_lib/param_manager.h b/vino_param_lib/include/vino_param_lib/param_manager.h index 1f0a5cc3..74c29c3d 100644 --- a/vino_param_lib/include/vino_param_lib/param_manager.h +++ b/vino_param_lib/include/vino_param_lib/param_manager.h @@ -1,18 +1,17 @@ -/* - * Copyright (c) 2018 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /** * @brief A header file with declaration for parameter management * @file param_manager.hpp @@ -43,7 +42,7 @@ namespace Params */ class ParamManager // singleton { - public: +public: /** * @brief Get the singleton instance of ParamManager class. * The instance will be created when first call. @@ -61,26 +60,37 @@ class ParamManager // singleton */ void print() const; - struct InferenceParams + struct InferenceRawData { std::string name; std::string engine; std::string model; + std::string model_type; std::string label; int batch; float confidence_threshold = 0.5; bool enable_roi_constraint = false; }; - struct PipelineParams + + struct FilterRawData + { + std::string input; + std::string output; + std::string filter_conditions; + }; + + struct PipelineRawData { std::string name; - std::vector infers; + std::vector infers; std::vector inputs; std::vector outputs; std::multimap connects; std::string input_meta; + std::vector filters; }; - struct CommonParams + + struct CommonRawData { std::string custom_cpu_library; std::string custom_cldnn_library; @@ -104,9 +114,9 @@ class ParamManager // singleton /** * @brief Retrieve pipeline parameters. - * @return A list of struct PipelineParams storing all pipeline parameters. + * @return A list of struct PipelineRawData storing all pipeline parameters. */ - std::vector getPipelines() const + std::vector getPipelines() const { return pipelines_; } @@ -117,27 +127,27 @@ class ParamManager // singleton * @param[in] name: the name of the pipeline to be retrieved. * @return The pipeline paratmeters, or throw a loginc error. */ - PipelineParams getPipeline(const std::string& name) const; + PipelineRawData getPipeline(const std::string& name) const; /** * @brief Retrieve common parameters. - * @return struct CommonParams storing all common parameters. + * @return struct CommonRawData storing all common parameters. */ - CommonParams getCommon() const + CommonRawData getCommon() const { return common_; } - private: +private: ParamManager() { } ParamManager(ParamManager const&); void operator=(ParamManager const&); - std::vector pipelines_; - CommonParams common_; + std::vector pipelines_; + CommonRawData common_; }; } // namespace Params -#endif // VINO_PARAM_LIB_PARAM_MANAGER_H +#endif // VINO_PARAM_LIB_PARAM_MANAGER_H \ No newline at end of file diff --git a/vino_param_lib/include/vino_param_lib/slog.h b/vino_param_lib/include/vino_param_lib/slog.h index 0b2de083..3838e6d8 100644 --- a/vino_param_lib/include/vino_param_lib/slog.h +++ b/vino_param_lib/include/vino_param_lib/slog.h @@ -49,13 +49,12 @@ class LogStream std::ostream* _log_stream; bool _new_line; - public: +public: /** * @brief A constructor. Creates an LogStream object * @param prefix The prefix to print */ - LogStream(const std::string& prefix, std::ostream& log_stream) - : _prefix(prefix), _new_line(true) + LogStream(const std::string& prefix, std::ostream& log_stream) : _prefix(prefix), _new_line(true) { _log_stream = &log_stream; } diff --git a/vino_param_lib/package.xml b/vino_param_lib/package.xml index 7023ef36..5c1d6ead 100644 --- a/vino_param_lib/package.xml +++ b/vino_param_lib/package.xml @@ -19,7 +19,7 @@ limitations under the License. vino_param_lib 0.3.0 - Library for ROS2 OpenVINO parameter management + Library for ROS OpenVINO parameter management Weizhi Liu Chao Li Hongkun Chen diff --git a/vino_param_lib/src/param_manager.cpp b/vino_param_lib/src/param_manager.cpp index 3d5d2bb6..d9695477 100644 --- a/vino_param_lib/src/param_manager.cpp +++ b/vino_param_lib/src/param_manager.cpp @@ -1,18 +1,16 @@ -/* - * Copyright (c) 2018 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "vino_param_lib/param_manager.h" #include @@ -25,46 +23,43 @@ namespace Params { - -void operator>>(const YAML::Node& node, ParamManager::PipelineParams& pipeline); -void operator>>(const YAML::Node& node, - std::vector& list); -void operator>>(const YAML::Node& node, ParamManager::InferenceParams& infer); +void operator>>(const YAML::Node& node, ParamManager::PipelineRawData& pipeline); +void operator>>(const YAML::Node& node, std::vector& list); +void operator>>(const YAML::Node& node, ParamManager::InferenceRawData& infer); void operator>>(const YAML::Node& node, std::vector& list); -void operator>>(const YAML::Node& node, - std::multimap& connect); +void operator>>(const YAML::Node& node, std::multimap& connect); +void operator>>(const YAML::Node& node, std::vector& filters); void operator>>(const YAML::Node& node, std::string& str); void operator>>(const YAML::Node& node, bool& val); void operator>>(const YAML::Node& node, int& val); void operator>>(const YAML::Node& node, float& val); -void operator>>(const YAML::Node& node, ParamManager::CommonParams& common); -#define YAML_PARSE(node, key, val) \ - try \ - { \ - node[key] >> val; \ - } \ - catch (YAML::Exception e) \ - { \ - slog::warn << e.msg << slog::endl; \ - } \ - catch (...) \ - { \ - slog::warn << "Exception occurs when parsing string." << slog::endl; \ +void operator>>(const YAML::Node& node, ParamManager::CommonRawData& common); +#define YAML_PARSE(node, key, val) \ + try \ + { \ + node[key] >> val; \ + } \ + catch (YAML::Exception e) \ + { \ + slog::warn << e.msg << slog::endl; \ + } \ + catch (...) \ + { \ + slog::warn << "Exception occurs when parsing string." << slog::endl; \ } -void operator>>(const YAML::Node& node, - std::vector& list) +void operator>>(const YAML::Node& node, std::vector& list) { slog::info << "Pipeline size: " << node.size() << slog::endl; for (unsigned i = 0; i < node.size(); i++) { - ParamManager::PipelineParams temp; + ParamManager::PipelineRawData temp; node[i] >> temp; list.push_back(temp); } } -void operator>>(const YAML::Node& node, ParamManager::CommonParams& common) +void operator>>(const YAML::Node& node, ParamManager::CommonRawData& common) { YAML_PARSE(node, "camera_topic", common.camera_topic) YAML_PARSE(node, "custom_cpu_library", common.custom_cpu_library) @@ -72,34 +67,34 @@ void operator>>(const YAML::Node& node, ParamManager::CommonParams& common) YAML_PARSE(node, "enable_performance_count", common.enable_performance_count) } -void operator>>(const YAML::Node& node, - ParamManager::PipelineParams& pipeline) +void operator>>(const YAML::Node& node, ParamManager::PipelineRawData& pipeline) { YAML_PARSE(node, "name", pipeline.name) YAML_PARSE(node, "inputs", pipeline.inputs) YAML_PARSE(node, "infers", pipeline.infers) YAML_PARSE(node, "outputs", pipeline.outputs) YAML_PARSE(node, "connects", pipeline.connects) + YAML_PARSE(node, "connects", pipeline.filters) YAML_PARSE(node, "input_path", pipeline.input_meta) slog::info << "Pipeline Params:name=" << pipeline.name << slog::endl; } -void operator>>(const YAML::Node& node, - std::vector& list) +void operator>>(const YAML::Node& node, std::vector& list) { slog::info << "Inferences size: " << node.size() << slog::endl; for (unsigned i = 0; i < node.size(); i++) { - ParamManager::InferenceParams temp_inf; + ParamManager::InferenceRawData temp_inf; node[i] >> temp_inf; list.push_back(temp_inf); } } -void operator>>(const YAML::Node& node, ParamManager::InferenceParams& infer) +void operator>>(const YAML::Node& node, ParamManager::InferenceRawData& infer) { YAML_PARSE(node, "name", infer.name) YAML_PARSE(node, "model", infer.model) + YAML_PARSE(node, "model_type", infer.model_type) YAML_PARSE(node, "engine", infer.engine) YAML_PARSE(node, "label", infer.label) YAML_PARSE(node, "batch", infer.batch) @@ -118,18 +113,46 @@ void operator>>(const YAML::Node& node, std::vector& list) } } -void operator>>(const YAML::Node& node, - std::multimap& connect) +void operator>>(const YAML::Node& node, std::multimap& connect) +{ + for (unsigned i = 0; i < node.size(); i++) + { + std::string left; + node[i]["left"] >> left; + YAML::Node rights = node[i]["right"]; + for (unsigned i = 0; i < rights.size(); i++) + { + std::string right; + if (rights[i].Type() == YAML::NodeType::Map) + { + rights[i].begin()->first >> right; + } + else + { + rights[i] >> right; + } + connect.insert({ left, right }); + } + } +} + +void operator>>(const YAML::Node& node, std::vector& filters) { for (unsigned i = 0; i < node.size(); i++) { std::string left; node[i]["left"] >> left; - std::vector rights; - node[i]["right"] >> rights; - for (auto& r : rights) + YAML::Node rights = node[i]["right"]; + for (unsigned i = 0; i < rights.size(); i++) { - connect.insert({left, r}); + if (rights[i].Type() == YAML::NodeType::Map) + { + ParamManager::FilterRawData filter; + filter.input = left; + rights[i].begin()->first >> filter.output; + rights[i].begin()->second >> filter.filter_conditions; + filters.push_back(filter); + } } } } @@ -144,12 +167,12 @@ void operator>>(const YAML::Node& node, bool& val) val = node.as(); } -void operator>>(const YAML::Node & node, int & val) +void operator>>(const YAML::Node& node, int& val) { val = node.as(); } -void operator>>(const YAML::Node & node, float & val) +void operator>>(const YAML::Node& node, float& val) { val = node.as(); } @@ -196,12 +219,9 @@ void ParamManager::print() const // Pring Common Info slog::info << "Common:" << slog::endl; slog::info << "\tcamera_topic: " << common_.camera_topic << slog::endl; - slog::info << "\tcustom_cpu_library: " << common_.custom_cpu_library - << slog::endl; - slog::info << "\tcustom_cldnn_library: " << common_.custom_cldnn_library - << slog::endl; - slog::info << "\tenable_performance_count: " - << common_.enable_performance_count << slog::endl; + slog::info << "\tcustom_cpu_library: " << common_.custom_cpu_library << slog::endl; + slog::info << "\tcustom_cldnn_library: " << common_.custom_cldnn_library << slog::endl; + slog::info << "\tenable_performance_count: " << common_.enable_performance_count << slog::endl; } void ParamManager::parse(std::string path) @@ -229,8 +249,7 @@ std::vector ParamManager::getPipelineNames() const return names; } -ParamManager::PipelineParams ParamManager::getPipeline( - const std::string& name) const +ParamManager::PipelineRawData ParamManager::getPipeline(const std::string& name) const { for (auto& p : pipelines_) { @@ -241,4 +260,4 @@ ParamManager::PipelineParams ParamManager::getPipeline( } throw std::logic_error("No parameters found for pipeline [" + name + "]"); } -} // namespace Params +} // namespace Params \ No newline at end of file