VDMS 2.5.0 Release (#162)

IntelLabs · Aug 9, 2023 · 3b6455a · 3b6455a
1 parent 9249408
commit 3b6455a
Show file tree

Hide file tree

Showing 190 changed files with 94,018 additions and 26,040 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,7 +2,11 @@ build/
 db/
 *.gcov
 **/__pycache__/
+fr_client
+tmp
+*.onnx
 
 # VS Code Specific
 .devcontainer
-.vscode
+.vscode
+.coverage
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,4 +1,6 @@
 cmake_minimum_required (VERSION 3.17)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
+set(CMAKE_CXX_STANDARD 17)
 
 IF(CODE_COVERAGE)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -Wall -coverage -fprofile-arcs -ftest-coverage")
@@ -7,15 +9,17 @@ IF(CODE_COVERAGE)
 ENDIF()
 
 project(vdms_application)
-add_compile_options(-g -fPIC)
+add_compile_options(-g -fPIC -std=c++17)
 
 find_package( OpenCV REQUIRED )
 find_package(Protobuf REQUIRED)
+find_package( CURL REQUIRED )
+find_package(AWSSDK REQUIRED COMPONENTS core s3)
 
 include_directories(${Protobuf_INCLUDE_DIRS})
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
-execute_process(COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json ${CMAKE_CURRENT_BINARY_DIR}/APISchema.h)
+execute_process(COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json ${CMAKE_CURRENT_BINARY_DIR}/APISchema.h)
 protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS utils/src/protobuf/partitionerMessages.proto utils/src/protobuf/pmgdMessages.proto utils/src/protobuf/queryMessage.proto)
 add_library(vdms_protobuf SHARED ${PROTO_SRCS} ${PROTO_HDRS})
 
@@ -28,7 +32,6 @@ if (CLIENT)
   add_subdirectory(utils)
   add_subdirectory(client/cpp)
 
-
 else()
   add_subdirectory(src/pmgd)
   add_subdirectory(utils)
@@ -41,29 +44,29 @@ else()
   link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/)
   include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf)
   add_library(dms SHARED
-                        src/BoundingBoxCommand.cc
-                        src/BlobCommand.cc
-                        src/CommunicationManager.cc
-                        src/DescriptorsCommand.cc
-                        src/DescriptorsManager.cc
-                        src/ExceptionsCommand.cc
-                        src/ImageCommand.cc
-                        src/PMGDIterators.cc
-                        src/PMGDQuery.cc
-                        src/PMGDQueryHandler.cc
-                        src/QueryHandler.cc
-                        src/QueryMessage.cc
-                        src/RSCommand.cc
-                        src/SearchExpression.cc
-                        src/Server.cc
-                        src/VDMSConfig.cc
-                        src/VideoCommand.cc
-                        src/AutoDeleteNode.cc
-                        ${PROTO_SRCS} ${PROTO_HDRS}
-                        )
-  target_link_libraries(dms vcl pmgd pmgd-util protobuf vdms-utils pthread)
-
+    src/BoundingBoxCommand.cc
+    src/BlobCommand.cc
+    src/CommunicationManager.cc
+    src/DescriptorsCommand.cc
+    src/DescriptorsManager.cc
+    src/ExceptionsCommand.cc
+    src/ImageCommand.cc
+    src/PMGDIterators.cc
+    src/PMGDQuery.cc
+    src/PMGDQueryHandler.cc
+    src/QueryHandler.cc
+    src/QueryMessage.cc
+    src/RSCommand.cc
+    src/SearchExpression.cc
+    src/Server.cc
+    src/VDMSConfig.cc
+    src/VideoCommand.cc
+    src/AutoDeleteNode.cc
+    src/ImageLoop.cc
+    ${PROTO_SRCS} ${PROTO_HDRS}
+)
+  target_link_libraries(dms vcl pmgd pmgd-util protobuf tbb tiledb vdms-utils pthread -lcurl -lzmq ${AWSSDK_LINK_LIBRARIES})
 
   add_executable(vdms src/vdms.cc)
-  target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS})
+  target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS} ${AWSSDK_LINK_LIBRARIES})
 endif ()
diff --git a/INSTALL.md b/INSTALL.md
@@ -2,117 +2,113 @@
 Here is the detailed process of installation of VDMS dependencies.
 
 ## Dependencies
-Here we will install the Ubuntu 20.04 and Python3 packages. We assume `python`/`pip` is an alias for `python3`/`pip3`. If your system has both  Python 2 and Python 3, please replace all pip and python commands with pip3 and python3, respectively.
+To install VDMS, we must install the necessary dependencies via apt, github, and pip.
+
+### Install Debian Packages
+Here we will install the Debian and Python3 packages.
 ```bash
 sudo apt-get update
-sudo apt-get -y install --no-install-recommends software-properties-common
-sudo add-apt-repository "deb http://security.ubuntu.com/ubuntu focal-security main"
-sudo apt-get -y install --no-install-recommends apt-transport-https autoconf automake bison build-essential \
-    bzip2 ca-certificates curl=7.68.0-1ubuntu2.18 ed flex g++ git gnupg-agent javacc libarchive-tools \
-    libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \
-    libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \
-    libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \
-    libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \
-    libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \
-    libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \
-    pkg-config python3-dev python3-pip unzip
-pip install --no-cache-dir "numpy>=1.23.2" "setuptools>=65.5.1"
-```
-### Clone/Download Dependencies
-Here we clone the repositories for grpc v1.40.0, libpng12, Swig v4.0.2, OpenCV 4.5.3, Valijson v0.6, CMake v3.21.2, Faiss v1.7.1, and FLINNG. Then download necesarry files for zlib v1.2.13, Json-simple v1.1.1, and TileDB v1.3.1.
-Here we assume `$VDMS_DEP_DIR` is the working directory for installing dependencies and `python` is Python 3.
+sudo apt-get install -y --no-install-suggests --no-install-recommends \
+    apt-transport-https autoconf automake bison build-essential bzip2 ca-certificates \
+    curl ed flex g++-9 gcc-9 git gnupg-agent javacc libarchive-tools libatlas-base-dev \
+    libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev libc-ares-dev libcurl4-openssl-dev \
+    libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgtk-3-dev libgtk2.0-dev \
+    libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \
+    liblz4-dev libopenblas-dev libopenmpi-dev libpng-dev librdkafka-dev libsnappy-dev libssl-dev \
+    libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev mpich \
+    openjdk-11-jdk-headless pkg-config procps python3-dev python3-pip software-properties-common \
+    swig unzip uuid-dev
+```
+Note: Your system may have g++ or gcc version 10+. If this is the case, please use version 9 to build VDMS. Optional method for setting version 9 as default:
+```bash
+update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 1
+update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 1
+```
+
+### Install Remaining Dependencies
+Here we assume `$VDMS_DEP_DIR` is the directory for installing additional dependencies.
+This directory is user-defined but here we use `/dependencies`.
+These instructions assume you have full permissions to your system.
+If not running as root, add `sudo` where necessary.
 ```bash
-cd $VDMS_DEP_DIR
-git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \
-git clone --branch v4.0.2 https://github.com/swig/swig.git && \
-git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \
-git clone https://github.com/tonyzhang617/FLINNG.git && \
-git clone --recurse-submodules -b v1.40.0 https://github.com/grpc/grpc.git && \
-git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \
-git clone --branch v0.6 https://github.com/tristanpenman/valijson.git
+VDMS_DEP_DIR=/dependencies  # Set to any directory
+BUILD_THREADS="-j`nproc`"
+mkdir -p $VDMS_DEP_DIR
+```
 
-sudo curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \
-sudo curl -L -o 1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \
-sudo curl -L -o zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz
+#### Python3 Packages
+Here we will install the necessary Python3 packages Numpy and Protobuf 3.20.3.
+You can also install the coverage package if interested in running the Python unit tests.
+```bash
+PROTOBUF_VERSION="3.20.3"
+pip3 install --no-cache-dir "numpy>=1.25.1" "protobuf==${PROTOBUF_VERSION}" "coverage>=7.2.7"
 ```
 
-### Install Dependencies
-These instructions assume you have full permissions to your system.
-If running as root, remove `sudo` where necessary.
 
-#### CMAKE
+#### CMAKE v3.26.4
+VDMS requires CMake v3.21+.  Here we install CMake v3.26.4.
 ```bash
-cd $VDMS_DEP_DIR/CMake && ./bootstrap
-make -j && sudo make install
+CMAKE_VERSION="v3.26.4"
+git clone --branch ${CMAKE_VERSION} https://github.com/Kitware/CMake.git $VDMS_DEP_DIR/CMake
+cd $VDMS_DEP_DIR/CMake
+./bootstrap
+make ${BUILD_THREADS}
+make install
 ```
 
-### Swig
+### gtest
+Unfortunately apt doesn't build gtest so you need to do the following:
 ```bash
-cd $VDMS_DEP_DIR/swig
-./autogen.sh && ./configure
-make -j && sudo make install
+cd /usr/src/gtest/
+cmake .
+make ${BUILD_THREADS}
+mv lib/libgtest* /usr/lib
 ```
 
-### Faiss
+### Faiss v1.7.3
 ```bash
+FAISS_VERSION="v1.7.3"
+git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git $VDMS_DEP_DIR/faiss
 cd $VDMS_DEP_DIR/faiss
 mkdir build && cd build
 cmake -DFAISS_ENABLE_GPU=OFF ..
-make -j && sudo make install
+make ${BUILD_THREADS}
+make install
 ```
 
 ### FLINNG
 ```bash
+git clone https://github.com/tonyzhang617/FLINNG.git $VDMS_DEP_DIR/FLINNG
 cd $VDMS_DEP_DIR/FLINNG
 mkdir build && cd build
 cmake ..
-make -j && sudo make install
+make ${BUILD_THREADS}
+make install
 ```
 
-### grpc
+### Protobuf 3.20.3
 ```bash
-cd $VDMS_DEP_DIR/grpc
-pip install --no-cache-dir -r requirements.txt
-GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install --no-cache-dir .
-
-cd tools/distrib/python/grpcio_tools
-python ../make_grpcio_tools.py
-GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install --no-cache-dir .
-
-cd $VDMS_DEP_DIR/grpc/third_party/zlib/ && mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
-make -j && sudo make install
-
-cd $VDMS_DEP_DIR/grpc/third_party/protobuf/cmake
-mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -Dprotobuf_BUILD_TESTS=OFF ..
-make -j && sudo make install
-
-cd ../../../abseil-cpp && mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
-make -j && sudo make install
-
-cd ../../re2/ && mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
-make -j && sudo make install
-
-cd $VDMS_DEP_DIR/grpc/cmake && mkdir build && cd build
-cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \
-    -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \
-    -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \
-    -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../..
-make -j && sudo make install
+PROTOBUF_VERSION="3.20.3"
+curl -L -o ${VDMS_DEP_DIR}/${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz
+cd ${VDMS_DEP_DIR} && tar -xvf ${PROTOBUF_VERSION}.tar.gz
+cd protobuf-${PROTOBUF_VERSION}
+./autogen.sh
+./configure
+make ${BUILD_THREADS}
+make install
+ldconfig
 ```
 
-### [OpenCV](https://opencv.org/)
-
-Below are instructions for installing ***OpenCV v4.5.3***. It may also work with newer versions of OpenCV.
+### [OpenCV](https://opencv.org/) 4.5.5
+Below are instructions for installing ***OpenCV v4.5.5***.
 ```bash
+OPENCV_VERSION="4.5.5"
+git clone --branch ${OPENCV_VERSION} https://github.com/opencv/opencv.git $VDMS_DEP_DIR/opencv
 cd $VDMS_DEP_DIR/opencv
 mkdir build && cd build
-cmake -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF ..
-make -j
-sudo make install
+cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF ..
+make ${BUILD_THREADS}
+make install
 ```
 
 **Note**: When using videos, and getting the following error: "Unable to stop the stream: Inappropriate ioctl for device", you may need to include more flags when compiling OpenCV. Follow these instructions ([source](https://stackoverflow.com/questions/41200201/opencv-unable-to-stop-the-stream-inappropriate-ioctl-for-device)):
@@ -124,51 +120,51 @@ cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF -D CMAKE_BUILD_TYPE=RELEASE -D
     -D WITH_FFMPEG=ON -D WITH_TBB=ON -D WITH_GTK=ON \
     -D WITH_V4L=ON -D WITH_OPENGL=ON -D WITH_CUBLAS=ON \
     -DWITH_QT=OFF -DCUDA_NVCC_FLAGS="-D_FORCE_INLINES" ..
-make -j
+make ${BUILD_THREADS}
 make install
 ```
 
-### Zlib
+### Valijson v0.6
+This is a headers-only library, no compilation/installation necessary
 ```bash
-cd $VDMS_DEP_DIR && tar -xvzf zlib-1.2.13.tar.gz
-cd zlib-1.2.13 && ./configure
-make -j && sudo make install
+VALIJSON_VERSION="v0.6"
+git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git $VDMS_DEP_DIR/valijson
+cd $VDMS_DEP_DIR/valijson
+cp -r include/* /usr/local/include/
 ```
 
-### [TileDB](https://tiledb.io/)
-VDMS works with ***TileDB v1.3.1.***<br>
-The directions below will help you install TileDB v1.3.1 from the source.
-You can also follow the directions listed
-[here](https://docs.tiledb.io/en/latest/installation.html).
-```bash
-cd $VDMS_DEP_DIR && tar -xvf 1.3.1.tar.gz
-cd TileDB-1.3.1 && mkdir build && cd build
-../bootstrap --prefix=/usr/local/
-make -j && sudo make install-tiledb
-```
 
-### gtest
-Unfortunately apt doesn't build gtest;
-you need to do the following steps to get it to work correctly:
+### [TileDB](https://tiledb.io/) 2.14.1
+The directions below will help you install TileDB v2.14.1 from the source.
+You can also follow the directions listed [here](https://docs.tiledb.io/en/latest/installation.html).
 ```bash
-cd /usr/src/gtest/
-sudo cmake .
-sudo make -j
-sudo mv lib/libgtest* /usr/lib
+TILEDB_VERSION="2.14.1"
+curl -L -o $VDMS_DEP_DIR/${TILEDB_VERSION}.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/${TILEDB_VERSION}.tar.gz && \
+cd $VDMS_DEP_DIR
+tar -xvf ${TILEDB_VERSION}.tar.gz
+cd TileDB-${TILEDB_VERSION}
+mkdir build && cd build
+../bootstrap --prefix=/usr/local/
+make ${BUILD_THREADS}
+make install-tiledb
 ```
 
-### Valijson
-This is a headers-only library, no compilation/installation necessary
+### AWS SDK CPP 1.11.0
 ```bash
-cd $VDMS_DEP_DIR/valijson
-sudo cp -r include/* /usr/local/include
+AWS_SDK_VERSION="1.11.0"
+git clone -b ${AWS_SDK_VERSION} --recurse-submodules https://github.com/aws/aws-sdk-cpp ${VDMS_DEP_DIR}/aws-sdk-cpp
+mkdir -p ${VDMS_DEP_DIR}/aws-sdk-cpp/build
+cd ${VDMS_DEP_DIR}/aws-sdk-cpp/build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DBUILD_ONLY="s3" -DCUSTOM_MEMORY_MANAGEMENT=OFF
+make ${BUILD_THREADS}
+make install
 ```
 
 ## Install VDMS
 This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories.
 ```bash
-git clone https://github.com/IntelLabs/vdms.git
-cd vdms && git checkout develop
+git clone -b develop https://github.com/IntelLabs/vdms.git
+cd vdms
 git submodule update --init --recursive
 ```
 
@@ -186,3 +182,4 @@ mkdir build && cd build
 cmake -DCMAKE_CXX_FLAGS='-DPM' ..
 make -j
 ```
+