Skip to content

Commit

Permalink
V2.6.0 (#165)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cwlacewe
cwlacewe authored Oct 4, 2023
1 parent 3b6455a commit 0a70563
Show file tree
Hide file tree
Showing 73 changed files with 6,458 additions and 3,096 deletions.
35 changes: 27 additions & 8 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
cmake_minimum_required (VERSION 3.17)

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
set(CMAKE_CXX_STANDARD 17)


IF(CODE_COVERAGE)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -Wall -coverage -fprofile-arcs -ftest-coverage")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -Wall -coverage -fprofile-arcs -ftest-coverage")
Expand All @@ -12,16 +14,32 @@ project(vdms_application)
add_compile_options(-g -fPIC -std=c++17)

find_package( OpenCV REQUIRED )
find_package(Protobuf REQUIRED)
find_package(Protobuf CONFIG REQUIRED)
find_package( CURL REQUIRED )
find_package(AWSSDK REQUIRED COMPONENTS core s3)

include_directories(${Protobuf_INCLUDE_DIRS})
include_directories(${CMAKE_CURRENT_BINARY_DIR})

execute_process(COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json ${CMAKE_CURRENT_BINARY_DIR}/APISchema.h)
protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS utils/src/protobuf/partitionerMessages.proto utils/src/protobuf/pmgdMessages.proto utils/src/protobuf/queryMessage.proto)
add_library(vdms_protobuf SHARED ${PROTO_SRCS} ${PROTO_HDRS})
execute_process(COMMAND python3
${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py
${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json
${CMAKE_CURRENT_BINARY_DIR}/APISchema.h
)
add_library(vdms_protobuf OBJECT
${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/partitionerMessages.proto
${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/pmgdMessages.proto
${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/queryMessage.proto
)
target_link_libraries(vdms_protobuf PUBLIC protobuf::libprotobuf)
set(PROTO_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
target_include_directories(vdms_protobuf PUBLIC "$<BUILD_INTERFACE:${PROTO_BINARY_DIR}>")
protobuf_generate(
LANGUAGE cpp
TARGET vdms_protobuf
IMPORT_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf"
PROTOC_OUT_DIR "${PROTO_BINARY_DIR}"
)

option(CLIENT "Built client library." OFF)
if (CLIENT)
Expand Down Expand Up @@ -54,7 +72,9 @@ else()
src/PMGDIterators.cc
src/PMGDQuery.cc
src/PMGDQueryHandler.cc
src/QueryHandler.cc
src/QueryHandlerExample.cc
src/QueryHandlerBase.cc
src/QueryHandlerPMGD.cc
src/QueryMessage.cc
src/RSCommand.cc
src/SearchExpression.cc
Expand All @@ -63,10 +83,9 @@ else()
src/VideoCommand.cc
src/AutoDeleteNode.cc
src/ImageLoop.cc
${PROTO_SRCS} ${PROTO_HDRS}
)
src/VideoLoop.cc
)
target_link_libraries(dms vcl pmgd pmgd-util protobuf tbb tiledb vdms-utils pthread -lcurl -lzmq ${AWSSDK_LINK_LIBRARIES})

add_executable(vdms src/vdms.cc)
target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS} ${AWSSDK_LINK_LIBRARIES})
endif ()
125 changes: 79 additions & 46 deletions INSTALL.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,20 @@
Here is the detailed process of installation of VDMS dependencies.

## Dependencies
To install VDMS, we must install the necessary dependencies via apt, github, and pip.
To install VDMS, we must install the necessary dependencies via apt, github, and pip (Python 3.9+).

### Install Debian Packages
Here we will install the Debian and Python3 packages.
### Install Debian/Ubuntu Packages
Here we will install the Debian/Ubuntu packages.
```bash
sudo apt-get update
sudo apt-get install -y --no-install-suggests --no-install-recommends \
apt-transport-https autoconf automake bison build-essential bzip2 ca-certificates \
curl ed flex g++-9 gcc-9 git gnupg-agent javacc libarchive-tools libatlas-base-dev \
libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev libc-ares-dev libcurl4-openssl-dev \
libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgtk-3-dev libgtk2.0-dev \
libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \
libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtk-3-dev libgtk2.0-dev \
libhdf5-dev libjpeg-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \
liblz4-dev libopenblas-dev libopenmpi-dev libpng-dev librdkafka-dev libsnappy-dev libssl-dev \
libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev mpich \
libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev linux-libc-dev mpich \
openjdk-11-jdk-headless pkg-config procps python3-dev python3-pip software-properties-common \
swig unzip uuid-dev
```
Expand All @@ -25,81 +25,110 @@ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 1
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 1
```

#### **Install JPEG package**
Please install the JPEG package based on the OS platform being used:
* ***Debian 10+:*** `sudo apt-get install -y libjpeg62-turbo-dev`
* ***Ubuntu 20.04+:*** `sudo apt-get install -y libjpeg8-dev`
<br>

### Install Remaining Dependencies
Here we assume `$VDMS_DEP_DIR` is the directory for installing additional dependencies.
This directory is user-defined but here we use `/dependencies`.
These instructions assume you have full permissions to your system.
If not running as root, add `sudo` where necessary.
***NOTE:*** If running as ***root***, remove `sudo` where applicable.
```bash
VDMS_DEP_DIR=/dependencies # Set to any directory
BUILD_THREADS="-j`nproc`"
mkdir -p $VDMS_DEP_DIR
```


#### Python3 Packages
Here we will install the necessary Python3 packages Numpy and Protobuf 3.20.3.
Here we will install the necessary Python 3.9+ packages Numpy and Protobuf v24.2.
It is expected that you have Python3.9 or higher installed on your system.
All python calls will use Python3.9+; therefore you may find it convenient to set alias for python.
```bash
alias python=/usr/bin/python3
```
***NOTE:*** If multiple versions of Python 3 are present on your system, verify you are using Python3.9 or higher. You can specify the specific verison in above command and also set the following with your specific version: `alias python3=/usr/bin/python3.x`.

You can also install the coverage package if interested in running the Python unit tests.
```bash
PROTOBUF_VERSION="3.20.3"
pip3 install --no-cache-dir "numpy>=1.25.1" "protobuf==${PROTOBUF_VERSION}" "coverage>=7.2.7"
python3 -m pip install --upgrade pip
python3 -m pip install --no-cache-dir "numpy>=1.25.1" "coverage>=7.2.7"
```


#### CMAKE v3.26.4
VDMS requires CMake v3.21+. Here we install CMake v3.26.4.
#### **CMAKE v3.27.2**
VDMS requires CMake v3.21+. Here we install CMake v3.27.2.
```bash
CMAKE_VERSION="v3.26.4"
CMAKE_VERSION="v3.27.2"
git clone --branch ${CMAKE_VERSION} https://github.com/Kitware/CMake.git $VDMS_DEP_DIR/CMake
cd $VDMS_DEP_DIR/CMake
./bootstrap
make ${BUILD_THREADS}
make install
sudo make install
```

### gtest
Unfortunately apt doesn't build gtest so you need to do the following:
```bash
cd /usr/src/gtest/
cmake .
make ${BUILD_THREADS}
mv lib/libgtest* /usr/lib
```

### Faiss v1.7.3
#### **Faiss v1.7.3**
Install the Faiss library for similarity search.
```bash
FAISS_VERSION="v1.7.3"
git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git $VDMS_DEP_DIR/faiss
cd $VDMS_DEP_DIR/faiss
mkdir build && cd build
cmake -DFAISS_ENABLE_GPU=OFF ..
cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=/usr/bin/python3 ..
make ${BUILD_THREADS}
make install
sudo make install
```

### FLINNG

#### **FLINNG**
Install the Filters to Identify Near-Neighbor Groups (FLINNG) library for similarity search.
```bash
git clone https://github.com/tonyzhang617/FLINNG.git $VDMS_DEP_DIR/FLINNG
cd $VDMS_DEP_DIR/FLINNG
mkdir build && cd build
cmake ..
make ${BUILD_THREADS}
make install
sudo make install
```

### Protobuf 3.20.3

#### **Protobuf v24.2 (4.24.2)**
Install Protobuf (C++ and Python) which requires GoogleTest and Abseil C++ as dependencies.
```bash
PROTOBUF_VERSION="3.20.3"
curl -L -o ${VDMS_DEP_DIR}/${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz
cd ${VDMS_DEP_DIR} && tar -xvf ${PROTOBUF_VERSION}.tar.gz
cd protobuf-${PROTOBUF_VERSION}
./autogen.sh
./configure
PROTOBUF_VERSION="24.2"
git clone -b v${PROTOBUF_VERSION} --recursive https://github.com/protocolbuffers/protobuf.git $VDMS_DEP_DIR/protobuf

cd $VDMS_DEP_DIR/protobuf/third_party/googletest
mkdir build && cd build
cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \
-DBUILD_GMOCK=ON -DCMAKE_CXX_STANDARD=17 ..
make ${BUILD_THREADS}
sudo make install
sudo ldconfig

cd $VDMS_DEP_DIR/protobuf/third_party/abseil-cpp
mkdir build && cd build
cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ \
-DABSL_BUILD_TESTING=ON -DABSL_ENABLE_INSTALL=ON -DABSL_USE_EXTERNAL_GOOGLETEST=ON \
-DABSL_FIND_GOOGLETEST=ON -DCMAKE_CXX_STANDARD=17 ..
make ${BUILD_THREADS}
sudo make install

cd $VDMS_DEP_DIR/protobuf
cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_CXX_STANDARD=17 \
-Dprotobuf_ABSL_PROVIDER=package -DCMAKE_PREFIX_PATH=/usr/local .
make ${BUILD_THREADS}
make install
ldconfig
sudo make install

python3 -m pip install --no-cache-dir "protobuf==4.${PROTOBUF_VERSION}"
```

### [OpenCV](https://opencv.org/) 4.5.5

#### **[OpenCV](https://opencv.org/) 4.5.5**
Below are instructions for installing ***OpenCV v4.5.5***.
```bash
OPENCV_VERSION="4.5.5"
Expand All @@ -108,7 +137,7 @@ cd $VDMS_DEP_DIR/opencv
mkdir build && cd build
cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF ..
make ${BUILD_THREADS}
make install
sudo make install
```

**Note**: When using videos, and getting the following error: "Unable to stop the stream: Inappropriate ioctl for device", you may need to include more flags when compiling OpenCV. Follow these instructions ([source](https://stackoverflow.com/questions/41200201/opencv-unable-to-stop-the-stream-inappropriate-ioctl-for-device)):
Expand All @@ -121,20 +150,21 @@ cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF -D CMAKE_BUILD_TYPE=RELEASE -D
-D WITH_V4L=ON -D WITH_OPENGL=ON -D WITH_CUBLAS=ON \
-DWITH_QT=OFF -DCUDA_NVCC_FLAGS="-D_FORCE_INLINES" ..
make ${BUILD_THREADS}
make install
sudo make install
```

### Valijson v0.6
This is a headers-only library, no compilation/installation necessary

#### **Valijson v0.6**
This is a headers-only library, no compilation/installation necessary.
```bash
VALIJSON_VERSION="v0.6"
git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git $VDMS_DEP_DIR/valijson
cd $VDMS_DEP_DIR/valijson
cp -r include/* /usr/local/include/
sudo cp -r include/* /usr/local/include/
```


### [TileDB](https://tiledb.io/) 2.14.1
#### **[TileDB](https://tiledb.io/) 2.14.1**
The directions below will help you install TileDB v2.14.1 from the source.
You can also follow the directions listed [here](https://docs.tiledb.io/en/latest/installation.html).
```bash
Expand All @@ -146,19 +176,22 @@ cd TileDB-${TILEDB_VERSION}
mkdir build && cd build
../bootstrap --prefix=/usr/local/
make ${BUILD_THREADS}
make install-tiledb
sudo make install-tiledb
```

### AWS SDK CPP 1.11.0

#### **AWS SDK CPP 1.11.0**
Use the following instructions to install AWS SDK for C++.
```bash
AWS_SDK_VERSION="1.11.0"
git clone -b ${AWS_SDK_VERSION} --recurse-submodules https://github.com/aws/aws-sdk-cpp ${VDMS_DEP_DIR}/aws-sdk-cpp
mkdir -p ${VDMS_DEP_DIR}/aws-sdk-cpp/build
cd ${VDMS_DEP_DIR}/aws-sdk-cpp/build
cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DBUILD_ONLY="s3" -DCUSTOM_MEMORY_MANAGEMENT=OFF
make ${BUILD_THREADS}
make install
sudo make install
```
<br>

## Install VDMS
This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories.
Expand Down
Loading

0 comments on commit 0a70563

Please sign in to comment.