Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

V2.6.0 #165

Merged
merged 7 commits into from
Oct 4, 2023
Merged

V2.6.0 #165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
00cfc42
Publicly available docker image (develop) (#151)
cwlacewe Oct 11, 2022
f09aaf5
VDMS 2.4.0 Release (#155)
cwlacewe Apr 17, 2023
58eaf53
Update v2.4.0 Dockerfile (#158)
cwlacewe Apr 17, 2023
5ac6fc0
VDMS 2.5.0 Release (#160)
cwlacewe Aug 9, 2023
99af889
Update Dockerfile
cwlacewe Aug 9, 2023
f5548b0
VDMS 2.6.0 Release (#163)
cwlacewe Oct 4, 2023
bee1a4c
Merge branch 'master' into v2.6.0
cwlacewe Oct 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 27 additions & 8 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
cmake_minimum_required (VERSION 3.17)

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
set(CMAKE_CXX_STANDARD 17)


IF(CODE_COVERAGE)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -Wall -coverage -fprofile-arcs -ftest-coverage")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -Wall -coverage -fprofile-arcs -ftest-coverage")
Expand All @@ -12,16 +14,32 @@ project(vdms_application)
add_compile_options(-g -fPIC -std=c++17)

find_package( OpenCV REQUIRED )
find_package(Protobuf REQUIRED)
find_package(Protobuf CONFIG REQUIRED)
find_package( CURL REQUIRED )
find_package(AWSSDK REQUIRED COMPONENTS core s3)

include_directories(${Protobuf_INCLUDE_DIRS})
include_directories(${CMAKE_CURRENT_BINARY_DIR})

execute_process(COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json ${CMAKE_CURRENT_BINARY_DIR}/APISchema.h)
protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS utils/src/protobuf/partitionerMessages.proto utils/src/protobuf/pmgdMessages.proto utils/src/protobuf/queryMessage.proto)
add_library(vdms_protobuf SHARED ${PROTO_SRCS} ${PROTO_HDRS})
execute_process(COMMAND python3
${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py
${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json
${CMAKE_CURRENT_BINARY_DIR}/APISchema.h
)
add_library(vdms_protobuf OBJECT
${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/partitionerMessages.proto
${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/pmgdMessages.proto
${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/queryMessage.proto
)
target_link_libraries(vdms_protobuf PUBLIC protobuf::libprotobuf)
set(PROTO_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
target_include_directories(vdms_protobuf PUBLIC "$<BUILD_INTERFACE:${PROTO_BINARY_DIR}>")
protobuf_generate(
LANGUAGE cpp
TARGET vdms_protobuf
IMPORT_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf"
PROTOC_OUT_DIR "${PROTO_BINARY_DIR}"
)

option(CLIENT "Built client library." OFF)
if (CLIENT)
Expand Down Expand Up @@ -54,7 +72,9 @@ else()
src/PMGDIterators.cc
src/PMGDQuery.cc
src/PMGDQueryHandler.cc
src/QueryHandler.cc
src/QueryHandlerExample.cc
src/QueryHandlerBase.cc
src/QueryHandlerPMGD.cc
src/QueryMessage.cc
src/RSCommand.cc
src/SearchExpression.cc
Expand All @@ -63,10 +83,9 @@ else()
src/VideoCommand.cc
src/AutoDeleteNode.cc
src/ImageLoop.cc
${PROTO_SRCS} ${PROTO_HDRS}
)
src/VideoLoop.cc
)
target_link_libraries(dms vcl pmgd pmgd-util protobuf tbb tiledb vdms-utils pthread -lcurl -lzmq ${AWSSDK_LINK_LIBRARIES})

add_executable(vdms src/vdms.cc)
target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS} ${AWSSDK_LINK_LIBRARIES})
endif ()
125 changes: 79 additions & 46 deletions INSTALL.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,20 @@
Here is the detailed process of installation of VDMS dependencies.

## Dependencies
To install VDMS, we must install the necessary dependencies via apt, github, and pip.
To install VDMS, we must install the necessary dependencies via apt, github, and pip (Python 3.9+).

### Install Debian Packages
Here we will install the Debian and Python3 packages.
### Install Debian/Ubuntu Packages
Here we will install the Debian/Ubuntu packages.
```bash
sudo apt-get update
sudo apt-get install -y --no-install-suggests --no-install-recommends \
apt-transport-https autoconf automake bison build-essential bzip2 ca-certificates \
curl ed flex g++-9 gcc-9 git gnupg-agent javacc libarchive-tools libatlas-base-dev \
libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev libc-ares-dev libcurl4-openssl-dev \
libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgtk-3-dev libgtk2.0-dev \
libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \
libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtk-3-dev libgtk2.0-dev \
libhdf5-dev libjpeg-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \
liblz4-dev libopenblas-dev libopenmpi-dev libpng-dev librdkafka-dev libsnappy-dev libssl-dev \
libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev mpich \
libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev linux-libc-dev mpich \
openjdk-11-jdk-headless pkg-config procps python3-dev python3-pip software-properties-common \
swig unzip uuid-dev
```
Expand All @@ -25,81 +25,110 @@ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 1
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 1
```

#### **Install JPEG package**
Please install the JPEG package based on the OS platform being used:
* ***Debian 10+:*** `sudo apt-get install -y libjpeg62-turbo-dev`
* ***Ubuntu 20.04+:*** `sudo apt-get install -y libjpeg8-dev`
<br>

### Install Remaining Dependencies
Here we assume `$VDMS_DEP_DIR` is the directory for installing additional dependencies.
This directory is user-defined but here we use `/dependencies`.
These instructions assume you have full permissions to your system.
If not running as root, add `sudo` where necessary.
***NOTE:*** If running as ***root***, remove `sudo` where applicable.
```bash
VDMS_DEP_DIR=/dependencies # Set to any directory
BUILD_THREADS="-j`nproc`"
mkdir -p $VDMS_DEP_DIR
```


#### Python3 Packages
Here we will install the necessary Python3 packages Numpy and Protobuf 3.20.3.
Here we will install the necessary Python 3.9+ packages Numpy and Protobuf v24.2.
It is expected that you have Python3.9 or higher installed on your system.
All python calls will use Python3.9+; therefore you may find it convenient to set alias for python.
```bash
alias python=/usr/bin/python3
```
***NOTE:*** If multiple versions of Python 3 are present on your system, verify you are using Python3.9 or higher. You can specify the specific verison in above command and also set the following with your specific version: `alias python3=/usr/bin/python3.x`.

You can also install the coverage package if interested in running the Python unit tests.
```bash
PROTOBUF_VERSION="3.20.3"
pip3 install --no-cache-dir "numpy>=1.25.1" "protobuf==${PROTOBUF_VERSION}" "coverage>=7.2.7"
python3 -m pip install --upgrade pip
python3 -m pip install --no-cache-dir "numpy>=1.25.1" "coverage>=7.2.7"
```


#### CMAKE v3.26.4
VDMS requires CMake v3.21+. Here we install CMake v3.26.4.
#### **CMAKE v3.27.2**
VDMS requires CMake v3.21+. Here we install CMake v3.27.2.
```bash
CMAKE_VERSION="v3.26.4"
CMAKE_VERSION="v3.27.2"
git clone --branch ${CMAKE_VERSION} https://github.com/Kitware/CMake.git $VDMS_DEP_DIR/CMake
cd $VDMS_DEP_DIR/CMake
./bootstrap
make ${BUILD_THREADS}
make install
sudo make install
```

### gtest
Unfortunately apt doesn't build gtest so you need to do the following:
```bash
cd /usr/src/gtest/
cmake .
make ${BUILD_THREADS}
mv lib/libgtest* /usr/lib
```

### Faiss v1.7.3
#### **Faiss v1.7.3**
Install the Faiss library for similarity search.
```bash
FAISS_VERSION="v1.7.3"
git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git $VDMS_DEP_DIR/faiss
cd $VDMS_DEP_DIR/faiss
mkdir build && cd build
cmake -DFAISS_ENABLE_GPU=OFF ..
cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=/usr/bin/python3 ..
make ${BUILD_THREADS}
make install
sudo make install
```

### FLINNG

#### **FLINNG**
Install the Filters to Identify Near-Neighbor Groups (FLINNG) library for similarity search.
```bash
git clone https://github.com/tonyzhang617/FLINNG.git $VDMS_DEP_DIR/FLINNG
cd $VDMS_DEP_DIR/FLINNG
mkdir build && cd build
cmake ..
make ${BUILD_THREADS}
make install
sudo make install
```

### Protobuf 3.20.3

#### **Protobuf v24.2 (4.24.2)**
Install Protobuf (C++ and Python) which requires GoogleTest and Abseil C++ as dependencies.
```bash
PROTOBUF_VERSION="3.20.3"
curl -L -o ${VDMS_DEP_DIR}/${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz
cd ${VDMS_DEP_DIR} && tar -xvf ${PROTOBUF_VERSION}.tar.gz
cd protobuf-${PROTOBUF_VERSION}
./autogen.sh
./configure
PROTOBUF_VERSION="24.2"
git clone -b v${PROTOBUF_VERSION} --recursive https://github.com/protocolbuffers/protobuf.git $VDMS_DEP_DIR/protobuf

cd $VDMS_DEP_DIR/protobuf/third_party/googletest
mkdir build && cd build
cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \
-DBUILD_GMOCK=ON -DCMAKE_CXX_STANDARD=17 ..
make ${BUILD_THREADS}
sudo make install
sudo ldconfig

cd $VDMS_DEP_DIR/protobuf/third_party/abseil-cpp
mkdir build && cd build
cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ \
-DABSL_BUILD_TESTING=ON -DABSL_ENABLE_INSTALL=ON -DABSL_USE_EXTERNAL_GOOGLETEST=ON \
-DABSL_FIND_GOOGLETEST=ON -DCMAKE_CXX_STANDARD=17 ..
make ${BUILD_THREADS}
sudo make install

cd $VDMS_DEP_DIR/protobuf
cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_CXX_STANDARD=17 \
-Dprotobuf_ABSL_PROVIDER=package -DCMAKE_PREFIX_PATH=/usr/local .
make ${BUILD_THREADS}
make install
ldconfig
sudo make install

python3 -m pip install --no-cache-dir "protobuf==4.${PROTOBUF_VERSION}"
```

### [OpenCV](https://opencv.org/) 4.5.5

#### **[OpenCV](https://opencv.org/) 4.5.5**
Below are instructions for installing ***OpenCV v4.5.5***.
```bash
OPENCV_VERSION="4.5.5"
Expand All @@ -108,7 +137,7 @@ cd $VDMS_DEP_DIR/opencv
mkdir build && cd build
cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF ..
make ${BUILD_THREADS}
make install
sudo make install
```

**Note**: When using videos, and getting the following error: "Unable to stop the stream: Inappropriate ioctl for device", you may need to include more flags when compiling OpenCV. Follow these instructions ([source](https://stackoverflow.com/questions/41200201/opencv-unable-to-stop-the-stream-inappropriate-ioctl-for-device)):
Expand All @@ -121,20 +150,21 @@ cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF -D CMAKE_BUILD_TYPE=RELEASE -D
-D WITH_V4L=ON -D WITH_OPENGL=ON -D WITH_CUBLAS=ON \
-DWITH_QT=OFF -DCUDA_NVCC_FLAGS="-D_FORCE_INLINES" ..
make ${BUILD_THREADS}
make install
sudo make install
```

### Valijson v0.6
This is a headers-only library, no compilation/installation necessary

#### **Valijson v0.6**
This is a headers-only library, no compilation/installation necessary.
```bash
VALIJSON_VERSION="v0.6"
git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git $VDMS_DEP_DIR/valijson
cd $VDMS_DEP_DIR/valijson
cp -r include/* /usr/local/include/
sudo cp -r include/* /usr/local/include/
```


### [TileDB](https://tiledb.io/) 2.14.1
#### **[TileDB](https://tiledb.io/) 2.14.1**
The directions below will help you install TileDB v2.14.1 from the source.
You can also follow the directions listed [here](https://docs.tiledb.io/en/latest/installation.html).
```bash
Expand All @@ -146,19 +176,22 @@ cd TileDB-${TILEDB_VERSION}
mkdir build && cd build
../bootstrap --prefix=/usr/local/
make ${BUILD_THREADS}
make install-tiledb
sudo make install-tiledb
```

### AWS SDK CPP 1.11.0

#### **AWS SDK CPP 1.11.0**
Use the following instructions to install AWS SDK for C++.
```bash
AWS_SDK_VERSION="1.11.0"
git clone -b ${AWS_SDK_VERSION} --recurse-submodules https://github.com/aws/aws-sdk-cpp ${VDMS_DEP_DIR}/aws-sdk-cpp
mkdir -p ${VDMS_DEP_DIR}/aws-sdk-cpp/build
cd ${VDMS_DEP_DIR}/aws-sdk-cpp/build
cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DBUILD_ONLY="s3" -DCUSTOM_MEMORY_MANAGEMENT=OFF
make ${BUILD_THREADS}
make install
sudo make install
```
<br>

## Install VDMS
This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories.
Expand Down
Loading