diff --git a/.gitignore b/.gitignore
index dfaf72db..0802dbea 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,7 +2,11 @@ build/
db/
*.gcov
**/__pycache__/
+fr_client
+tmp
+*.onnx
# VS Code Specific
.devcontainer
-.vscode
\ No newline at end of file
+.vscode
+.coverage
\ No newline at end of file
diff --git a/CMakeLists.txt b/CMakeLists.txt
index afda38bf..713b4f66 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,4 +1,6 @@
cmake_minimum_required (VERSION 3.17)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
+set(CMAKE_CXX_STANDARD 17)
IF(CODE_COVERAGE)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -Wall -coverage -fprofile-arcs -ftest-coverage")
@@ -7,15 +9,17 @@ IF(CODE_COVERAGE)
ENDIF()
project(vdms_application)
-add_compile_options(-g -fPIC)
+add_compile_options(-g -fPIC -std=c++17)
find_package( OpenCV REQUIRED )
find_package(Protobuf REQUIRED)
+find_package( CURL REQUIRED )
+find_package(AWSSDK REQUIRED COMPONENTS core s3)
include_directories(${Protobuf_INCLUDE_DIRS})
include_directories(${CMAKE_CURRENT_BINARY_DIR})
-execute_process(COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json ${CMAKE_CURRENT_BINARY_DIR}/APISchema.h)
+execute_process(COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json ${CMAKE_CURRENT_BINARY_DIR}/APISchema.h)
protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS utils/src/protobuf/partitionerMessages.proto utils/src/protobuf/pmgdMessages.proto utils/src/protobuf/queryMessage.proto)
add_library(vdms_protobuf SHARED ${PROTO_SRCS} ${PROTO_HDRS})
@@ -28,7 +32,6 @@ if (CLIENT)
add_subdirectory(utils)
add_subdirectory(client/cpp)
-
else()
add_subdirectory(src/pmgd)
add_subdirectory(utils)
@@ -41,29 +44,29 @@ else()
link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/)
include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf)
add_library(dms SHARED
- src/BoundingBoxCommand.cc
- src/BlobCommand.cc
- src/CommunicationManager.cc
- src/DescriptorsCommand.cc
- src/DescriptorsManager.cc
- src/ExceptionsCommand.cc
- src/ImageCommand.cc
- src/PMGDIterators.cc
- src/PMGDQuery.cc
- src/PMGDQueryHandler.cc
- src/QueryHandler.cc
- src/QueryMessage.cc
- src/RSCommand.cc
- src/SearchExpression.cc
- src/Server.cc
- src/VDMSConfig.cc
- src/VideoCommand.cc
- src/AutoDeleteNode.cc
- ${PROTO_SRCS} ${PROTO_HDRS}
- )
- target_link_libraries(dms vcl pmgd pmgd-util protobuf vdms-utils pthread)
-
+ src/BoundingBoxCommand.cc
+ src/BlobCommand.cc
+ src/CommunicationManager.cc
+ src/DescriptorsCommand.cc
+ src/DescriptorsManager.cc
+ src/ExceptionsCommand.cc
+ src/ImageCommand.cc
+ src/PMGDIterators.cc
+ src/PMGDQuery.cc
+ src/PMGDQueryHandler.cc
+ src/QueryHandler.cc
+ src/QueryMessage.cc
+ src/RSCommand.cc
+ src/SearchExpression.cc
+ src/Server.cc
+ src/VDMSConfig.cc
+ src/VideoCommand.cc
+ src/AutoDeleteNode.cc
+ src/ImageLoop.cc
+ ${PROTO_SRCS} ${PROTO_HDRS}
+)
+ target_link_libraries(dms vcl pmgd pmgd-util protobuf tbb tiledb vdms-utils pthread -lcurl -lzmq ${AWSSDK_LINK_LIBRARIES})
add_executable(vdms src/vdms.cc)
- target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS})
+ target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS} ${AWSSDK_LINK_LIBRARIES})
endif ()
diff --git a/INSTALL.md b/INSTALL.md
index b22c4e78..9348eb44 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -2,117 +2,113 @@
Here is the detailed process of installation of VDMS dependencies.
## Dependencies
-Here we will install the Ubuntu 20.04 and Python3 packages. We assume `python`/`pip` is an alias for `python3`/`pip3`. If your system has both Python 2 and Python 3, please replace all pip and python commands with pip3 and python3, respectively.
+To install VDMS, we must install the necessary dependencies via apt, github, and pip.
+
+### Install Debian Packages
+Here we will install the Debian and Python3 packages.
```bash
sudo apt-get update
-sudo apt-get -y install --no-install-recommends software-properties-common
-sudo add-apt-repository "deb http://security.ubuntu.com/ubuntu focal-security main"
-sudo apt-get -y install --no-install-recommends apt-transport-https autoconf automake bison build-essential \
- bzip2 ca-certificates curl=7.68.0-1ubuntu2.18 ed flex g++ git gnupg-agent javacc libarchive-tools \
- libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \
- libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \
- libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \
- libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \
- libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \
- libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \
- pkg-config python3-dev python3-pip unzip
-pip install --no-cache-dir "numpy>=1.23.2" "setuptools>=65.5.1"
-```
-### Clone/Download Dependencies
-Here we clone the repositories for grpc v1.40.0, libpng12, Swig v4.0.2, OpenCV 4.5.3, Valijson v0.6, CMake v3.21.2, Faiss v1.7.1, and FLINNG. Then download necesarry files for zlib v1.2.13, Json-simple v1.1.1, and TileDB v1.3.1.
-Here we assume `$VDMS_DEP_DIR` is the working directory for installing dependencies and `python` is Python 3.
+sudo apt-get install -y --no-install-suggests --no-install-recommends \
+ apt-transport-https autoconf automake bison build-essential bzip2 ca-certificates \
+ curl ed flex g++-9 gcc-9 git gnupg-agent javacc libarchive-tools libatlas-base-dev \
+ libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev libc-ares-dev libcurl4-openssl-dev \
+ libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgtk-3-dev libgtk2.0-dev \
+ libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \
+ liblz4-dev libopenblas-dev libopenmpi-dev libpng-dev librdkafka-dev libsnappy-dev libssl-dev \
+ libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev mpich \
+ openjdk-11-jdk-headless pkg-config procps python3-dev python3-pip software-properties-common \
+ swig unzip uuid-dev
+```
+Note: Your system may have g++ or gcc version 10+. If this is the case, please use version 9 to build VDMS. Optional method for setting version 9 as default:
+```bash
+update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 1
+update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 1
+```
+
+### Install Remaining Dependencies
+Here we assume `$VDMS_DEP_DIR` is the directory for installing additional dependencies.
+This directory is user-defined but here we use `/dependencies`.
+These instructions assume you have full permissions to your system.
+If not running as root, add `sudo` where necessary.
```bash
-cd $VDMS_DEP_DIR
-git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \
-git clone --branch v4.0.2 https://github.com/swig/swig.git && \
-git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \
-git clone https://github.com/tonyzhang617/FLINNG.git && \
-git clone --recurse-submodules -b v1.40.0 https://github.com/grpc/grpc.git && \
-git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \
-git clone --branch v0.6 https://github.com/tristanpenman/valijson.git
+VDMS_DEP_DIR=/dependencies # Set to any directory
+BUILD_THREADS="-j`nproc`"
+mkdir -p $VDMS_DEP_DIR
+```
-sudo curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \
-sudo curl -L -o 1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \
-sudo curl -L -o zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz
+#### Python3 Packages
+Here we will install the necessary Python3 packages Numpy and Protobuf 3.20.3.
+You can also install the coverage package if interested in running the Python unit tests.
+```bash
+PROTOBUF_VERSION="3.20.3"
+pip3 install --no-cache-dir "numpy>=1.25.1" "protobuf==${PROTOBUF_VERSION}" "coverage>=7.2.7"
```
-### Install Dependencies
-These instructions assume you have full permissions to your system.
-If running as root, remove `sudo` where necessary.
-#### CMAKE
+#### CMAKE v3.26.4
+VDMS requires CMake v3.21+. Here we install CMake v3.26.4.
```bash
-cd $VDMS_DEP_DIR/CMake && ./bootstrap
-make -j && sudo make install
+CMAKE_VERSION="v3.26.4"
+git clone --branch ${CMAKE_VERSION} https://github.com/Kitware/CMake.git $VDMS_DEP_DIR/CMake
+cd $VDMS_DEP_DIR/CMake
+./bootstrap
+make ${BUILD_THREADS}
+make install
```
-### Swig
+### gtest
+Unfortunately apt doesn't build gtest so you need to do the following:
```bash
-cd $VDMS_DEP_DIR/swig
-./autogen.sh && ./configure
-make -j && sudo make install
+cd /usr/src/gtest/
+cmake .
+make ${BUILD_THREADS}
+mv lib/libgtest* /usr/lib
```
-### Faiss
+### Faiss v1.7.3
```bash
+FAISS_VERSION="v1.7.3"
+git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git $VDMS_DEP_DIR/faiss
cd $VDMS_DEP_DIR/faiss
mkdir build && cd build
cmake -DFAISS_ENABLE_GPU=OFF ..
-make -j && sudo make install
+make ${BUILD_THREADS}
+make install
```
### FLINNG
```bash
+git clone https://github.com/tonyzhang617/FLINNG.git $VDMS_DEP_DIR/FLINNG
cd $VDMS_DEP_DIR/FLINNG
mkdir build && cd build
cmake ..
-make -j && sudo make install
+make ${BUILD_THREADS}
+make install
```
-### grpc
+### Protobuf 3.20.3
```bash
-cd $VDMS_DEP_DIR/grpc
-pip install --no-cache-dir -r requirements.txt
-GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install --no-cache-dir .
-
-cd tools/distrib/python/grpcio_tools
-python ../make_grpcio_tools.py
-GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install --no-cache-dir .
-
-cd $VDMS_DEP_DIR/grpc/third_party/zlib/ && mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
-make -j && sudo make install
-
-cd $VDMS_DEP_DIR/grpc/third_party/protobuf/cmake
-mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -Dprotobuf_BUILD_TESTS=OFF ..
-make -j && sudo make install
-
-cd ../../../abseil-cpp && mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
-make -j && sudo make install
-
-cd ../../re2/ && mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
-make -j && sudo make install
-
-cd $VDMS_DEP_DIR/grpc/cmake && mkdir build && cd build
-cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \
- -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \
- -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \
- -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../..
-make -j && sudo make install
+PROTOBUF_VERSION="3.20.3"
+curl -L -o ${VDMS_DEP_DIR}/${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz
+cd ${VDMS_DEP_DIR} && tar -xvf ${PROTOBUF_VERSION}.tar.gz
+cd protobuf-${PROTOBUF_VERSION}
+./autogen.sh
+./configure
+make ${BUILD_THREADS}
+make install
+ldconfig
```
-### [OpenCV](https://opencv.org/)
-
-Below are instructions for installing ***OpenCV v4.5.3***. It may also work with newer versions of OpenCV.
+### [OpenCV](https://opencv.org/) 4.5.5
+Below are instructions for installing ***OpenCV v4.5.5***.
```bash
+OPENCV_VERSION="4.5.5"
+git clone --branch ${OPENCV_VERSION} https://github.com/opencv/opencv.git $VDMS_DEP_DIR/opencv
cd $VDMS_DEP_DIR/opencv
mkdir build && cd build
-cmake -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF ..
-make -j
-sudo make install
+cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF ..
+make ${BUILD_THREADS}
+make install
```
**Note**: When using videos, and getting the following error: "Unable to stop the stream: Inappropriate ioctl for device", you may need to include more flags when compiling OpenCV. Follow these instructions ([source](https://stackoverflow.com/questions/41200201/opencv-unable-to-stop-the-stream-inappropriate-ioctl-for-device)):
@@ -124,51 +120,51 @@ cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF -D CMAKE_BUILD_TYPE=RELEASE -D
-D WITH_FFMPEG=ON -D WITH_TBB=ON -D WITH_GTK=ON \
-D WITH_V4L=ON -D WITH_OPENGL=ON -D WITH_CUBLAS=ON \
-DWITH_QT=OFF -DCUDA_NVCC_FLAGS="-D_FORCE_INLINES" ..
-make -j
+make ${BUILD_THREADS}
make install
```
-### Zlib
+### Valijson v0.6
+This is a headers-only library, no compilation/installation necessary
```bash
-cd $VDMS_DEP_DIR && tar -xvzf zlib-1.2.13.tar.gz
-cd zlib-1.2.13 && ./configure
-make -j && sudo make install
+VALIJSON_VERSION="v0.6"
+git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git $VDMS_DEP_DIR/valijson
+cd $VDMS_DEP_DIR/valijson
+cp -r include/* /usr/local/include/
```
-### [TileDB](https://tiledb.io/)
-VDMS works with ***TileDB v1.3.1.***
-The directions below will help you install TileDB v1.3.1 from the source.
-You can also follow the directions listed
-[here](https://docs.tiledb.io/en/latest/installation.html).
-```bash
-cd $VDMS_DEP_DIR && tar -xvf 1.3.1.tar.gz
-cd TileDB-1.3.1 && mkdir build && cd build
-../bootstrap --prefix=/usr/local/
-make -j && sudo make install-tiledb
-```
-### gtest
-Unfortunately apt doesn't build gtest;
-you need to do the following steps to get it to work correctly:
+### [TileDB](https://tiledb.io/) 2.14.1
+The directions below will help you install TileDB v2.14.1 from the source.
+You can also follow the directions listed [here](https://docs.tiledb.io/en/latest/installation.html).
```bash
-cd /usr/src/gtest/
-sudo cmake .
-sudo make -j
-sudo mv lib/libgtest* /usr/lib
+TILEDB_VERSION="2.14.1"
+curl -L -o $VDMS_DEP_DIR/${TILEDB_VERSION}.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/${TILEDB_VERSION}.tar.gz && \
+cd $VDMS_DEP_DIR
+tar -xvf ${TILEDB_VERSION}.tar.gz
+cd TileDB-${TILEDB_VERSION}
+mkdir build && cd build
+../bootstrap --prefix=/usr/local/
+make ${BUILD_THREADS}
+make install-tiledb
```
-### Valijson
-This is a headers-only library, no compilation/installation necessary
+### AWS SDK CPP 1.11.0
```bash
-cd $VDMS_DEP_DIR/valijson
-sudo cp -r include/* /usr/local/include
+AWS_SDK_VERSION="1.11.0"
+git clone -b ${AWS_SDK_VERSION} --recurse-submodules https://github.com/aws/aws-sdk-cpp ${VDMS_DEP_DIR}/aws-sdk-cpp
+mkdir -p ${VDMS_DEP_DIR}/aws-sdk-cpp/build
+cd ${VDMS_DEP_DIR}/aws-sdk-cpp/build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DBUILD_ONLY="s3" -DCUSTOM_MEMORY_MANAGEMENT=OFF
+make ${BUILD_THREADS}
+make install
```
## Install VDMS
This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories.
```bash
-git clone https://github.com/IntelLabs/vdms.git
-cd vdms && git checkout develop
+git clone -b develop https://github.com/IntelLabs/vdms.git
+cd vdms
git submodule update --init --recursive
```
@@ -186,3 +182,4 @@ mkdir build && cd build
cmake -DCMAKE_CXX_FLAGS='-DPM' ..
make -j
```
+
diff --git a/client/cpp/BoundingBoxQueryParser.h b/client/cpp/BoundingBoxQueryParser.h
index 77928157..7ba8d075 100644
--- a/client/cpp/BoundingBoxQueryParser.h
+++ b/client/cpp/BoundingBoxQueryParser.h
@@ -1,88 +1,92 @@
#include "CSVParserUtil.h"
namespace VDMS {
-class BoundingBoxQueryParser : public CSVParserUtil{
- private:
- vector rectangleKeys{"x","y","w","h"};
- void parseRectangle(string row,string queryType,Json::Value &aquery);
- public:
- VDMS::Response ParseAddBoundingBox(vector row, vector cols);
- // VDMS::Response ParseUpdateBoundingBox(vector row, vector& cols);
+class BoundingBoxQueryParser : public CSVParserUtil {
+private:
+ vector rectangleKeys{"x", "y", "w", "h"};
+ void parseRectangle(string row, string queryType, Json::Value &aquery);
+public:
+ VDMS::Response ParseAddBoundingBox(vector row, vector cols);
+ // VDMS::Response ParseUpdateBoundingBox(vector row, vector&
+ // cols);
};
-};
+}; // namespace VDMS
-VDMS::Response VDMS::BoundingBoxQueryParser::ParseAddBoundingBox(vector row, vector columnNames){
- if (row.empty() || row[0].empty()) {
- throw "please provide rectangle details";
- }
+VDMS::Response
+VDMS::BoundingBoxQueryParser::ParseAddBoundingBox(vector row,
+ vector columnNames) {
+ if (row.empty() || row[0].empty()) {
+ throw "please provide rectangle details";
+ }
- Json::Value aquery;
- Json::Value allquery;
- std::string command_name = "AddBoundingBox";
+ Json::Value aquery;
+ Json::Value allquery;
+ std::string command_name = "AddBoundingBox";
- aquery["AddBoundingBox"]["_ref"] = 1;
- // aquery["AddBoundingBox"]["image"] = 3;
+ aquery["AddBoundingBox"]["_ref"] = 1;
+ // aquery["AddBoundingBox"]["image"] = 3;
- Json::Value aqueryf;
- // aqueryf["FindImage"]["_ref"] = 3;
- bool cons = false;
+ Json::Value aqueryf;
+ // aqueryf["FindImage"]["_ref"] = 3;
+ bool cons = false;
- parseRectangle(row[0], "AddBoundingBox", aquery);
+ parseRectangle(row[0], "AddBoundingBox", aquery);
- for (int j = 1; j < columnNames.size(); j++){
- const string& columnName = columnNames[j];
- const string& cellValue = row[j];
+ for (int j = 1; j < columnNames.size(); j++) {
+ const string &columnName = columnNames[j];
+ const string &cellValue = row[j];
- if (cellValue.empty()) {
- continue;
- }
+ if (cellValue.empty()) {
+ continue;
+ }
- if (columnName.find("prop_") != string::npos){
- parseProperty(columnName, cellValue, command_name, aquery);
- }
- else if (columnName.find("cons_") != string::npos){
- std::string find_image = "FindImage";
- parseConstraints(columnName, cellValue, find_image, aqueryf);
- cons = true;
- }
+ if (columnName.find("prop_") != string::npos) {
+ parseProperty(columnName, cellValue, command_name, aquery);
+ } else if (columnName.find("cons_") != string::npos) {
+ std::string find_image = "FindImage";
+ parseConstraints(columnName, cellValue, find_image, aqueryf);
+ cons = true;
}
+ }
- if (cons)
- allquery.append(aqueryf);
+ if (cons)
+ allquery.append(aqueryf);
- allquery.append(aquery);
- // std::cout< row, vector& columnNames){
+// VDMS::Response
+// VDMS::BoundingBoxQueryParser::ParseUpdateBoundingBox(vector row,
+// vector& columnNames){
// Json::Value aquery;
// Json::Value allquery;
// aquery["UpdateBoundingBox"]["_ref"]=3;
diff --git a/client/cpp/CSVParser.h b/client/cpp/CSVParser.h
index eac01aee..401ae8f1 100644
--- a/client/cpp/CSVParser.h
+++ b/client/cpp/CSVParser.h
@@ -1,73 +1,79 @@
-#include
+#include "CSVParserUtil.h"
+#include "rapidcsv.h"
#include
-#include
+#include
+#include
#include
#include
-#include
-#include "rapidcsv.h"
-#include "CSVParserUtil.h"
+#include
namespace VDMS {
class CSVParser {
public:
- CSVParser(std::string filename, size_t num_threads, std::string server, int port) : m_filename(filename), m_num_threads(num_threads),vdms_server(server),vdms_port(port) {}
- std::vector parse_csv_lines(const std::string& filename, int start_line, int end_line, std::mutex& mutex, std::vector& all_results, const size_t thread_id)
- {
+ CSVParser(std::string filename, size_t num_threads, std::string server,
+ int port)
+ : m_filename(filename), m_num_threads(num_threads), vdms_server(server),
+ vdms_port(port) {}
+ std::vector
+ parse_csv_lines(const std::string &filename, int start_line, int end_line,
+ std::mutex &mutex, std::vector &all_results,
+ const size_t thread_id) {
rapidcsv::Document csv(filename);
size_t rowCount = csv.GetRowCount();
std::vector columnNames = csv.GetColumnNames();
- VDMS::CSVParserUtil csv_util(vdms_server, vdms_port, columnNames, static_cast(thread_id));
- for (int i = start_line; i < end_line; ++i)
- {
- std::vector row = csv.GetRow(i);
- VDMS::Response result = csv_util.parse_row(row);
+ VDMS::CSVParserUtil csv_util(vdms_server, vdms_port, columnNames,
+ static_cast(thread_id));
+ for (int i = start_line; i < end_line; ++i) {
+ std::vector row = csv.GetRow(i);
+ VDMS::Response result = csv_util.parse_row(row);
- std::lock_guard lock(mutex);
- all_results.push_back(result);
+ std::lock_guard lock(mutex);
+ all_results.push_back(result);
}
return all_results;
}
-std::vector parse() {
- auto start = std::chrono::high_resolution_clock::now();
+ std::vector parse() {
+ auto start = std::chrono::high_resolution_clock::now();
- std::ifstream file(m_filename);
- if (!file) {
- std::cerr << "Error opening file\n";
-
- }
-
- int num_lines = std::count(std::istreambuf_iterator(file), std::istreambuf_iterator(), '\n');
- std::size_t lines_per_thread = num_lines / m_num_threads;
-
- std::mutex mutex;
- std::vector threads;
- std::vector all_results;
+ std::ifstream file(m_filename);
+ if (!file) {
+ std::cerr << "Error opening file\n";
+ }
- for (size_t i = 0; i < m_num_threads; i++) {
- size_t start_line = i * lines_per_thread;
- size_t end_line = (i == m_num_threads - 1) ? num_lines-1 : (i + 1) * lines_per_thread;
+ int num_lines = std::count(std::istreambuf_iterator(file),
+ std::istreambuf_iterator(), '\n');
+ std::size_t lines_per_thread = num_lines / m_num_threads;
- threads.emplace_back(&CSVParser::parse_csv_lines, this, std::ref(m_filename), start_line, end_line, std::ref(mutex),std::ref(all_results), i);
- }
+ std::mutex mutex;
+ std::vector threads;
+ std::vector all_results;
- for (auto& thread : threads) {
- thread.join();
- }
+ for (size_t i = 0; i < m_num_threads; i++) {
+ size_t start_line = i * lines_per_thread;
+ size_t end_line =
+ (i == m_num_threads - 1) ? num_lines - 1 : (i + 1) * lines_per_thread;
- auto finish = std::chrono::high_resolution_clock::now();
- std::chrono::duration elapsed = finish - start;
- //std::cout << "Elapsed time: " << elapsed.count() << " s\n";
- return all_results;
+ threads.emplace_back(&CSVParser::parse_csv_lines, this,
+ std::ref(m_filename), start_line, end_line,
+ std::ref(mutex), std::ref(all_results), i);
}
-private:
- std::string m_filename;
- size_t m_num_threads;
- std::string vdms_server;
- int vdms_port;
+ for (auto &thread : threads) {
+ thread.join();
+ }
+ auto finish = std::chrono::high_resolution_clock::now();
+ std::chrono::duration elapsed = finish - start;
+ // std::cout << "Elapsed time: " << elapsed.count() << " s\n";
+ return all_results;
+ }
- };
-};
\ No newline at end of file
+private:
+ std::string m_filename;
+ size_t m_num_threads;
+ std::string vdms_server;
+ int vdms_port;
+};
+}; // namespace VDMS
\ No newline at end of file
diff --git a/client/cpp/CSVParserUtil.cpp b/client/cpp/CSVParserUtil.cpp
index 298345c5..d5b10dbe 100644
--- a/client/cpp/CSVParserUtil.cpp
+++ b/client/cpp/CSVParserUtil.cpp
@@ -1,15 +1,15 @@
-#include
-#include
#include "CSVParserUtil.h"
-#include "rapidcsv.h"
+#include "BoundingBoxQueryParser.h"
+#include "ConnectionQueryParser.h"
+#include "DescriptorQueryParser.h"
+#include "DescriptorSetQueryParser.h"
#include "EntityQueryParser.h"
#include "ImageQueryParser.h"
#include "VideoQueryParser.h"
-#include "DescriptorQueryParser.h"
-#include "DescriptorSetQueryParser.h"
-#include "BoundingBoxQueryParser.h"
-#include "ConnectionQueryParser.h"
+#include "rapidcsv.h"
+#include
+#include
#include
static std::mutex barrier;
std::mutex mtx;
@@ -17,619 +17,531 @@ using namespace std;
using namespace std::chrono;
using namespace VDMS;
-VDMS::CSVParserUtil::CSVParserUtil() : vdms_server("localhost"), vdms_port(55558),
- vdms_client(std::unique_ptr(new VDMS::VDMSClient(vdms_server, vdms_port)))
-{
- initCommandsMap();
+VDMS::CSVParserUtil::CSVParserUtil()
+ : vdms_server("localhost"), vdms_port(55558),
+ vdms_client(std::unique_ptr(
+ new VDMS::VDMSClient(vdms_server, vdms_port))) {
+ initCommandsMap();
}
-VDMS::CSVParserUtil::CSVParserUtil(const std::string &server, int port, const std::vector columnNames, int index) : vdms_server(server),
- vdms_port(port),
- _columnNames(columnNames),
- id(index), vdms_client(std::unique_ptr(new VDMS::VDMSClient(vdms_server, vdms_port)))
-{
- initCommandsMap();
+VDMS::CSVParserUtil::CSVParserUtil(const std::string &server, int port,
+ const std::vector columnNames,
+ int index)
+ : vdms_server(server), vdms_port(port), _columnNames(columnNames),
+ id(index), vdms_client(std::unique_ptr(
+ new VDMS::VDMSClient(vdms_server, vdms_port))) {
+ initCommandsMap();
}
-void VDMS::CSVParserUtil::initCommandsMap()
-{
- commands = {
- {"EntityClass", EntityClass},
- {"ConnectionClass", ConnectionClass},
- {"ImagePath", ImagePath},
- {"VideoPath", VideoPath},
- {"DescriptorType", DescriptorType},
- {"DescriptorClass", DescriptorClass},
- {"RectangleBound", RectangleBound},
- {"EntityUpdate", EntityUpdate},
- {"ConnectionUpdate", ConnectionUpdate},
- {"ImageUpdate", ImageUpdate},
- {"RectangleUpdate", RectangleUpdate}};
+void VDMS::CSVParserUtil::initCommandsMap() {
+ commands = {{"EntityClass", EntityClass},
+ {"ConnectionClass", ConnectionClass},
+ {"ImagePath", ImagePath},
+ {"VideoPath", VideoPath},
+ {"DescriptorType", DescriptorType},
+ {"DescriptorClass", DescriptorClass},
+ {"RectangleBound", RectangleBound},
+ {"EntityUpdate", EntityUpdate},
+ {"ConnectionUpdate", ConnectionUpdate},
+ {"ImageUpdate", ImageUpdate},
+ {"RectangleUpdate", RectangleUpdate}};
}
-string VDMS::CSVParserUtil::function_accessing_columnNames(int i)
-{
- std::lock_guard lock(mtx);
+string VDMS::CSVParserUtil::function_accessing_columnNames(int i) {
+ std::lock_guard lock(mtx);
- return _columnNames[i];
+ return _columnNames[i];
}
-VDMS::Response VDMS::CSVParserUtil::parse_row(std::vector &row)
-{
- VDMS::Response result;
-
- VDMS::CSVParserUtil::commandType queryType = get_query_type(_columnNames[0]);
- switch (queryType)
- {
- case commandType::AddEntity:
- {
- EntityQueryParser entityquery;
- result = entityquery.ParseAddEntity(row, _columnNames);
- }
-
- break;
- case commandType::AddConnection:
- {
-
- ConnectionQueryParser connectionquery;
- result = connectionquery.ParseAddConnection(row, _columnNames);
- }
- break;
-
- case commandType::AddImage:
- {
- ImageQueryParser imagequery;
- result = imagequery.ParseAddImage(row, _columnNames);
- }
- break;
- case commandType::AddVideo:
- {
- VideoQueryParser videoquery;
- result = videoquery.ParseAddVideo(row, _columnNames);
- }
- break;
- case commandType::AddDescriptorSet:
- {
- DescriptorSetQueryParser descriptorsetquery;
-
- result = descriptorsetquery.ParseAddDescriptorSet(row, _columnNames);
- }
- break;
- case commandType::AddDescriptor:
- {
- DescriptorQueryParser descriptorquery;
- result = descriptorquery.ParseAddDescriptor(row, _columnNames, id);
- }
- break;
- case commandType::AddBoundingBox:
- {
- BoundingBoxQueryParser boundingboxquery;
- result = boundingboxquery.ParseAddBoundingBox(row, _columnNames);
- }
- break;
- // case commandType::UpdateEntity:{
- // EntityQueryParser update_entityquery;
- // update_entityquery.ParseUpdateEntity(row, _columnNames);
- // }
- // break;
- // case commandType::UpdateConnection:{
- // ConnectionQueryParser update_connectionquery;
- // update_connectionquery.ParseUpdateConnection(row,allquery,i+1,_columnNames);
- // }
- // break;
- // case commandType::UpdateImage:{
- // ImageQueryParser update_image_query;
- // update_image_query.ParseUpdateImage(row,allquery,i+1,_columnNames);
- // }
- // break;
- // case commandType::UpdateBoundingBox:{
- // BoundingBoxQueryParser update_boundingboxquery;
- // update_boundingboxquery.ParseUpdateBoundingBox(row,allquery,i+1,_columnNames);
- // }
- // break;
- case commandType::UNKNOWN:{
- Json::Value results;
- results["status"] = -1;
- results["info"] = "Command does not exist";
- result.json = results.toStyledString();
- }
- break;
- }
- return result;
+VDMS::Response VDMS::CSVParserUtil::parse_row(std::vector &row) {
+ VDMS::Response result;
+
+ VDMS::CSVParserUtil::commandType queryType = get_query_type(_columnNames[0]);
+ switch (queryType) {
+ case commandType::AddEntity: {
+ EntityQueryParser entityquery;
+ result = entityquery.ParseAddEntity(row, _columnNames);
+ }
+
+ break;
+ case commandType::AddConnection: {
+
+ ConnectionQueryParser connectionquery;
+ result = connectionquery.ParseAddConnection(row, _columnNames);
+ } break;
+
+ case commandType::AddImage: {
+ ImageQueryParser imagequery;
+ result = imagequery.ParseAddImage(row, _columnNames);
+ } break;
+ case commandType::AddVideo: {
+ VideoQueryParser videoquery;
+ result = videoquery.ParseAddVideo(row, _columnNames);
+ } break;
+ case commandType::AddDescriptorSet: {
+ DescriptorSetQueryParser descriptorsetquery;
+
+ result = descriptorsetquery.ParseAddDescriptorSet(row, _columnNames);
+ } break;
+ case commandType::AddDescriptor: {
+ DescriptorQueryParser descriptorquery;
+ result = descriptorquery.ParseAddDescriptor(row, _columnNames, id);
+ } break;
+ case commandType::AddBoundingBox: {
+ BoundingBoxQueryParser boundingboxquery;
+ result = boundingboxquery.ParseAddBoundingBox(row, _columnNames);
+ } break;
+ // case commandType::UpdateEntity:{
+ // EntityQueryParser update_entityquery;
+ // update_entityquery.ParseUpdateEntity(row, _columnNames);
+ // }
+ // break;
+ // case commandType::UpdateConnection:{
+ // ConnectionQueryParser update_connectionquery;
+ // update_connectionquery.ParseUpdateConnection(row,allquery,i+1,_columnNames);
+ // }
+ // break;
+ // case commandType::UpdateImage:{
+ // ImageQueryParser update_image_query;
+ // update_image_query.ParseUpdateImage(row,allquery,i+1,_columnNames);
+ // }
+ // break;
+ // case commandType::UpdateBoundingBox:{
+ // BoundingBoxQueryParser update_boundingboxquery;
+ // update_boundingboxquery.ParseUpdateBoundingBox(row,allquery,i+1,_columnNames);
+ // }
+ // break;
+ case commandType::UNKNOWN: {
+ Json::Value results;
+ results["status"] = -1;
+ results["info"] = "Command does not exist";
+ result.json = results.toStyledString();
+ } break;
+ }
+ return result;
}
-int VDMS::CSVParserUtil::isBool(const std::string &s)
-{
- std::string lower = s;
- std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower);
- if (lower == "true")
- return 1;
- else if (lower == "false")
- return 2;
- return 0;
+int VDMS::CSVParserUtil::isBool(const std::string &s) {
+ std::string lower = s;
+ std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower);
+ if (lower == "true")
+ return 1;
+ else if (lower == "false")
+ return 2;
+ return 0;
}
-bool VDMS::CSVParserUtil::isFloat(const std::string &s)
-{
- std::istringstream ss(s);
- float x;
- char c;
- return (ss >> x) && !(ss >> c);
+bool VDMS::CSVParserUtil::isFloat(const std::string &s) {
+ std::istringstream ss(s);
+ float x;
+ char c;
+ return (ss >> x) && !(ss >> c);
}
-bool VDMS::CSVParserUtil::isInt(const std::string &s)
-{
- std::istringstream ss(s);
- int x;
- char c;
- return (ss >> x) && (floor(x)) && !(ss >> c);
+bool VDMS::CSVParserUtil::isInt(const std::string &s) {
+ std::istringstream ss(s);
+ int x;
+ char c;
+ return (ss >> x) && (floor(x)) && !(ss >> c);
}
-VDMS::CSVParserUtil::commandType VDMS::CSVParserUtil::get_query_type(const string &str)
-{
- CSVParserUtil::commandType querytype = commandType::UNKNOWN;
-
- std::lock_guard lock(CSVParserUtil::querytype_mutex);
- std::map::iterator iter;
- iter = commands.find(str);
- if (iter != commands.end()) {
- switch (commands[str])
- {
- case EntityClass:
- querytype = commandType::AddEntity;
- break;
- case ConnectionClass:
- querytype = commandType::AddConnection;
- break;
- case ImagePath:
- querytype = commandType::AddImage;
- break;
- case VideoPath:
- querytype = commandType::AddVideo;
- break;
- case DescriptorType:
- querytype = commandType::AddDescriptorSet;
- break;
- case DescriptorClass:
- querytype = commandType::AddDescriptor;
- break;
- case RectangleBound:
- querytype = commandType::AddBoundingBox;
- break;
- }
- // std::cout << " I executed queryType "<< querytype << std::endl;
- // return querytype;
- }
-
- return querytype;
+VDMS::CSVParserUtil::commandType
+VDMS::CSVParserUtil::get_query_type(const string &str) {
+ CSVParserUtil::commandType querytype = commandType::UNKNOWN;
+
+ std::lock_guard lock(CSVParserUtil::querytype_mutex);
+ std::map::iterator iter;
+ iter = commands.find(str);
+ if (iter != commands.end()) {
+ switch (commands[str]) {
+ case EntityClass:
+ querytype = commandType::AddEntity;
+ break;
+ case ConnectionClass:
+ querytype = commandType::AddConnection;
+ break;
+ case ImagePath:
+ querytype = commandType::AddImage;
+ break;
+ case VideoPath:
+ querytype = commandType::AddVideo;
+ break;
+ case DescriptorType:
+ querytype = commandType::AddDescriptorSet;
+ break;
+ case DescriptorClass:
+ querytype = commandType::AddDescriptor;
+ break;
+ case RectangleBound:
+ querytype = commandType::AddBoundingBox;
+ break;
+ }
+ // std::cout << " I executed queryType "<< querytype << std::endl;
+ // return querytype;
+ }
+
+ return querytype;
}
-VDMS::CSVParserUtil::DATATYPE VDMS::CSVParserUtil::getDataType(const string &str, const string &propname)
-{
- if (propname.substr(0, 5) == "date:")
- return DATATYPE::DATE;
- else if (isInt(str))
- return DATATYPE::INTEGER;
- else if (isFloat(str))
- return DATATYPE::FLOAT;
- else if (isBool(str) == 1)
- return DATATYPE::TRUE;
- else if (isBool(str) == 2)
- return DATATYPE::FALSE;
- else
- return DATATYPE::STRING;
+VDMS::CSVParserUtil::DATATYPE
+VDMS::CSVParserUtil::getDataType(const string &str, const string &propname) {
+ if (propname.substr(0, 5) == "date:")
+ return DATATYPE::DATE;
+ else if (isInt(str))
+ return DATATYPE::INTEGER;
+ else if (isFloat(str))
+ return DATATYPE::FLOAT;
+ else if (isBool(str) == 1)
+ return DATATYPE::TRUE;
+ else if (isBool(str) == 2)
+ return DATATYPE::FALSE;
+ else
+ return DATATYPE::STRING;
}
-void VDMS::CSVParserUtil::parseProperty(const string &columnNames, const string &row, const string &queryType, Json::Value &aquery)
-{
- std::lock_guard lock(CSVParserUtil::aquery_mutex);
- string propname = columnNames.substr(5, string::npos);
- // std::cout << "Inside parseProp " << propname < lock(CSVParserUtil::aquery_mutex);
+ string propname = columnNames.substr(5, string::npos);
+ // std::cout << "Inside parseProp " << propname < lock(CSVParserUtil::cons_mutex);
- vector consvals = spiltrow(row);
- string consname = consvals[0];
- if (consname.substr(0, 5) == "date:")
- {
- for (int z = 1; z < consvals.size(); z++)
- {
- if (z % 2 == 1)
- {
- aquery[queryType]["constraints"][consname.substr(5, string::npos)].append(consvals[z]);
- }
- else
- {
- Json::Value date;
- date["_date"] = consvals[z];
- aquery[queryType]["constraints"][consname.substr(5, string::npos)].append(date);
- }
- }
- }
- else
- {
- for (int z = 1; z < consvals.size(); z++)
- {
- CSVParserUtil::DATATYPE dtype = getDataType(consvals[z], consname);
- if (dtype == DATATYPE::TRUE)
- {
- aquery[queryType]["constraints"][consname].append(true);
- }
- else if (dtype == DATATYPE::FALSE)
- {
- aquery[queryType]["constraints"][consname].append(false);
- }
- else if (dtype == DATATYPE::INTEGER)
- {
- aquery[queryType]["constraints"][consname].append(stoi(consvals[z]));
- }
- else if (dtype == DATATYPE::FLOAT)
- {
- aquery[queryType]["constraints"][consname].append(stof(consvals[z]));
- }
- else
- {
- aquery[queryType]["constraints"][consname].append(consvals[z]);
- }
- }
- }
+void VDMS::CSVParserUtil::parseConstraints(const string &columnNames,
+ const string &row, string &queryType,
+ Json::Value &aquery) {
+ std::lock_guard lock(CSVParserUtil::cons_mutex);
+ vector consvals = spiltrow(row);
+ string consname = consvals[0];
+ if (consname.substr(0, 5) == "date:") {
+ for (int z = 1; z < consvals.size(); z++) {
+ if (z % 2 == 1) {
+ aquery[queryType]["constraints"][consname.substr(5, string::npos)]
+ .append(consvals[z]);
+ } else {
+ Json::Value date;
+ date["_date"] = consvals[z];
+ aquery[queryType]["constraints"][consname.substr(5, string::npos)]
+ .append(date);
+ }
+ }
+ } else {
+ for (int z = 1; z < consvals.size(); z++) {
+ CSVParserUtil::DATATYPE dtype = getDataType(consvals[z], consname);
+ if (dtype == DATATYPE::TRUE) {
+ aquery[queryType]["constraints"][consname].append(true);
+ } else if (dtype == DATATYPE::FALSE) {
+ aquery[queryType]["constraints"][consname].append(false);
+ } else if (dtype == DATATYPE::INTEGER) {
+ aquery[queryType]["constraints"][consname].append(stoi(consvals[z]));
+ } else if (dtype == DATATYPE::FLOAT) {
+ aquery[queryType]["constraints"][consname].append(stof(consvals[z]));
+ } else {
+ aquery[queryType]["constraints"][consname].append(consvals[z]);
+ }
+ }
+ }
}
-vector VDMS::CSVParserUtil::spiltrow(const string &str)
-{
- string row = str;
- vector rowv;
- int start = 0;
- for (int i = 0; i < row.size(); i++)
- {
- if ((row[i] == '<') || (row[i] == '>') || (row[i] == '='))
- {
- if (row[i + 1] == '=')
- {
- rowv.push_back(row.substr(start, i - start));
- rowv.push_back(row.substr(i, 2));
- i++;
- }
- else
- {
- rowv.push_back(row.substr(start, i - start));
- rowv.push_back(row.substr(i, 1));
- }
- start = i + 1;
- }
- else if (row[i] == ',')
- {
- row.erase(i, 1);
- i--;
- }
- }
- rowv.push_back(row.substr(start, string::npos));
- return rowv;
+vector VDMS::CSVParserUtil::spiltrow(const string &str) {
+ string row = str;
+ vector rowv;
+ int start = 0;
+ for (int i = 0; i < row.size(); i++) {
+ if ((row[i] == '<') || (row[i] == '>') || (row[i] == '=')) {
+ if (row[i + 1] == '=') {
+ rowv.push_back(row.substr(start, i - start));
+ rowv.push_back(row.substr(i, 2));
+ i++;
+ } else {
+ rowv.push_back(row.substr(start, i - start));
+ rowv.push_back(row.substr(i, 1));
+ }
+ start = i + 1;
+ } else if (row[i] == ',') {
+ row.erase(i, 1);
+ i--;
+ }
+ }
+ rowv.push_back(row.substr(start, string::npos));
+ return rowv;
}
-void VDMS::CSVParserUtil::parseOperations(string columnNames, string row, string queryType, Json::Value &aquery)
-{
-
- std::lock_guard lock(CSVParserUtil::ops_mutex);
- string type = columnNames.substr(4, string::npos);
- Json::Value opsjson;
- vector opsKeys;
- if (isValidOpsType(type))
- opsjson["type"] = type;
- else
- throw "invalid operation command name";
- vector rowvec;
- int c;
- splitRowOnComma(row, rowvec);
- if (type == "crop")
- {
- if (rowvec.size() <= 3 || rowvec.size() > 4)
- throw "For crop data should be of size 4";
- opsKeys = {"x", "y", "width", "height"};
- for (c = 0; c < rowvec.size(); c++)
- {
- string substr = rowvec[c];
- DATATYPE dType = isValidDataType(substr, 2);
-
- if (dType == DATATYPE::INTEGER)
- opsjson[opsKeys[c]] = stoi(substr);
- else if (dType == DATATYPE::FLOAT)
- opsjson[opsKeys[c]] = stof(substr);
-
- else
- {
- throw "Numeric data is required for crop command";
- }
- }
- }
-
- else if (type == "threshold")
- {
- DATATYPE dType = isValidDataType(row, 2);
+void VDMS::CSVParserUtil::parseOperations(string columnNames, string row,
+ string queryType,
+ Json::Value &aquery) {
+
+ std::lock_guard lock(CSVParserUtil::ops_mutex);
+ string type = columnNames.substr(4, string::npos);
+ Json::Value opsjson;
+ vector opsKeys;
+ if (isValidOpsType(type))
+ opsjson["type"] = type;
+ else
+ throw "invalid operation command name";
+ vector rowvec;
+ int c;
+ splitRowOnComma(row, rowvec);
+ if (type == "crop") {
+ if (rowvec.size() <= 3 || rowvec.size() > 4)
+ throw "For crop data should be of size 4";
+ opsKeys = {"x", "y", "width", "height"};
+ for (c = 0; c < rowvec.size(); c++) {
+ string substr = rowvec[c];
+ DATATYPE dType = isValidDataType(substr, 2);
+
+ if (dType == DATATYPE::INTEGER)
+ opsjson[opsKeys[c]] = stoi(substr);
+ else if (dType == DATATYPE::FLOAT)
+ opsjson[opsKeys[c]] = stof(substr);
+
+ else {
+ throw "Numeric data is required for crop command";
+ }
+ }
+ }
+
+ else if (type == "threshold") {
+ DATATYPE dType = isValidDataType(row, 2);
+ if (dType == DATATYPE::INTEGER)
+ opsjson["value"] = stoi(row);
+ else if (dType == DATATYPE::FLOAT)
+ opsjson["value"] = stof(row);
+
+ else {
+ throw "Numeric data is required for threshold command";
+ }
+ }
+
+ else if (type == "resize") {
+ if (rowvec.size() <= 1 || rowvec.size() > 2)
+ throw "For resize data should be of size 2";
+ opsKeys = {"width", "height"};
+ for (c = 0; c < rowvec.size(); c++) {
+ string substr = rowvec[c];
+ DATATYPE dType = isValidDataType(substr, 2);
+
+ if (dType == DATATYPE::INTEGER)
+ opsjson[opsKeys[c]] = stoi(substr);
+ else if (dType == DATATYPE::FLOAT)
+ opsjson[opsKeys[c]] = stof(substr);
+
+ else {
+ throw "Numeric data is required for resize command";
+ }
+ }
+ }
+
+ else if (type == "flip") {
+ DATATYPE dType = isValidDataType(row, 2);
+ if (dType == DATATYPE::INTEGER) {
+ opsjson["code"] = stoi(row);
+ } else {
+ throw "Numeric data is required for flip command";
+ }
+ }
+
+ else if (type == "rotate") {
+ if (rowvec.size() <= 1 || rowvec.size() > 2)
+ throw "For rotate data should be of size 2";
+ opsKeys = {"angle", "resize"};
+ for (c = 0; c < rowvec.size(); c++) {
+ string substr = rowvec[c];
+ if (c == 0) {
+ DATATYPE dType = isValidDataType(substr, 2);
if (dType == DATATYPE::INTEGER)
- opsjson["value"] = stoi(row);
+ opsjson[opsKeys[c]] = stoi(substr);
else if (dType == DATATYPE::FLOAT)
- opsjson["value"] = stof(row);
-
- else
- {
- throw "Numeric data is required for threshold command";
- }
- }
+ opsjson[opsKeys[c]] = stof(substr);
- else if (type == "resize")
- {
- if (rowvec.size() <= 1 || rowvec.size() > 2)
- throw "For resize data should be of size 2";
- opsKeys = {"width", "height"};
- for (c = 0; c < rowvec.size(); c++)
- {
- string substr = rowvec[c];
- DATATYPE dType = isValidDataType(substr, 2);
-
- if (dType == DATATYPE::INTEGER)
- opsjson[opsKeys[c]] = stoi(substr);
- else if (dType == DATATYPE::FLOAT)
- opsjson[opsKeys[c]] = stof(substr);
-
- else
- {
- throw "Numeric data is required for resize command";
- }
+ else {
+ throw "Numeric data is required for rotate angle";
}
- }
-
- else if (type == "flip")
- {
- DATATYPE dType = isValidDataType(row, 2);
- if (dType == DATATYPE::INTEGER)
- {
- opsjson["code"] = stoi(row);
- }
- else
- {
- throw "Numeric data is required for flip command";
- }
- }
-
- else if (type == "rotate")
- {
- if (rowvec.size() <= 1 || rowvec.size() > 2)
- throw "For rotate data should be of size 2";
- opsKeys = {"angle", "resize"};
- for (c = 0; c < rowvec.size(); c++)
- {
- string substr = rowvec[c];
- if (c == 0)
- {
- DATATYPE dType = isValidDataType(substr, 2);
- if (dType == DATATYPE::INTEGER)
- opsjson[opsKeys[c]] = stoi(substr);
- else if (dType == DATATYPE::FLOAT)
- opsjson[opsKeys[c]] = stof(substr);
-
- else
- {
- throw "Numeric data is required for rotate angle";
- }
- }
- else if (c == 1)
- {
- DATATYPE dType = isValidDataType(substr, 1);
- if (dType == DATATYPE::TRUE)
- opsjson[opsKeys[c]] = true;
- else if (dType == DATATYPE::FALSE)
- opsjson[opsKeys[c]] = false;
-
- else
- {
- throw "Boolean data is required for rotate resize";
- }
- }
- }
- }
-
- else if (type == "interval")
- {
- if (rowvec.size() <= 2 || rowvec.size() > 3)
- throw "interval operation has 3 values to specify";
- opsKeys = {"start", "stop", "step"};
- for (c = 0; c < rowvec.size(); c++)
- {
- string substr = rowvec[c];
- DATATYPE dType = isValidDataType(substr, 2);
- if (dType == DATATYPE::INTEGER)
- {
- opsjson[opsKeys[c]] = stoi(substr);
- }
- else
- {
- throw "Numeric datatype is required for the interval";
- }
+ } else if (c == 1) {
+ DATATYPE dType = isValidDataType(substr, 1);
+ if (dType == DATATYPE::TRUE)
+ opsjson[opsKeys[c]] = true;
+ else if (dType == DATATYPE::FALSE)
+ opsjson[opsKeys[c]] = false;
+
+ else {
+ throw "Boolean data is required for rotate resize";
}
- }
-
- aquery[queryType]["operations"].append(opsjson);
+ }
+ }
+ }
+
+ else if (type == "interval") {
+ if (rowvec.size() <= 2 || rowvec.size() > 3)
+ throw "interval operation has 3 values to specify";
+ opsKeys = {"start", "stop", "step"};
+ for (c = 0; c < rowvec.size(); c++) {
+ string substr = rowvec[c];
+ DATATYPE dType = isValidDataType(substr, 2);
+ if (dType == DATATYPE::INTEGER) {
+ opsjson[opsKeys[c]] = stoi(substr);
+ } else {
+ throw "Numeric datatype is required for the interval";
+ }
+ }
+ }
+
+ aquery[queryType]["operations"].append(opsjson);
}
-void VDMS::CSVParserUtil::splitRowOnComma(const std::string &row, std::vector &rowvec)
-{
- std::string::size_type start = 0;
- while (start != std::string::npos)
- {
- std::string::size_type end = row.find(',', start);
- if (end == std::string::npos)
- {
- if (start < row.size())
- {
- rowvec.emplace_back(std::move(row.substr(start)));
- }
- break;
- }
- if (end > start)
- {
- rowvec.emplace_back(std::move(row.substr(start, end - start)));
- }
- start = end + 1;
- }
+void VDMS::CSVParserUtil::splitRowOnComma(const std::string &row,
+ std::vector &rowvec) {
+ std::string::size_type start = 0;
+ while (start != std::string::npos) {
+ std::string::size_type end = row.find(',', start);
+ if (end == std::string::npos) {
+ if (start < row.size()) {
+ rowvec.emplace_back(std::move(row.substr(start)));
+ }
+ break;
+ }
+ if (end > start) {
+ rowvec.emplace_back(std::move(row.substr(start, end - start)));
+ }
+ start = end + 1;
+ }
}
-VDMS::CSVParserUtil::DATATYPE VDMS::CSVParserUtil ::isValidDataType(string data, int type)
-{
- CSVParserUtil::DATATYPE actualtype = getDataType(data, "");
- return actualtype;
-
- // if(type==2 && (actualt=3 || acype=tualtype==4))//2 is for num
- // return actualtype;
- // else if(type==1 && (actualtype==1 || actualtype==2))//1 is for bool
- // return actualtype;
- // else
- // return -1;
+VDMS::CSVParserUtil::DATATYPE VDMS::CSVParserUtil ::isValidDataType(string data,
+ int type) {
+ CSVParserUtil::DATATYPE actualtype = getDataType(data, "");
+ return actualtype;
+
+ // if(type==2 && (actualt=3 || acype=tualtype==4))//2 is for num
+ // return actualtype;
+ // else if(type==1 && (actualtype==1 || actualtype==2))//1 is for bool
+ // return actualtype;
+ // else
+ // return -1;
}
-bool VDMS::CSVParserUtil::isValidOpsType(string &type)
-{
- if (type == "resize" || type == "threshold" || type == "flip" || type == "rotate" || type == "interval" || type == "crop")
- return true;
- return false;
+bool VDMS::CSVParserUtil::isValidOpsType(string &type) {
+ if (type == "resize" || type == "threshold" || type == "flip" ||
+ type == "rotate" || type == "interval" || type == "crop")
+ return true;
+ return false;
}
-void VDMS::CSVParserUtil::parseBlobFile(const std::string &filename, std::string **descriptor_ptr)
-{
- std::vector v;
-
- std::ifstream input(filename);
- if (!input.is_open())
- {
- // handle error if file cannot be opened
- std::cerr << "Error: Could not open file " << filename << std::endl;
- *descriptor_ptr = nullptr;
- return;
- }
+void VDMS::CSVParserUtil::parseBlobFile(const std::string &filename,
+ std::string **descriptor_ptr) {
+ std::vector v;
- std::string str;
- input >> str;
+ std::ifstream input(filename);
+ if (!input.is_open()) {
+ // handle error if file cannot be opened
+ std::cerr << "Error: Could not open file " << filename << std::endl;
+ *descriptor_ptr = nullptr;
+ return;
+ }
- std::stringstream ss(str);
- while (ss.good())
- {
- std::string substr;
- getline(ss, substr, ';');
- v.push_back(std::stof(substr));
- }
+ std::string str;
+ input >> str;
- input.close();
+ std::stringstream ss(str);
+ while (ss.good()) {
+ std::string substr;
+ getline(ss, substr, ';');
+ v.push_back(std::stof(substr));
+ }
- // Convert vector to array of bytes
- const size_t byteSize = v.size() * sizeof(float);
- unsigned char *bytes = new unsigned char[byteSize];
- std::memcpy(bytes, v.data(), byteSize);
+ input.close();
- // Copy bytes to dynamically allocated string
- *descriptor_ptr = new std::string(reinterpret_cast(bytes), byteSize);
+ // Convert vector to array of bytes
+ const size_t byteSize = v.size() * sizeof(float);
+ unsigned char *bytes = new unsigned char[byteSize];
+ std::memcpy(bytes, v.data(), byteSize);
- // Clean up dynamically-allocated memory
- delete[] bytes;
-}
+ // Copy bytes to dynamically allocated string
+ *descriptor_ptr = new std::string(reinterpret_cast(bytes), byteSize);
-void VDMS::CSVParserUtil::videoToString(const std::string &filename, std::string **video_data_ptr)
-{
- // Open the video file in binary mode
- std::ifstream file(filename, std::ios::binary);
-
- if (!file)
- {
- std::cerr << "Failed to open file: " << filename << std::endl;
- *video_data_ptr = nullptr;
- }
- else
- {
- // Read the entire content of the file into a string
- std::stringstream buffer;
- buffer << file.rdbuf();
- std::string video_data = buffer.str();
-
- *video_data_ptr = new std::string(video_data);
- }
+ // Clean up dynamically-allocated memory
+ delete[] bytes;
+}
- // Close the file
- file.close();
+void VDMS::CSVParserUtil::videoToString(const std::string &filename,
+ std::string **video_data_ptr) {
+ // Open the video file in binary mode
+ std::ifstream file(filename, std::ios::binary);
+
+ if (!file) {
+ std::cerr << "Failed to open file: " << filename << std::endl;
+ *video_data_ptr = nullptr;
+ } else {
+ // Read the entire content of the file into a string
+ std::stringstream buffer;
+ buffer << file.rdbuf();
+ std::string video_data = buffer.str();
+
+ *video_data_ptr = new std::string(video_data);
+ }
+
+ // Close the file
+ file.close();
}
-void VDMS::CSVParserUtil::read_blob_image(const std::string &filename, std::string **image_data_ptr)
-{
- std::ifstream file(filename, std::ios::binary);
+void VDMS::CSVParserUtil::read_blob_image(const std::string &filename,
+ std::string **image_data_ptr) {
+ std::ifstream file(filename, std::ios::binary);
- if (file.is_open())
- {
+ if (file.is_open()) {
- // Get the size of the file
- file.seekg(0, std::ios::end);
- size_t size = file.tellg();
- file.seekg(0, std::ios::beg);
+ // Get the size of the file
+ file.seekg(0, std::ios::end);
+ size_t size = file.tellg();
+ file.seekg(0, std::ios::beg);
- // Allocate a buffer to hold the file data
- char *buffer = new char[size];
+ // Allocate a buffer to hold the file data
+ char *buffer = new char[size];
- // Read the file data into the buffer
- file.read(buffer, size);
+ // Read the file data into the buffer
+ file.read(buffer, size);
- if (file.gcount() != size)
- {
- std::cerr << "Error: Failed to read entire file." << std::endl;
- delete[] buffer;
- *image_data_ptr = nullptr;
- return;
- }
+ if (file.gcount() != size) {
+ std::cerr << "Error: Failed to read entire file." << std::endl;
+ delete[] buffer;
+ *image_data_ptr = nullptr;
+ return;
+ }
- // Close the file
- file.close();
+ // Close the file
+ file.close();
- // Allocate a new std::string to hold the image data
- std::string *image_data = new std::string;
- image_data->assign(buffer, size);
+ // Allocate a new std::string to hold the image data
+ std::string *image_data = new std::string;
+ image_data->assign(buffer, size);
- // Free the buffer
- delete[] buffer;
+ // Free the buffer
+ delete[] buffer;
- // Assign the std::string pointer to the image_data_ptr
- *image_data_ptr = image_data;
- }
- else
- {
- std::cerr << "Error: Failed to open file." << std::endl;
- *image_data_ptr = nullptr;
- }
+ // Assign the std::string pointer to the image_data_ptr
+ *image_data_ptr = image_data;
+ } else {
+ std::cerr << "Error: Failed to open file." << std::endl;
+ *image_data_ptr = nullptr;
+ }
}
-VDMS::Response VDMS::CSVParserUtil::send_to_vdms(const Json::Value &query,
- const std::vector blobs)
-{
- Json::StyledWriter _fastwriter;
+VDMS::Response
+VDMS::CSVParserUtil::send_to_vdms(const Json::Value &query,
+ const std::vector blobs) {
+ Json::StyledWriter _fastwriter;
- return vdms_client->query(_fastwriter.write(query), blobs);
+ return vdms_client->query(_fastwriter.write(query), blobs);
}
diff --git a/client/cpp/CSVParserUtil.h b/client/cpp/CSVParserUtil.h
index ad30bd9d..9d223fe7 100644
--- a/client/cpp/CSVParserUtil.h
+++ b/client/cpp/CSVParserUtil.h
@@ -1,14 +1,14 @@
#pragma once
-#include
-#include
-#include
+#include "VDMSClient.h"
#include "rapidcsv.h"
-#include
-#include
#include
-#include
#include
-#include "VDMSClient.h"
+#include
+#include
+#include
+#include
+#include
+#include
using namespace std;
using namespace std::chrono;
@@ -16,87 +16,92 @@ using namespace std::chrono;
namespace VDMS {
class CSVParserUtil {
- enum QueryType { EntityClass,
- ConnectionClass,
- ImagePath,
- VideoPath,
- DescriptorType,
- DescriptorClass,
- RectangleBound,
- EntityUpdate,
- ConnectionUpdate,
- ImageUpdate,
- RectangleUpdate
- };
- enum class commandType {
- AddEntity,
- AddConnection,
- AddImage,
- AddVideo,
- AddDescriptorSet,
- AddDescriptor,
- AddBoundingBox,
- UpdateEntity,
- UpdateConnection,
- UpdateImage,
- UpdateBoundingBox,
- UNKNOWN
-
- };
- enum class DATATYPE{
- TRUE,
- FALSE,
- INTEGER,
- FLOAT,
- STRING,
- DATE,
- WRONG
+ enum QueryType {
+ EntityClass,
+ ConnectionClass,
+ ImagePath,
+ VideoPath,
+ DescriptorType,
+ DescriptorClass,
+ RectangleBound,
+ EntityUpdate,
+ ConnectionUpdate,
+ ImageUpdate,
+ RectangleUpdate
+ };
+ enum class commandType {
+ AddEntity,
+ AddConnection,
+ AddImage,
+ AddVideo,
+ AddDescriptorSet,
+ AddDescriptor,
+ AddBoundingBox,
+ UpdateEntity,
+ UpdateConnection,
+ UpdateImage,
+ UpdateBoundingBox,
+ UNKNOWN
- };
- std::map commands;
- std::map command_list;
+ };
+ enum class DATATYPE {
+ TRUE,
+ FALSE,
+ INTEGER,
+ FLOAT,
+ STRING,
+ DATE,
+ WRONG
+ };
+ std::map commands;
+ std::map command_list;
+public:
+ CSVParserUtil();
+ CSVParserUtil(const std::string &, int port, const std::vector,
+ int id);
+ void initCommandsMap();
+ int isBool(const string &data);
+ bool isFloat(const string &);
+ bool isInt(const string &);
+ VDMS::Response parse_row(std::vector &fields);
+ commandType get_query_type(const string &data);
+ CSVParserUtil::DATATYPE getDataType(const string &data,
+ const string &colname);
+ void parseProperty(const string &columnNames, const string &row,
+ const string &queryType, Json::Value &aquery);
+ void parseConstraints(const string &columnNames, const string &row,
+ string &queryType, Json::Value &aquery);
+ void parseOperations(const string columnNames, string row, string queryType,
+ Json::Value &aquery);
- public:
- CSVParserUtil();
- CSVParserUtil(const std::string&, int port, const std::vector, int id );
- void initCommandsMap();
- int isBool( const string& data);
- bool isFloat(const string &);
- bool isInt(const string &);
- VDMS::Response parse_row(std::vector& fields);
- commandType get_query_type(const string &data);
- CSVParserUtil::DATATYPE getDataType(const string& data,const string& colname);
- void parseProperty(const string& columnNames,const string& row,const string& queryType, Json::Value &aquery);
- void parseConstraints(const string &columnNames,const string& row, string& queryType, Json::Value &aquery);
- void parseOperations(const string columnNames,string row,string queryType,Json::Value &aquery);
+ // void parseCSVdata(int startrowcount,int endcount,rapidcsv::Document &doc,
+ // int &);
+ vector spiltrow(const string &row);
+ bool isValidOpsType(string &type);
+ void splitRowOnComma(const std::string &row,
+ std::vector &rowvec);
- // void parseCSVdata(int startrowcount,int endcount,rapidcsv::Document &doc, int &);
- vector spiltrow(const string& row);
- bool isValidOpsType(string& type);
- void splitRowOnComma(const std::string& row, std::vector& rowvec);
+ string function_accessing_columnNames(int i);
+ DATATYPE isValidDataType(string data, int type);
+ void read_blob_image(const std::string &filename,
+ std::string **image_data_ptr);
+ void videoToString(const std::string &filename, std::string **video_data);
+ void parseBlobFile(const std::string &filename, std::string **descriptor_ptr);
- string function_accessing_columnNames(int i);
- DATATYPE isValidDataType(string data,int type);
- void read_blob_image(const std::string& filename, std::string** image_data_ptr);
- void videoToString(const std::string& filename, std::string** video_data);
- void parseBlobFile(const std::string& filename, std::string** descriptor_ptr);
+ VDMS::Response send_to_vdms(const Json::Value &json_query,
+ const std::vector blobs = {});
- VDMS::Response send_to_vdms(const Json::Value &json_query, const std::vector blobs = {});
-
- public:
- std::string vdms_server;
- int vdms_port;
- std::vector _columnNames;
- std::mutex querytype_mutex;
- std::mutex aquery_mutex;
- std::mutex cons_mutex;
- std::mutex ops_mutex;
- int id;
- std::unique_ptr vdms_client;
-
- };
+public:
+ std::string vdms_server;
+ int vdms_port;
+ std::vector _columnNames;
+ std::mutex querytype_mutex;
+ std::mutex aquery_mutex;
+ std::mutex cons_mutex;
+ std::mutex ops_mutex;
+ int id;
+ std::unique_ptr vdms_client;
};
-
-
+}; // namespace VDMS
diff --git a/client/cpp/ConnectionQueryParser.h b/client/cpp/ConnectionQueryParser.h
index 6b79a011..d57a3d5e 100644
--- a/client/cpp/ConnectionQueryParser.h
+++ b/client/cpp/ConnectionQueryParser.h
@@ -1,81 +1,76 @@
#include "CSVParserUtil.h"
namespace VDMS {
-class ConnectionQueryParser : public CSVParserUtil{
- public:
- VDMS::Response ParseAddConnection(vector row, vector & cols);
- // VDMS::Response ParseUpdateConnection(vector row, vector & cols);
-};
+class ConnectionQueryParser : public CSVParserUtil {
+public:
+ VDMS::Response ParseAddConnection(vector row, vector &cols);
+ // VDMS::Response ParseUpdateConnection(vector row, vector
+ // & cols);
};
+}; // namespace VDMS
-VDMS::Response VDMS::ConnectionQueryParser::ParseAddConnection(vector row, vector & columnNames){
- Json::Value aquery;
- Json::Value allquery;
- Json::Value find_query1, find_query2,find_query;
+VDMS::Response
+VDMS::ConnectionQueryParser::ParseAddConnection(vector row,
+ vector &columnNames) {
+ Json::Value aquery;
+ Json::Value allquery;
+ Json::Value find_query1, find_query2, find_query;
- if (row[0].empty()) {
+ if (row[0].empty()) {
std::cerr << "Error: Connection Class not provided\n";
+ }
-}
-
-// Set command name and connection class
-const string command_name = "AddConnection";
-const string connection_class = row[0];
-int ref1=1, ref2=3;
-aquery["AddConnection"]["class"] = connection_class;
-
-// Parse class1 and class2 columns
-for (int i = 1; i < columnNames.size(); i++) {
- string column_name = columnNames[i];
- string column_value = row[i];
- string column_type = column_name.substr(0, 5);
- string command="FindEntity";
+ // Set command name and connection class
+ const string command_name = "AddConnection";
+ const string connection_class = row[0];
+ int ref1 = 1, ref2 = 3;
+ aquery["AddConnection"]["class"] = connection_class;
+ // Parse class1 and class2 columns
+ for (int i = 1; i < columnNames.size(); i++) {
+ string column_name = columnNames[i];
+ string column_value = row[i];
+ string column_type = column_name.substr(0, 5);
+ string command = "FindEntity";
if (column_value.empty()) {
- continue;
+ continue;
}
-
-
if (column_name.find('@') != std::string::npos) {
- std::size_t at_pos = column_name.find('@');
-
- if (at_pos != std::string::npos) {
- // Extract the name and id substrings.
- std::string class1 = column_name.substr(0, at_pos);
- std::string class_prop = column_name.substr(at_pos + 1);
-
- find_query["FindEntity"]["class"]=class1;
- find_query["FindEntity"]["_ref"]=ref1++;
- find_query["FindEntity"]["constraints"][class_prop][0] = "==";
- find_query["FindEntity"]["constraints"][class_prop][1]=column_value;
- }
- allquery.append(find_query);
-
+ std::size_t at_pos = column_name.find('@');
+
+ if (at_pos != std::string::npos) {
+ // Extract the name and id substrings.
+ std::string class1 = column_name.substr(0, at_pos);
+ std::string class_prop = column_name.substr(at_pos + 1);
+
+ find_query["FindEntity"]["class"] = class1;
+ find_query["FindEntity"]["_ref"] = ref1++;
+ find_query["FindEntity"]["constraints"][class_prop][0] = "==";
+ find_query["FindEntity"]["constraints"][class_prop][1] = column_value;
+ }
+ allquery.append(find_query);
}
if (column_type == "prop_") {
- parseProperty(column_name, column_value, command_name, aquery);
+ parseProperty(column_name, column_value, command_name, aquery);
}
find_query.clear();
+ }
+ // Set connection references
+ aquery["AddConnection"]["ref1"] = allquery[0]["FindEntity"]["_ref"];
+ aquery["AddConnection"]["ref2"] = allquery[1]["FindEntity"]["_ref"];
-}
-
-// Set connection references
-aquery["AddConnection"]["ref1"] = allquery[0]["FindEntity"]["_ref"];
-aquery["AddConnection"]["ref2"] = allquery[1]["FindEntity"]["_ref"];
-
-
-
-allquery.append(aquery);
-// std::cout< row, vector & columnNames){
+// VDMS::Response
+// VDMS::ConnectionQueryParser::ParseUpdateConnection(vector row,
+// vector & columnNames){
// Json::Value aquery;
// Json::Value aqueryf;
// Json::Value allquery;
diff --git a/client/cpp/DescriptorQueryParser.h b/client/cpp/DescriptorQueryParser.h
index 6a5d634e..1e7dc3b0 100644
--- a/client/cpp/DescriptorQueryParser.h
+++ b/client/cpp/DescriptorQueryParser.h
@@ -1,56 +1,48 @@
#pragma once
#include "CSVParserUtil.h"
namespace VDMS {
-class DescriptorQueryParser : public CSVParserUtil{
- public:
- VDMS::Response ParseAddDescriptor(vector row, vector& columnNames, int id);
-
-};
+class DescriptorQueryParser : public CSVParserUtil {
+public:
+ VDMS::Response ParseAddDescriptor(vector row,
+ vector &columnNames, int id);
};
+}; // namespace VDMS
-VDMS::Response VDMS::DescriptorQueryParser::ParseAddDescriptor(vector row, vector& columnNames, int id){
-
- if(row[0]==""){
- throw "Set not provided";
- }
- Json::Value aquery;
- Json::Value fullquery;
- std::vector blobs;
- std::string* descriptor;
- std::string command_name="AddDescriptor";
- aquery["AddDescriptor"]["set"]=row[0];
- aquery["AddDescriptor"]["_ref"]=id+3;
- for(int j=1;j row, vector &columnNames, int id) {
- parseBlobFile(row[j], &descriptor);
- if (descriptor == nullptr) {
- std::cout << "Failed to parse blob file" << std::endl;
-
- }
-
- if(descriptor!=nullptr){
- blobs.push_back(descriptor);
-
- }
-
- }
-
- }
-
-
-}
-fullquery.append(aquery);
-return send_to_vdms(fullquery,blobs);
-}
-
+ if (row[0] == "") {
+ throw "Set not provided";
+ }
+ Json::Value aquery;
+ Json::Value fullquery;
+ std::vector blobs;
+ std::string *descriptor;
+ std::string command_name = "AddDescriptor";
+ aquery["AddDescriptor"]["set"] = row[0];
+ aquery["AddDescriptor"]["_ref"] = id + 3;
+ for (int j = 1; j < columnNames.size(); j++) {
+ if (row[j] != "") {
+ if (columnNames[j].find("prop_") != string::npos) {
+ parseProperty(columnNames[j], row[j], command_name, aquery);
+ }
+ if (columnNames[j] == "label") {
+ aquery["AddDescriptor"]["label"] = row[j];
+ }
+ if (columnNames[j] == "inputdata") {
+ parseBlobFile(row[j], &descriptor);
+ if (descriptor == nullptr) {
+ std::cout << "Failed to parse blob file" << std::endl;
+ }
+
+ if (descriptor != nullptr) {
+ blobs.push_back(descriptor);
+ }
+ }
+ }
+ }
+ fullquery.append(aquery);
+ return send_to_vdms(fullquery, blobs);
+}
diff --git a/client/cpp/DescriptorSetQueryParser.h b/client/cpp/DescriptorSetQueryParser.h
index 31d35d23..27640b14 100644
--- a/client/cpp/DescriptorSetQueryParser.h
+++ b/client/cpp/DescriptorSetQueryParser.h
@@ -1,53 +1,54 @@
#pragma once
#include "CSVParserUtil.h"
namespace VDMS {
-class DescriptorSetQueryParser : public CSVParserUtil{
- public:
- VDMS::Response ParseAddDescriptorSet(vector row, vector& columnNames);
- bool isValidMetric(string& metric);
- bool isValidEngine(string& engine);
+class DescriptorSetQueryParser : public CSVParserUtil {
+public:
+ VDMS::Response ParseAddDescriptorSet(vector row,
+ vector &columnNames);
+ bool isValidMetric(string &metric);
+ bool isValidEngine(string &engine);
};
-};
-VDMS::Response VDMS::DescriptorSetQueryParser::ParseAddDescriptorSet(vector row, vector& columnNames){
- if(row[0]==""){
- throw "Descriptor Name not provided";
- }
- Json::Value aquery;
- Json::Value fullquery;
- std::string command_name="AddDescriptorSet";
- aquery["AddDescriptorSet"]["name"]=row[0];
-
- for(int j=1;j row, vector &columnNames) {
+ if (row[0] == "") {
+ throw "Descriptor Name not provided";
+ }
+ Json::Value aquery;
+ Json::Value fullquery;
+ std::string command_name = "AddDescriptorSet";
+ aquery["AddDescriptorSet"]["name"] = row[0];
+ for (int j = 1; j < columnNames.size(); j++) {
+ if (!row[j].empty()) {
+ if (columnNames[j].find("prop_") != string::npos) {
+ parseProperty(columnNames[j], row[j], command_name, aquery);
+ }
+ if (columnNames[j] == "dimensions") {
+ aquery["AddDescriptorSet"]["dimensions"] = stoi(row[j]);
+ }
+ if (columnNames[j] == "distancemetric") {
+ if (!isValidMetric(row[j]))
+ throw "Metric value is not valid";
+ aquery["AddDescriptorSet"]["metric"] = row[j];
+ }
+ if (columnNames[j] == "searchengine") {
+ if (!isValidEngine(row[j]))
+ throw "Engine value is not valid";
+ aquery["AddDescriptorSet"]["engine"] = row[j];
+ }
}
+ }
- fullquery.append(aquery);
- return send_to_vdms(fullquery);
+ fullquery.append(aquery);
+ return send_to_vdms(fullquery);
}
-bool VDMS::DescriptorSetQueryParser::isValidMetric(string& metric) {
- return (metric=="L2" || metric=="IP");
+bool VDMS::DescriptorSetQueryParser::isValidMetric(string &metric) {
+ return (metric == "L2" || metric == "IP");
}
-bool VDMS::DescriptorSetQueryParser::isValidEngine(string& engine) {
- return (engine=="TileDBDense" || engine=="TileDBSparse" || engine=="FaissFlat" || engine=="FaissIVFFlat");
+bool VDMS::DescriptorSetQueryParser::isValidEngine(string &engine) {
+ return (engine == "TileDBDense" || engine == "TileDBSparse" ||
+ engine == "FaissFlat" || engine == "FaissIVFFlat");
}
diff --git a/client/cpp/EntityQueryParser.h b/client/cpp/EntityQueryParser.h
index a6e334f6..4f966a73 100644
--- a/client/cpp/EntityQueryParser.h
+++ b/client/cpp/EntityQueryParser.h
@@ -2,58 +2,52 @@
#include "CSVParserUtil.h"
#include
-namespace VDMS
-{
+namespace VDMS {
- class EntityQueryParser : public CSVParserUtil
- {
- public:
- VDMS::Response ParseAddEntity(vector row, vector &cols);
- // VDMS::Response ParseUpdateEntity(vector row, vector & cols);
- };
+class EntityQueryParser : public CSVParserUtil {
+public:
+ VDMS::Response ParseAddEntity(vector row, vector &cols);
+ // VDMS::Response ParseUpdateEntity(vector row, vector &
+ // cols);
};
+}; // namespace VDMS
-VDMS::Response VDMS::EntityQueryParser::ParseAddEntity(vector row, vector &cols)
-{
- Json::Value aquery;
- Json::Value fullquery;
+VDMS::Response VDMS::EntityQueryParser::ParseAddEntity(vector row,
+ vector &cols) {
+ Json::Value aquery;
+ Json::Value fullquery;
- std::string command_name = "AddEntity";
- if (row[0].empty())
- {
- throw "Entity Class not specified";
- }
- if (cols.size() == 0)
- {
- throw std::invalid_argument("Error: Column names vector is empty.");
- }
- aquery[command_name]["class"] = row[0];
- aquery[command_name]["_ref"] = 11;
+ std::string command_name = "AddEntity";
+ if (row[0].empty()) {
+ throw "Entity Class not specified";
+ }
+ if (cols.size() == 0) {
+ throw std::invalid_argument("Error: Column names vector is empty.");
+ }
+ aquery[command_name]["class"] = row[0];
+ aquery[command_name]["_ref"] = 11;
- for (int j = 1; j < cols.size(); j++)
- {
+ for (int j = 1; j < cols.size(); j++) {
- if (!row[j].empty())
- {
+ if (!row[j].empty()) {
- string columnType = cols[j].substr(0, 5);
- if (columnType == "prop_")
- {
+ string columnType = cols[j].substr(0, 5);
+ if (columnType == "prop_") {
- parseProperty(cols[j], row[j], command_name, aquery);
- }
- else if(columnType=="cons_"){
+ parseProperty(cols[j], row[j], command_name, aquery);
+ } else if (columnType == "cons_") {
- parseConstraints(cols[j],row[j],command_name,aquery);
- }
- }
+ parseConstraints(cols[j], row[j], command_name, aquery);
+ }
}
- fullquery.append(aquery);
+ }
+ fullquery.append(aquery);
- return send_to_vdms(fullquery);
+ return send_to_vdms(fullquery);
}
-// VDMS::Response VDMS::EntityQueryParser::ParseUpdateEntity(vector row, vector & cols){
+// VDMS::Response VDMS::EntityQueryParser::ParseUpdateEntity(vector row,
+// vector & cols){
// Json:: Value aquery;
// Json::Value all_query;
// Json::Value find_query;
diff --git a/client/cpp/ImageQueryParser.h b/client/cpp/ImageQueryParser.h
index c485926d..f56c18ff 100644
--- a/client/cpp/ImageQueryParser.h
+++ b/client/cpp/ImageQueryParser.h
@@ -1,79 +1,78 @@
#pragma once
#include "CSVParserUtil.h"
namespace VDMS {
-class ImageQueryParser : public CSVParserUtil{
- private:
- std::mutex file_access_mutex;
- public:
- // ImageQueryParser();
- VDMS::Response ParseAddImage(vector row, vector columnNames);
- // VDMS::Response ParseUpdateImage(vector row, vector columnNames);
- bool ValidImageFormat(string data);
-
-
-};
+class ImageQueryParser : public CSVParserUtil {
+private:
+ std::mutex file_access_mutex;
+
+public:
+ // ImageQueryParser();
+ VDMS::Response ParseAddImage(vector row, vector columnNames);
+ // VDMS::Response ParseUpdateImage(vector row, vector
+ // columnNames);
+ bool ValidImageFormat(string data);
};
-
-
-
-VDMS::Response VDMS::ImageQueryParser::ParseAddImage(vector row, vector columnNames){
- Json::Value aquery;
- Json::Value fullquery;
- std::vector blobs;
- //
- if(row[0].empty())
- throw "Image path is not specified";
- if (columnNames.size() == 0) {
- throw std::invalid_argument("Error: Column names vector is empty.");
- }
-
- std::string command_name="AddImage";
-
- aquery["AddImage"]["_ref"]=11;
-
- std::string name =row[0];
-
- std::string* image_data_ptr = nullptr;
-
- read_blob_image(name, &image_data_ptr);
-
- // std::cout << *image_data_ptr << std::endl;
- if(image_data_ptr!=nullptr){
- blobs.push_back(image_data_ptr);
- // std::cout <<*blobs[0] < row,
+ vector columnNames) {
+ Json::Value aquery;
+ Json::Value fullquery;
+ std::vector blobs;
+ //
+ if (row[0].empty())
+ throw "Image path is not specified";
+ if (columnNames.size() == 0) {
+ throw std::invalid_argument("Error: Column names vector is empty.");
+ }
+
+ std::string command_name = "AddImage";
+
+ aquery["AddImage"]["_ref"] = 11;
+
+ std::string name = row[0];
+
+ std::string *image_data_ptr = nullptr;
+
+ read_blob_image(name, &image_data_ptr);
+
+ // std::cout << *image_data_ptr << std::endl;
+ if (image_data_ptr != nullptr) {
+ blobs.push_back(image_data_ptr);
+ // std::cout <<*blobs[0] < row, vector columnNames){
+// VDMS::Response VDMS::ImageQueryParser::ParseUpdateImage(vector row,
+// vector columnNames){
// Json :: Value aquery;
// Json::Value fullquery;
diff --git a/client/cpp/VDMSClient.cc b/client/cpp/VDMSClient.cc
index c8c259cd..c72ab42d 100644
--- a/client/cpp/VDMSClient.cc
+++ b/client/cpp/VDMSClient.cc
@@ -30,48 +30,45 @@
#include "VDMSClient.h"
#include "queryMessage.pb.h"
-
using namespace VDMS;
-VDMSClient::VDMSClient(std::string addr, int port) : _conn(addr, port)
-{
-}
-// void VDMSClient::parse_csv_file(std::string filename, std::string server, int p){
+VDMSClient::VDMSClient(std::string addr, int port) : _conn(addr, port) {}
+// void VDMSClient::parse_csv_file(std::string filename, std::string server, int
+// p){
// CSVParser _csv_parser(filename, server, p);
-// auto start = high_resolution_clock::now();
+// auto start = high_resolution_clock::now();
// // _csv_parser.create_thread_pool();
// auto end = high_resolution_clock::now();
// auto duration = duration_cast(end - start);
-// cout << "duaration in ms is "< blobs)
-{
- protobufs::queryMessage cmd;
- cmd.set_json(json);
+ const std::vector blobs) {
+ protobufs::queryMessage cmd;
+ cmd.set_json(json);
- for (auto& it : blobs) {
- std::string *blob = cmd.add_blobs();
- *blob = *it;
- }
+ for (auto &it : blobs) {
+ std::string *blob = cmd.add_blobs();
+ *blob = *it;
+ }
- std::basic_string msg(cmd.ByteSize(),0);
- cmd.SerializeToArray((void*)msg.data(), msg.length());
- _conn.send_message(msg.data(), msg.length());
+ std::basic_string msg(cmd.ByteSize(), 0);
+ cmd.SerializeToArray((void *)msg.data(), msg.length());
+ _conn.send_message(msg.data(), msg.length());
- // Wait for response (blocking call)
- msg = _conn.recv_message();
+ // Wait for response (blocking call)
+ msg = _conn.recv_message();
- protobufs::queryMessage protobuf_response;
- protobuf_response.ParseFromArray((const void*)msg.data(), msg.length());
+ protobufs::queryMessage protobuf_response;
+ protobuf_response.ParseFromArray((const void *)msg.data(), msg.length());
- VDMS::Response response;
- response.json = protobuf_response.json();
+ VDMS::Response response;
+ response.json = protobuf_response.json();
- for (auto& it : protobuf_response.blobs()) {
- response.blobs.push_back(it);
- }
+ for (auto &it : protobuf_response.blobs()) {
+ response.blobs.push_back(it);
+ }
- return response;
+ return response;
}
diff --git a/client/cpp/VDMSClient.h b/client/cpp/VDMSClient.h
index ced571d7..67e20938 100644
--- a/client/cpp/VDMSClient.h
+++ b/client/cpp/VDMSClient.h
@@ -29,37 +29,34 @@
#pragma once
+#include "comm/Connection.h"
#include
#include
-#include "comm/Connection.h"
// #include "CSVParser.h"
namespace VDMS {
- struct Response {
- std::string json;
- std::vector blobs;
- };
-
+struct Response {
+ std::string json;
+ std::vector blobs;
+};
- class VDMSClient {
- static const int VDMS_PORT = 55555;
+class VDMSClient {
+ static const int VDMS_PORT = 55555;
- // The constructor of the ConnClient class already connects to the
- // server if instantiated with the right address and port and it gets
- // disconnected when the class goes out of scope. For now, we
- // will leave the functioning like that. If the client has a need to
- // disconnect and connect specifically, then we can add explicit calls.
- comm::ConnClient _conn;
-
+ // The constructor of the ConnClient class already connects to the
+ // server if instantiated with the right address and port and it gets
+ // disconnected when the class goes out of scope. For now, we
+ // will leave the functioning like that. If the client has a need to
+ // disconnect and connect specifically, then we can add explicit calls.
+ comm::ConnClient _conn;
- public:
- VDMSClient(std::string addr = "localhost", int port = VDMS_PORT);
+public:
+ VDMSClient(std::string addr = "localhost", int port = VDMS_PORT);
- // Blocking call
- VDMS::Response query(const std::string &json_query,
- const std::vector blobs = {});
- // void parse_csv_file(std::string filename, std::string , int);
-
- };
+ // Blocking call
+ VDMS::Response query(const std::string &json_query,
+ const std::vector blobs = {});
+ // void parse_csv_file(std::string filename, std::string , int);
};
+}; // namespace VDMS
diff --git a/client/cpp/VideoQueryParser.h b/client/cpp/VideoQueryParser.h
index 3c89758b..6ac5747a 100644
--- a/client/cpp/VideoQueryParser.h
+++ b/client/cpp/VideoQueryParser.h
@@ -1,84 +1,83 @@
#pragma once
#include "CSVParserUtil.h"
namespace VDMS {
-class VideoQueryParser : public CSVParserUtil{
- public:
- VDMS::Response ParseAddVideo(vector row, vector& columnNames);
- bool isValidCodec(string& row);
- bool isValidContainer(string& row);
-
+class VideoQueryParser : public CSVParserUtil {
+public:
+ VDMS::Response ParseAddVideo(vector row, vector &columnNames);
+ bool isValidCodec(string &row);
+ bool isValidContainer(string &row);
};
-}
-VDMS::Response VDMS::VideoQueryParser::ParseAddVideo(vector row, vector& columnNames){
- Json::Value aquery;
- Json::Value fullquery;
- std::vector blobs;
- if(row[0]=="")
- throw "Video not provided";
- std::string command_name="AddVideo";
-
- std::string video_name=row[0];
- try {
- std::string* video_data_ptr;
- CSVParserUtil::videoToString(video_name, &video_data_ptr);
+} // namespace VDMS
+VDMS::Response
+VDMS::VideoQueryParser::ParseAddVideo(vector row,
+ vector &columnNames) {
+ Json::Value aquery;
+ Json::Value fullquery;
+ std::vector blobs;
+ if (row[0] == "")
+ throw "Video not provided";
+ std::string command_name = "AddVideo";
- if(video_data_ptr!=nullptr){
- blobs.push_back(video_data_ptr);
- // std::cout <<*blobs[0] <::signaling_NaN(),
- const long long pDefaultInteger = 0)
- : mHasDefaultConverter(pHasDefaultConverter)
- , mDefaultFloat(pDefaultFloat)
- , mDefaultInteger(pDefaultInteger)
- {
- }
+ * @brief Constructor
+ * @param pHasDefaultConverter specifies if conversion of non-numerical
+ * strings shall be converted to a default numerical value, instead of causing
+ * an exception to be thrown (default).
+ * @param pDefaultFloat floating-point default value to represent
+ * invalid numbers.
+ * @param pDefaultInteger integer default value to represent invalid
+ * numbers.
+ */
+ explicit ConverterParams(
+ const bool pHasDefaultConverter = false,
+ const long double pDefaultFloat =
+ std::numeric_limits::signaling_NaN(),
+ const long long pDefaultInteger = 0)
+ : mHasDefaultConverter(pHasDefaultConverter),
+ mDefaultFloat(pDefaultFloat), mDefaultInteger(pDefaultInteger) {}
- /**
- * @brief specifies if conversion of non-numerical strings shall be converted to a default
- * numerical value, instead of causing an exception to be thrown (default).
- */
- bool mHasDefaultConverter;
+ /**
+ * @brief specifies if conversion of non-numerical strings shall be
+ * converted to a default numerical value, instead of causing an exception to
+ * be thrown (default).
+ */
+ bool mHasDefaultConverter;
- /**
- * @brief floating-point default value to represent invalid numbers.
- */
- long double mDefaultFloat;
+ /**
+ * @brief floating-point default value to represent invalid numbers.
+ */
+ long double mDefaultFloat;
- /**
- * @brief integer default value to represent invalid numbers.
- */
- long long mDefaultInteger;
- };
+ /**
+ * @brief integer default value to represent invalid numbers.
+ */
+ long long mDefaultInteger;
+};
+/**
+ * @brief Exception thrown when attempting to access Document data in a
+ * datatype which is not supported by the Converter class.
+ */
+class no_converter : public std::exception {
/**
- * @brief Exception thrown when attempting to access Document data in a datatype which
- * is not supported by the Converter class.
+ * @brief Provides details about the exception
+ * @returns an explanatory string
*/
- class no_converter : public std::exception
- {
- /**
- * @brief Provides details about the exception
- * @returns an explanatory string
- */
- virtual const char* what() const throw()
- {
- return "unsupported conversion datatype";
- }
- };
-
- /**
- * @brief Class providing conversion to/from numerical datatypes and strings. Only
- * intended for rapidcsv internal usage, but exposed externally to allow
- * specialization for custom datatype conversions.
- */
- template
- class Converter
- {
- public:
- /**
- * @brief Constructor
- * @param pConverterParams specifies how conversion of non-numerical values to
- * numerical datatype shall be handled.
- */
- Converter(const ConverterParams& pConverterParams)
- : mConverterParams(pConverterParams)
- {
- }
+ virtual const char *what() const throw() {
+ return "unsupported conversion datatype";
+ }
+};
- /**
- * @brief Converts numerical value to string representation.
- * @param pVal numerical value
- * @param pStr output string
- */
- void ToStr(const T& pVal, std::string& pStr) const
- {
- if (typeid(T) == typeid(int) ||
- typeid(T) == typeid(long) ||
- typeid(T) == typeid(long long) ||
- typeid(T) == typeid(unsigned) ||
- typeid(T) == typeid(unsigned long) ||
- typeid(T) == typeid(unsigned long long) ||
- typeid(T) == typeid(float) ||
- typeid(T) == typeid(double) ||
- typeid(T) == typeid(long double) ||
- typeid(T) == typeid(char))
- {
- std::ostringstream out;
- out << pVal;
- pStr = out.str();
- }
- else
- {
- throw no_converter();
- }
- }
+/**
+ * @brief Class providing conversion to/from numerical datatypes and
+ * strings. Only intended for rapidcsv internal usage, but exposed externally to
+ * allow specialization for custom datatype conversions.
+ */
+template class Converter {
+public:
+ /**
+ * @brief Constructor
+ * @param pConverterParams specifies how conversion of non-numerical
+ * values to numerical datatype shall be handled.
+ */
+ Converter(const ConverterParams &pConverterParams)
+ : mConverterParams(pConverterParams) {}
- /**
- * @brief Converts string holding a numerical value to numerical datatype representation.
- * @param pVal numerical value
- * @param pStr output string
- */
- void ToVal(const std::string& pStr, T& pVal) const
- {
- try
- {
- if (typeid(T) == typeid(int))
- {
- pVal = static_cast(std::stoi(pStr));
- return;
- }
- else if (typeid(T) == typeid(long))
- {
- pVal = static_cast(std::stol(pStr));
- return;
- }
- else if (typeid(T) == typeid(long long))
- {
- pVal = static_cast(std::stoll(pStr));
- return;
- }
- else if (typeid(T) == typeid(unsigned))
- {
- pVal = static_cast(std::stoul(pStr));
- return;
- }
- else if (typeid(T) == typeid(unsigned long))
- {
- pVal = static_cast(std::stoul(pStr));
- return;
- }
- else if (typeid(T) == typeid(unsigned long long))
- {
- pVal = static_cast(std::stoull(pStr));
- return;
- }
- }
- catch (...)
- {
- if (!mConverterParams.mHasDefaultConverter)
- {
- throw;
- }
- else
- {
- pVal = static_cast(mConverterParams.mDefaultInteger);
- return;
- }
- }
+ /**
+ * @brief Converts numerical value to string representation.
+ * @param pVal numerical value
+ * @param pStr output string
+ */
+ void ToStr(const T &pVal, std::string &pStr) const {
+ if (typeid(T) == typeid(int) || typeid(T) == typeid(long) ||
+ typeid(T) == typeid(long long) || typeid(T) == typeid(unsigned) ||
+ typeid(T) == typeid(unsigned long) ||
+ typeid(T) == typeid(unsigned long long) || typeid(T) == typeid(float) ||
+ typeid(T) == typeid(double) || typeid(T) == typeid(long double) ||
+ typeid(T) == typeid(char)) {
+ std::ostringstream out;
+ out << pVal;
+ pStr = out.str();
+ } else {
+ throw no_converter();
+ }
+ }
- try
- {
- if (typeid(T) == typeid(float))
- {
- pVal = static_cast(std::stof(pStr));
- return;
- }
- else if (typeid(T) == typeid(double))
- {
- pVal = static_cast(std::stod(pStr));
- return;
- }
- else if (typeid(T) == typeid(long double))
- {
- pVal = static_cast(std::stold(pStr));
- return;
- }
+ /**
+ * @brief Converts string holding a numerical value to numerical datatype
+ * representation.
+ * @param pVal numerical value
+ * @param pStr output string
+ */
+ void ToVal(const std::string &pStr, T &pVal) const {
+ try {
+ if (typeid(T) == typeid(int)) {
+ pVal = static_cast(std::stoi(pStr));
+ return;
+ } else if (typeid(T) == typeid(long)) {
+ pVal = static_cast(std::stol(pStr));
+ return;
+ } else if (typeid(T) == typeid(long long)) {
+ pVal = static_cast(std::stoll(pStr));
+ return;
+ } else if (typeid(T) == typeid(unsigned)) {
+ pVal = static_cast(std::stoul(pStr));
+ return;
+ } else if (typeid(T) == typeid(unsigned long)) {
+ pVal = static_cast(std::stoul(pStr));
+ return;
+ } else if (typeid(T) == typeid(unsigned long long)) {
+ pVal = static_cast(std::stoull(pStr));
+ return;
}
- catch (...)
- {
- if (!mConverterParams.mHasDefaultConverter)
- {
- throw;
- }
- else
- {
- pVal = static_cast(mConverterParams.mDefaultFloat);
- return;
- }
+ } catch (...) {
+ if (!mConverterParams.mHasDefaultConverter) {
+ throw;
+ } else {
+ pVal = static_cast(mConverterParams.mDefaultInteger);
+ return;
}
+ }
- if (typeid(T) == typeid(char))
- {
- pVal = static_cast(pStr[0]);
+ try {
+ if (typeid(T) == typeid(float)) {
+ pVal = static_cast(std::stof(pStr));
+ return;
+ } else if (typeid(T) == typeid(double)) {
+ pVal = static_cast(std::stod(pStr));
+ return;
+ } else if (typeid(T) == typeid(long double)) {
+ pVal = static_cast(std::stold(pStr));
return;
}
- else
- {
- throw no_converter();
+ } catch (...) {
+ if (!mConverterParams.mHasDefaultConverter) {
+ throw;
+ } else {
+ pVal = static_cast(mConverterParams.mDefaultFloat);
+ return;
}
}
- private:
- const ConverterParams& mConverterParams;
- };
+ if (typeid(T) == typeid(char)) {
+ pVal = static_cast(pStr[0]);
+ return;
+ } else {
+ throw no_converter();
+ }
+ }
+private:
+ const ConverterParams &mConverterParams;
+};
+
+/**
+ * @brief Specialized implementation handling string to string conversion.
+ * @param pVal string
+ * @param pStr string
+ */
+template <>
+inline void Converter::ToStr(const std::string &pVal,
+ std::string &pStr) const {
+ pStr = pVal;
+}
+
+/**
+ * @brief Specialized implementation handling string to string conversion.
+ * @param pVal string
+ * @param pStr string
+ */
+template <>
+inline void Converter::ToVal(const std::string &pStr,
+ std::string &pVal) const {
+ pVal = pStr;
+}
+
+template
+using ConvFunc = std::function;
+
+/**
+ * @brief Datastructure holding parameters controlling which row and column
+ * should be treated as labels.
+ */
+struct LabelParams {
/**
- * @brief Specialized implementation handling string to string conversion.
- * @param pVal string
- * @param pStr string
+ * @brief Constructor
+ * @param pColumnNameIdx specifies the zero-based row index of the
+ * column labels, setting it to -1 prevents column lookup by label name, and
+ * gives access to all rows as document data. Default: 0
+ * @param pRowNameIdx specifies the zero-based column index of the
+ * row labels, setting it to -1 prevents row lookup by label name, and gives
+ * access to all columns as document data. Default: -1
*/
- template<>
- inline void Converter::ToStr(const std::string& pVal, std::string& pStr) const
- {
- pStr = pVal;
- }
+ explicit LabelParams(const int pColumnNameIdx = 0, const int pRowNameIdx = -1)
+ : mColumnNameIdx(pColumnNameIdx), mRowNameIdx(pRowNameIdx) {}
/**
- * @brief Specialized implementation handling string to string conversion.
- * @param pVal string
- * @param pStr string
+ * @brief specifies the zero-based row index of the column labels.
*/
- template<>
- inline void Converter::ToVal(const std::string& pStr, std::string& pVal) const
- {
- pVal = pStr;
- }
+ int mColumnNameIdx;
- template
- using ConvFunc = std::function;
-
- /**
- * @brief Datastructure holding parameters controlling which row and column should be
- * treated as labels.
- */
- struct LabelParams
- {
- /**
- * @brief Constructor
- * @param pColumnNameIdx specifies the zero-based row index of the column labels, setting
- * it to -1 prevents column lookup by label name, and gives access
- * to all rows as document data. Default: 0
- * @param pRowNameIdx specifies the zero-based column index of the row labels, setting
- * it to -1 prevents row lookup by label name, and gives access
- * to all columns as document data. Default: -1
- */
- explicit LabelParams(const int pColumnNameIdx = 0, const int pRowNameIdx = -1)
- : mColumnNameIdx(pColumnNameIdx)
- , mRowNameIdx(pRowNameIdx)
- {
- }
+ /**
+ * @brief specifies the zero-based column index of the row labels.
+ */
+ int mRowNameIdx;
+};
- /**
- * @brief specifies the zero-based row index of the column labels.
- */
- int mColumnNameIdx;
-
- /**
- * @brief specifies the zero-based column index of the row labels.
- */
- int mRowNameIdx;
- };
-
- /**
- * @brief Datastructure holding parameters controlling how the CSV data fields are separated.
- */
- struct SeparatorParams
- {
- /**
- * @brief Constructor
- * @param pSeparator specifies the column separator (default ',').
- * @param pTrim specifies whether to trim leading and trailing spaces from
- * cells read (default false).
- * @param pHasCR specifies whether a new document (i.e. not an existing document read)
- * should use CR/LF instead of only LF (default is to use standard
- * behavior of underlying platforms - CR/LF for Win, and LF for others).
- * @param pQuotedLinebreaks specifies whether to allow line breaks in quoted text (default false)
- * @param pAutoQuote specifies whether to automatically dequote data during read, and add
- * quotes during write (default true).
- */
- explicit SeparatorParams(const char pSeparator = ',', const bool pTrim = false,
- const bool pHasCR = sPlatformHasCR, const bool pQuotedLinebreaks = false,
- const bool pAutoQuote = true)
- : mSeparator(pSeparator)
- , mTrim(pTrim)
- , mHasCR(pHasCR)
- , mQuotedLinebreaks(pQuotedLinebreaks)
- , mAutoQuote(pAutoQuote)
- {
- }
+/**
+ * @brief Datastructure holding parameters controlling how the CSV data
+ * fields are separated.
+ */
+struct SeparatorParams {
+ /**
+ * @brief Constructor
+ * @param pSeparator specifies the column separator (default
+ * ',').
+ * @param pTrim specifies whether to trim leading and
+ * trailing spaces from cells read (default false).
+ * @param pHasCR specifies whether a new document (i.e. not
+ * an existing document read) should use CR/LF instead of only LF (default is
+ * to use standard behavior of underlying platforms - CR/LF for Win, and LF
+ * for others).
+ * @param pQuotedLinebreaks specifies whether to allow line breaks in
+ * quoted text (default false)
+ * @param pAutoQuote specifies whether to automatically dequote
+ * data during read, and add quotes during write (default true).
+ */
+ explicit SeparatorParams(const char pSeparator = ',',
+ const bool pTrim = false,
+ const bool pHasCR = sPlatformHasCR,
+ const bool pQuotedLinebreaks = false,
+ const bool pAutoQuote = true)
+ : mSeparator(pSeparator), mTrim(pTrim), mHasCR(pHasCR),
+ mQuotedLinebreaks(pQuotedLinebreaks), mAutoQuote(pAutoQuote) {}
- /**
- * @brief specifies the column separator.
- */
- char mSeparator;
-
- /**
- * @brief specifies whether to trim leading and trailing spaces from cells read.
- */
- bool mTrim;
-
- /**
- * @brief specifies whether new documents should use CR/LF instead of LF.
- */
- bool mHasCR;
-
- /**
- * @brief specifies whether to allow line breaks in quoted text.
- */
- bool mQuotedLinebreaks;
-
- /**
- * @brief specifies whether to automatically dequote cell data.
- */
- bool mAutoQuote;
- };
-
- /**
- * @brief Datastructure holding parameters controlling how special line formats should be
- * treated.
- */
- struct LineReaderParams
- {
- /**
- * @brief Constructor
- * @param pSkipCommentLines specifies whether to skip lines prefixed with
- * mCommentPrefix. Default: false
- * @param pCommentPrefix specifies which prefix character to indicate a comment
- * line. Default: #
- * @param pSkipEmptyLines specifies whether to skip empty lines. Default: false
- */
- explicit LineReaderParams(const bool pSkipCommentLines = false,
- const char pCommentPrefix = '#',
- const bool pSkipEmptyLines = false)
- : mSkipCommentLines(pSkipCommentLines)
- , mCommentPrefix(pCommentPrefix)
- , mSkipEmptyLines(pSkipEmptyLines)
- {
- }
+ /**
+ * @brief specifies the column separator.
+ */
+ char mSeparator;
- /**
- * @brief specifies whether to skip lines prefixed with mCommentPrefix.
- */
- bool mSkipCommentLines;
-
- /**
- * @brief specifies which prefix character to indicate a comment line.
- */
- char mCommentPrefix;
-
- /**
- * @brief specifies whether to skip empty lines.
- */
- bool mSkipEmptyLines;
- };
-
- /**
- * @brief Class representing a CSV document.
- */
- class Document
- {
- public:
- /**
- * @brief Constructor
- * @param pPath specifies the path of an existing CSV-file to populate the Document
- * data with.
- * @param pLabelParams specifies which row and column should be treated as labels.
- * @param pSeparatorParams specifies which field and row separators should be used.
- * @param pConverterParams specifies how invalid numbers (including empty strings) should be
- * handled.
- * @param pLineReaderParams specifies how special line formats should be treated.
- */
- explicit Document(const std::string& pPath = std::string(),
- const LabelParams& pLabelParams = LabelParams(),
- const SeparatorParams& pSeparatorParams = SeparatorParams(),
- const ConverterParams& pConverterParams = ConverterParams(),
- const LineReaderParams& pLineReaderParams = LineReaderParams())
- : mPath(pPath)
- , mLabelParams(pLabelParams)
- , mSeparatorParams(pSeparatorParams)
- , mConverterParams(pConverterParams)
- , mLineReaderParams(pLineReaderParams)
- {
- if (!mPath.empty())
- {
- ReadCsv();
- }
- }
+ /**
+ * @brief specifies whether to trim leading and trailing spaces from cells
+ * read.
+ */
+ bool mTrim;
- /**
- * @brief Constructor
- * @param pStream specifies an input stream to read CSV data from.
- * @param pLabelParams specifies which row and column should be treated as labels.
- * @param pSeparatorParams specifies which field and row separators should be used.
- * @param pConverterParams specifies how invalid numbers (including empty strings) should be
- * handled.
- * @param pLineReaderParams specifies how special line formats should be treated.
- */
- explicit Document(std::istream& pStream,
- const LabelParams& pLabelParams = LabelParams(),
- const SeparatorParams& pSeparatorParams = SeparatorParams(),
- const ConverterParams& pConverterParams = ConverterParams(),
- const LineReaderParams& pLineReaderParams = LineReaderParams())
- : mPath()
- , mLabelParams(pLabelParams)
- , mSeparatorParams(pSeparatorParams)
- , mConverterParams(pConverterParams)
- , mLineReaderParams(pLineReaderParams)
- {
- ReadCsv(pStream);
- }
-
- /**
- * @brief Read Document data from file.
- * @param pPath specifies the path of an existing CSV-file to populate the Document
- * data with.
- * @param pLabelParams specifies which row and column should be treated as labels.
- * @param pSeparatorParams specifies which field and row separators should be used.
- * @param pConverterParams specifies how invalid numbers (including empty strings) should be
- * handled.
- * @param pLineReaderParams specifies how special line formats should be treated.
- */
- void Load(const std::string& pPath,
- const LabelParams& pLabelParams = LabelParams(),
- const SeparatorParams& pSeparatorParams = SeparatorParams(),
- const ConverterParams& pConverterParams = ConverterParams(),
- const LineReaderParams& pLineReaderParams = LineReaderParams())
- {
- mPath = pPath;
- mLabelParams = pLabelParams;
- mSeparatorParams = pSeparatorParams;
- mConverterParams = pConverterParams;
- mLineReaderParams = pLineReaderParams;
+ /**
+ * @brief specifies whether new documents should use CR/LF instead of LF.
+ */
+ bool mHasCR;
+
+ /**
+ * @brief specifies whether to allow line breaks in quoted text.
+ */
+ bool mQuotedLinebreaks;
+
+ /**
+ * @brief specifies whether to automatically dequote cell data.
+ */
+ bool mAutoQuote;
+};
+
+/**
+ * @brief Datastructure holding parameters controlling how special line
+ * formats should be treated.
+ */
+struct LineReaderParams {
+ /**
+ * @brief Constructor
+ * @param pSkipCommentLines specifies whether to skip lines prefixed
+ * with mCommentPrefix. Default: false
+ * @param pCommentPrefix specifies which prefix character to indicate
+ * a comment line. Default: #
+ * @param pSkipEmptyLines specifies whether to skip empty lines.
+ * Default: false
+ */
+ explicit LineReaderParams(const bool pSkipCommentLines = false,
+ const char pCommentPrefix = '#',
+ const bool pSkipEmptyLines = false)
+ : mSkipCommentLines(pSkipCommentLines), mCommentPrefix(pCommentPrefix),
+ mSkipEmptyLines(pSkipEmptyLines) {}
+
+ /**
+ * @brief specifies whether to skip lines prefixed with mCommentPrefix.
+ */
+ bool mSkipCommentLines;
+
+ /**
+ * @brief specifies which prefix character to indicate a comment line.
+ */
+ char mCommentPrefix;
+
+ /**
+ * @brief specifies whether to skip empty lines.
+ */
+ bool mSkipEmptyLines;
+};
+
+/**
+ * @brief Class representing a CSV document.
+ */
+class Document {
+public:
+ /**
+ * @brief Constructor
+ * @param pPath specifies the path of an existing CSV-file
+ * to populate the Document data with.
+ * @param pLabelParams specifies which row and column should be
+ * treated as labels.
+ * @param pSeparatorParams specifies which field and row separators
+ * should be used.
+ * @param pConverterParams specifies how invalid numbers (including
+ * empty strings) should be handled.
+ * @param pLineReaderParams specifies how special line formats should be
+ * treated.
+ */
+ explicit Document(
+ const std::string &pPath = std::string(),
+ const LabelParams &pLabelParams = LabelParams(),
+ const SeparatorParams &pSeparatorParams = SeparatorParams(),
+ const ConverterParams &pConverterParams = ConverterParams(),
+ const LineReaderParams &pLineReaderParams = LineReaderParams())
+ : mPath(pPath), mLabelParams(pLabelParams),
+ mSeparatorParams(pSeparatorParams), mConverterParams(pConverterParams),
+ mLineReaderParams(pLineReaderParams) {
+ if (!mPath.empty()) {
ReadCsv();
}
+ }
- /**
- * @brief Read Document data from stream.
- * @param pStream specifies an input stream to read CSV data from.
- * @param pLabelParams specifies which row and column should be treated as labels.
- * @param pSeparatorParams specifies which field and row separators should be used.
- * @param pConverterParams specifies how invalid numbers (including empty strings) should be
- * handled.
- * @param pLineReaderParams specifies how special line formats should be treated.
- */
- void Load(std::istream& pStream,
- const LabelParams& pLabelParams = LabelParams(),
- const SeparatorParams& pSeparatorParams = SeparatorParams(),
- const ConverterParams& pConverterParams = ConverterParams(),
- const LineReaderParams& pLineReaderParams = LineReaderParams())
- {
- mPath = "";
- mLabelParams = pLabelParams;
- mSeparatorParams = pSeparatorParams;
- mConverterParams = pConverterParams;
- mLineReaderParams = pLineReaderParams;
- ReadCsv(pStream);
- }
-
- /**
- * @brief Write Document data to file.
- * @param pPath optionally specifies the path where the CSV-file will be created
- * (if not specified, the original path provided when creating or
- * loading the Document data will be used).
- */
- void Save(const std::string& pPath = std::string())
- {
- if (!pPath.empty())
- {
- mPath = pPath;
- }
- WriteCsv();
- }
+ /**
+ * @brief Constructor
+ * @param pStream specifies an input stream to read CSV data
+ * from.
+ * @param pLabelParams specifies which row and column should be
+ * treated as labels.
+ * @param pSeparatorParams specifies which field and row separators
+ * should be used.
+ * @param pConverterParams specifies how invalid numbers (including
+ * empty strings) should be handled.
+ * @param pLineReaderParams specifies how special line formats should be
+ * treated.
+ */
+ explicit Document(
+ std::istream &pStream, const LabelParams &pLabelParams = LabelParams(),
+ const SeparatorParams &pSeparatorParams = SeparatorParams(),
+ const ConverterParams &pConverterParams = ConverterParams(),
+ const LineReaderParams &pLineReaderParams = LineReaderParams())
+ : mPath(), mLabelParams(pLabelParams), mSeparatorParams(pSeparatorParams),
+ mConverterParams(pConverterParams),
+ mLineReaderParams(pLineReaderParams) {
+ ReadCsv(pStream);
+ }
- /**
- * @brief Write Document data to stream.
- * @param pStream specifies an output stream to write the data to.
- */
- void Save(std::ostream& pStream)
- {
- WriteCsv(pStream);
+ /**
+ * @brief Read Document data from file.
+ * @param pPath specifies the path of an existing CSV-file
+ * to populate the Document data with.
+ * @param pLabelParams specifies which row and column should be
+ * treated as labels.
+ * @param pSeparatorParams specifies which field and row separators
+ * should be used.
+ * @param pConverterParams specifies how invalid numbers (including
+ * empty strings) should be handled.
+ * @param pLineReaderParams specifies how special line formats should be
+ * treated.
+ */
+ void Load(const std::string &pPath,
+ const LabelParams &pLabelParams = LabelParams(),
+ const SeparatorParams &pSeparatorParams = SeparatorParams(),
+ const ConverterParams &pConverterParams = ConverterParams(),
+ const LineReaderParams &pLineReaderParams = LineReaderParams()) {
+ mPath = pPath;
+ mLabelParams = pLabelParams;
+ mSeparatorParams = pSeparatorParams;
+ mConverterParams = pConverterParams;
+ mLineReaderParams = pLineReaderParams;
+ ReadCsv();
+ }
+
+ /**
+ * @brief Read Document data from stream.
+ * @param pStream specifies an input stream to read CSV data
+ * from.
+ * @param pLabelParams specifies which row and column should be
+ * treated as labels.
+ * @param pSeparatorParams specifies which field and row separators
+ * should be used.
+ * @param pConverterParams specifies how invalid numbers (including
+ * empty strings) should be handled.
+ * @param pLineReaderParams specifies how special line formats should be
+ * treated.
+ */
+ void Load(std::istream &pStream,
+ const LabelParams &pLabelParams = LabelParams(),
+ const SeparatorParams &pSeparatorParams = SeparatorParams(),
+ const ConverterParams &pConverterParams = ConverterParams(),
+ const LineReaderParams &pLineReaderParams = LineReaderParams()) {
+ mPath = "";
+ mLabelParams = pLabelParams;
+ mSeparatorParams = pSeparatorParams;
+ mConverterParams = pConverterParams;
+ mLineReaderParams = pLineReaderParams;
+ ReadCsv(pStream);
+ }
+
+ /**
+ * @brief Write Document data to file.
+ * @param pPath optionally specifies the path where the
+ * CSV-file will be created (if not specified, the original path provided when
+ * creating or loading the Document data will be used).
+ */
+ void Save(const std::string &pPath = std::string()) {
+ if (!pPath.empty()) {
+ mPath = pPath;
}
+ WriteCsv();
+ }
- /**
- * @brief Clears loaded Document data.
- *
- */
- void Clear()
- {
- mData.clear();
- mColumnNames.clear();
- mRowNames.clear();
+ /**
+ * @brief Write Document data to stream.
+ * @param pStream specifies an output stream to write the data
+ * to.
+ */
+ void Save(std::ostream &pStream) { WriteCsv(pStream); }
+
+ /**
+ * @brief Clears loaded Document data.
+ *
+ */
+ void Clear() {
+ mData.clear();
+ mColumnNames.clear();
+ mRowNames.clear();
#ifdef HAS_CODECVT
- mIsUtf16 = false;
- mIsLE = false;
+ mIsUtf16 = false;
+ mIsLE = false;
#endif
- }
+ }
- /**
- * @brief Get column index by name.
- * @param pColumnName column label name.
- * @returns zero-based column index.
- */
- ssize_t GetColumnIdx(const std::string& pColumnName) const
- {
- if (mLabelParams.mColumnNameIdx >= 0)
- {
- if (mColumnNames.find(pColumnName) != mColumnNames.end())
- {
- return mColumnNames.at(pColumnName) - (mLabelParams.mRowNameIdx + 1);
- }
+ /**
+ * @brief Get column index by name.
+ * @param pColumnName column label name.
+ * @returns zero-based column index.
+ */
+ ssize_t GetColumnIdx(const std::string &pColumnName) const {
+ if (mLabelParams.mColumnNameIdx >= 0) {
+ if (mColumnNames.find(pColumnName) != mColumnNames.end()) {
+ return mColumnNames.at(pColumnName) - (mLabelParams.mRowNameIdx + 1);
}
- return -1;
}
+ return -1;
+ }
- /**
- * @brief Get column by index.
- * @param pColumnIdx zero-based column index.
- * @returns vector of column data.
- */
- template