diff --git a/.gitignore b/.gitignore index 55b0d1f8..dfaf72db 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ build/ db/ *.gcov +**/__pycache__/ # VS Code Specific .devcontainer diff --git a/CMakeLists.txt b/CMakeLists.txt index b629dde3..afda38bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,10 +40,30 @@ else() link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/) include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf) - add_library(dms SHARED src/BoundingBoxCommand.cc src/CommunicationManager.cc src/DescriptorsCommand.cc src/DescriptorsManager.cc src/ExceptionsCommand.cc src/ImageCommand.cc src/PMGDIterators.cc src/PMGDQuery.cc src/PMGDQueryHandler.cc src/QueryHandler.cc src/QueryMessage.cc src/RSCommand.cc src/SearchExpression.cc src/Server.cc src/VDMSConfig.cc src/VideoCommand.cc src/AutoDeleteNode.cc ${PROTO_SRCS} ${PROTO_HDRS}) + add_library(dms SHARED + src/BoundingBoxCommand.cc + src/BlobCommand.cc + src/CommunicationManager.cc + src/DescriptorsCommand.cc + src/DescriptorsManager.cc + src/ExceptionsCommand.cc + src/ImageCommand.cc + src/PMGDIterators.cc + src/PMGDQuery.cc + src/PMGDQueryHandler.cc + src/QueryHandler.cc + src/QueryMessage.cc + src/RSCommand.cc + src/SearchExpression.cc + src/Server.cc + src/VDMSConfig.cc + src/VideoCommand.cc + src/AutoDeleteNode.cc + ${PROTO_SRCS} ${PROTO_HDRS} + ) target_link_libraries(dms vcl pmgd pmgd-util protobuf vdms-utils pthread) add_executable(vdms src/vdms.cc) target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS}) -endif () +endif () diff --git a/INSTALL.md b/INSTALL.md index 6439df9d..a35b876c 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -2,126 +2,117 @@ Here is the detailed process of installation of VDMS dependencies. ## Dependencies -Here we will install the Ubuntu 20.04 packages. +Here we will install the Ubuntu 20.04 and Python3 packages. We assume `python`/`pip` is an alias for `python3`/`pip3`. If your system has both Python 2 and Python 3, please replace all pip and python commands with pip3 and python3, respectively. ```bash -apt-get update -apt-get -y install software-properties-common -add-apt-repository "deb http://security.ubuntu.com/ubuntu focal-security main" -apt-get -y install apt-transport-https autoconf automake bison build-essential \ - bzip2 ca-certificates curl ed flex g++ git gnupg-agent javacc libarchive-tools \ +sudo apt-get update +sudo apt-get -y install --no-install-recommends software-properties-common +sudo add-apt-repository "deb http://security.ubuntu.com/ubuntu focal-security main" +sudo apt-get -y install --no-install-recommends apt-transport-https autoconf automake bison build-essential \ + bzip2 ca-certificates curl=7.68.0-1ubuntu2.18 ed flex g++ git gnupg-agent javacc libarchive-tools \ libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \ libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \ libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \ libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \ libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \ - libtbb2 libtiff-dev libtiff5-dev libtool maven mpich openjdk-11-jdk-headless \ - pkg-config python python-dev python3-pip unzip wget -pip3 install numpy + libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \ + pkg-config python3-dev python3-pip unzip +pip install --no-cache-dir "numpy>=1.23.2" "setuptools>=65.5.1" ``` ### Clone/Download Dependencies -Here we clone the repositories for grpc v1.40.0, libpng12, Swig v4.0.2, OpenCV 4.5.3, Valijson v0.6, CMake v3.21.2, Faiss v1.7.1, and FLINNG. Then download necesarry files for zlib v1.2.12, Json-simple v1.1.1, and TileDB v1.3.1. -Here we assume `/` is the working directory. This is important when installing the dependencies. +Here we clone the repositories for grpc v1.40.0, libpng12, Swig v4.0.2, OpenCV 4.5.3, Valijson v0.6, CMake v3.21.2, Faiss v1.7.1, and FLINNG. Then download necesarry files for zlib v1.2.13, Json-simple v1.1.1, and TileDB v1.3.1. +Here we assume `$VDMS_DEP_DIR` is the working directory for installing dependencies and `python` is Python 3. ```bash -git clone --branch v1.40.0 https://github.com/grpc/grpc.git && \ -git clone --branch v4.0.2 https://github.com/swig/swig.git && \ -git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ -git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ +cd $VDMS_DEP_DIR git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ +git clone --branch v4.0.2 https://github.com/swig/swig.git && \ git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \ -git clone https://github.com/tonyzhang617/FLINNG.git +git clone https://github.com/tonyzhang617/FLINNG.git && \ +git clone --recurse-submodules -b v1.40.0 https://github.com/grpc/grpc.git && \ +git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ +git clone --branch v0.6 https://github.com/tristanpenman/valijson.git -curl http://zlib.net/zlib-1.2.12.tar.gz -o zlib-1.2.12.tar.gz && \ -curl https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar \ - -o /usr/share/java/json-simple-1.1.1.jar && \ -wget https://github.com/TileDB-Inc/TileDB/archive/1.3.1.tar.gz +sudo curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ +sudo curl -L -o 1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ +sudo curl -L -o zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz ``` ### Install Dependencies These instructions assume you have full permissions to your system. -If needed, use `sudo` where necessary. +If running as root, remove `sudo` where necessary. + #### CMAKE ```bash -cd /CMake && ./bootstrap -make -j && make install +cd $VDMS_DEP_DIR/CMake && ./bootstrap +make -j && sudo make install ``` ### Swig ```bash -cd /swig +cd $VDMS_DEP_DIR/swig ./autogen.sh && ./configure -make -j && make install +make -j && sudo make install ``` ### Faiss ```bash -cd /faiss +cd $VDMS_DEP_DIR/faiss mkdir build && cd build cmake -DFAISS_ENABLE_GPU=OFF .. -make -j && make install +make -j && sudo make install ``` ### FLINNG ```bash -cd /FLINNG +cd $VDMS_DEP_DIR/FLINNG mkdir build && cd build cmake .. -make -j && make install +make -j && sudo make install ``` ### grpc ```bash -cd /grpc && git submodule update --init --recursive -cd third_party/protobuf/cmake && mkdir build && cd build +cd $VDMS_DEP_DIR/grpc +pip install --no-cache-dir -r requirements.txt +GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install --no-cache-dir . + +cd tools/distrib/python/grpcio_tools +python ../make_grpcio_tools.py +GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install --no-cache-dir . + +cd $VDMS_DEP_DIR/grpc/third_party/zlib/ && mkdir build && cd build cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. -make -j && make install +make -j && sudo make install + +cd $VDMS_DEP_DIR/grpc/third_party/protobuf/cmake +mkdir build && cd build +cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -Dprotobuf_BUILD_TESTS=OFF .. +make -j && sudo make install cd ../../../abseil-cpp && mkdir build && cd build cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. -make -j && make install +make -j && sudo make install cd ../../re2/ && mkdir build && cd build cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. -make -j && make install +make -j && sudo make install -cd ../../zlib/ && mkdir build && cd build -cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. -make -j && make install - -cd /grpc/cmake && mkdir build && cd build +cd $VDMS_DEP_DIR/grpc/cmake && mkdir build && cd build cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. -make -j && make install -``` - -### Zlib -```bash -cd / && gunzip zlib-1.2.12.tar.gz && tar -xvf zlib-1.2.12.tar -cd zlib-1.2.12 && ./configure -make -j && make install -cd / && rm -rf zlib-1.2.12.tar zlib-1.2.12 -``` - -### gtest -Unfortunately apt doesn't build gtest; -you need to do the following steps to get it to work correctly: -```bash -cd /usr/src/gtest/ -cmake . -make -j -mv lib/libgtest* /usr/lib +make -j && sudo make install ``` ### [OpenCV](https://opencv.org/) Below are instructions for installing ***OpenCV v4.5.3***. It may also work with newer versions of OpenCV. ```bash -cd /opencv +cd $VDMS_DEP_DIR/opencv mkdir build && cd build -cmake -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. +cmake -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF .. make -j -make install +sudo make install ``` **Note**: When using videos, and getting the following error: "Unable to stop the stream: Inappropriate ioctl for device", you may need to include more flags when compiling OpenCV. Follow these instructions ([source](https://stackoverflow.com/questions/41200201/opencv-unable-to-stop-the-stream-inappropriate-ioctl-for-device)): @@ -137,66 +128,58 @@ make -j make install ``` +### Zlib +```bash +cd $VDMS_DEP_DIR && tar -xvzf zlib-1.2.13.tar.gz +cd zlib-1.2.13 && ./configure +make -j && sudo make install +``` + ### [TileDB](https://tiledb.io/) VDMS works with ***TileDB v1.3.1.***
The directions below will help you install TileDB v1.3.1 from the source. You can also follow the directions listed [here](https://docs.tiledb.io/en/latest/installation.html). ```bash -cd / && tar xf 1.3.1.tar.gz && rm 1.3.1.tar.gz +cd $VDMS_DEP_DIR && tar -xvf 1.3.1.tar.gz cd TileDB-1.3.1 && mkdir build && cd build ../bootstrap --prefix=/usr/local/ -make -j && make install-tiledb -rm -rf /TileDB-1.3.1 +make -j && sudo make install-tiledb ``` -### Maven +### gtest +Unfortunately apt doesn't build gtest; +you need to do the following steps to get it to work correctly: ```bash -ln -s /grpc/third_party/protobuf/cmake/build/protoc grpc/third_party/protobuf/src/protoc -cd /grpc/third_party/protobuf/java/core -mvn package -cp target/protobuf-java-3.13.0.jar /usr/share/java/protobuf.jar -``` - -You may need to change proxy setting for Maven if you are behind a proxy like this example. -Add setting.xml file to ~/.m2 folder -``` - - - optional - - https - - prox-address - proxy-port - - - +cd /usr/src/gtest/ +sudo cmake . +sudo make -j +sudo mv lib/libgtest* /usr/lib ``` ### Valijson This is a headers-only library, no compilation/installation necessary ```bash -cd /valijson -cp -r include/* /usr/local/include +cd $VDMS_DEP_DIR/valijson +sudo cp -r include/* /usr/local/include ``` ## Install VDMS +This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories. ```bash git clone https://github.com/IntelLabs/vdms.git cd vdms && git checkout develop git submodule update --init --recursive +``` + +When compiling on a target without Optane persistent memory, use the following: +```bash mkdir build && cd build cmake .. make -j cp ../config-vdms.json . ``` - -### Compilation -This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories. - When compiling on a target with Optane persistent memory, use the command set: ```bash mkdir build && cd build @@ -204,9 +187,3 @@ cmake -DCMAKE_CXX_FLAGS='-DPM' .. make -j ``` -For systems without Optane, use the command set: -```bash -mkdir build && cd build -cmake .. -make -j -``` diff --git a/Security.md b/Security.md new file mode 100644 index 00000000..ccbbdc59 --- /dev/null +++ b/Security.md @@ -0,0 +1,5 @@ +# Security Policy +Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. + +## Reporting a Vulnerability +Please report any security vulnerabilities in this project [utilizing the guidelines here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). diff --git a/client/cpp/BoundingBoxQueryParser.h b/client/cpp/BoundingBoxQueryParser.h new file mode 100644 index 00000000..63c3e7d0 --- /dev/null +++ b/client/cpp/BoundingBoxQueryParser.h @@ -0,0 +1,109 @@ +#include "CSVParserUtil.h" +namespace VDMS { +class BoundingBoxQueryParser : public CSVParserUtil{ + private: + vector rectangleKeys{"x","y","w","h"}; + void parseRectangle(string row,string queryType,Json::Value &aquery); + public: + VDMS::Response ParseAddBoundingBox(vector row, vector cols); + // VDMS::Response ParseUpdateBoundingBox(vector row, vector& cols); + +}; +}; + +VDMS::Response VDMS::BoundingBoxQueryParser::ParseAddBoundingBox(vector row, vector columnNames){ + if (row.empty() || row[0].empty()) { + throw "please provide rectangle details"; + } + + Json::Value aquery; + Json::Value allquery; + std::string command_name = "AddBoundingBox"; + + aquery["AddBoundingBox"]["_ref"] = 1; + // aquery["AddBoundingBox"]["image"] = 3; + + Json::Value aqueryf; + // aqueryf["FindImage"]["_ref"] = 3; + + parseRectangle(row[0], "AddBoundingBox", aquery); + + for (int j = 1; j < columnNames.size(); j++){ + const string& columnName = columnNames[j]; + const string& cellValue = row[j]; + + if (cellValue.empty()) { + continue; + } + + if (columnName.find("prop_") != string::npos){ + parseProperty(columnName, cellValue, command_name, aquery); + } + // else if (columnName.find("cons_") != string::npos){ + // std::string find_image = "FindImage"; + // parseConstraints(columnName, cellValue, find_image, aqueryf); + // } + } + + // allquery.append(aqueryf); + + allquery.append(aquery); + return send_to_vdms(allquery); +} + + +void VDMS::BoundingBoxQueryParser::parseRectangle(string row, string queryType, Json::Value& aquery) { + Json::Value rec; + string::size_type start = 0; + for (int c = 0; c < 4; c++) { + string::size_type end = row.find(',', start); + if (end == string::npos && c < 3) { + throw "rectangle data should have four values"; + } + string substr = row.substr(start, end - start); + try { + int intVal = stoi(substr); + rec[rectangleKeys[c]] = intVal; + } catch (const invalid_argument&) { + try { + float floatVal = stof(substr); + rec[rectangleKeys[c]] = floatVal; + } catch (const invalid_argument&) { + throw "invalid datatype of the rectangle data"; + } + } + + start = end + 1; + } + aquery[queryType]["rectangle"] = rec; +} + +// VDMS::Response VDMS::BoundingBoxQueryParser::ParseUpdateBoundingBox(vector row, vector& columnNames){ +// Json::Value aquery; +// Json::Value allquery; +// aquery["UpdateBoundingBox"]["_ref"]=3; +// std::string command_name="UpdateBoundingBox"; +// if(row[0]!=""){ +// parseRectangle(row[0],command_name,aquery); +// } +// for(int j=1;j rowvec; +// splitRowOnComma(row[j],rowvec); +// for(int v=0;v +#include +#include +#include +#include +#include +#include "rapidcsv.h" +#include "CSVParserUtil.h" + +namespace VDMS { +class CSVParser { +public: + CSVParser(std::string filename, size_t num_threads, std::string server, int port) : m_filename(filename), m_num_threads(num_threads),vdms_server(server),vdms_port(port) {} + std::vector parse_csv_lines(const std::string& filename, int start_line, int end_line, std::mutex& mutex, std::vector& all_results, const size_t thread_id) + { + rapidcsv::Document csv(filename); + + size_t rowCount = csv.GetRowCount(); + std::vector columnNames = csv.GetColumnNames(); + VDMS::CSVParserUtil csv_util(vdms_server, vdms_port, columnNames, static_cast(thread_id)); + for (int i = start_line; i < end_line; ++i) + { + std::vector row = csv.GetRow(i); + VDMS::Response result = csv_util.parse_row(row); + + std::lock_guard lock(mutex); + all_results.push_back(result); + } + + return all_results; + } +std::vector parse() { + auto start = std::chrono::high_resolution_clock::now(); + + std::ifstream file(m_filename); + if (!file) { + std::cerr << "Error opening file\n"; + + } + + int num_lines = std::count(std::istreambuf_iterator(file), std::istreambuf_iterator(), '\n'); + std::size_t lines_per_thread = num_lines / m_num_threads; + + std::mutex mutex; + std::vector threads; + std::vector all_results; + + for (size_t i = 0; i < m_num_threads; i++) { + size_t start_line = i * lines_per_thread; + size_t end_line = (i == m_num_threads - 1) ? num_lines-1 : (i + 1) * lines_per_thread; + + threads.emplace_back(&CSVParser::parse_csv_lines, this, std::ref(m_filename), start_line, end_line, std::ref(mutex),std::ref(all_results), i); + } + + for (auto& thread : threads) { + thread.join(); + } + + auto finish = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = finish - start; + std::cout << "Elapsed time: " << elapsed.count() << " s\n"; + return all_results; + } + +private: + std::string m_filename; + size_t m_num_threads; + std::string vdms_server; + int vdms_port; + + + }; +}; \ No newline at end of file diff --git a/client/cpp/CSVParserUtil.cpp b/client/cpp/CSVParserUtil.cpp new file mode 100644 index 00000000..75041c9f --- /dev/null +++ b/client/cpp/CSVParserUtil.cpp @@ -0,0 +1,635 @@ + +#include +#include +#include "CSVParserUtil.h" +#include "rapidcsv.h" +#include "EntityQueryParser.h" +#include "ImageQueryParser.h" +#include "VideoQueryParser.h" +#include "DescriptorQueryParser.h" +#include "DescriptorSetQueryParser.h" +#include "BoundingBoxQueryParser.h" +#include "ConnectionQueryParser.h" +#include +static std::mutex barrier; +std::mutex mtx; +using namespace std; +using namespace std::chrono; +using namespace VDMS; + +VDMS::CSVParserUtil::CSVParserUtil() : vdms_server("localhost"), vdms_port(55558), + vdms_client(std::unique_ptr(new VDMS::VDMSClient(vdms_server, vdms_port))) +{ + initCommandsMap(); +} + +VDMS::CSVParserUtil::CSVParserUtil(const std::string &server, int port, const std::vector columnNames, int index) : vdms_server(server), + vdms_port(port), + _columnNames(columnNames), + id(index), vdms_client(std::unique_ptr(new VDMS::VDMSClient(vdms_server, vdms_port))) +{ + initCommandsMap(); +} +void VDMS::CSVParserUtil::initCommandsMap() +{ + commands = { + {"EntityClass", EntityClass}, + {"ConnectionClass", ConnectionClass}, + {"ImagePath", ImagePath}, + {"VideoPath", VideoPath}, + {"DescriptorType", DescriptorType}, + {"DescriptorClass", DescriptorClass}, + {"RectangleBound", RectangleBound}, + {"EntityUpdate", EntityUpdate}, + {"ConnectionUpdate", ConnectionUpdate}, + {"ImageUpdate", ImageUpdate}, + {"RectangleUpdate", RectangleUpdate}}; +} +string VDMS::CSVParserUtil::function_accessing_columnNames(int i) +{ + std::lock_guard lock(mtx); + + return _columnNames[i]; +} +VDMS::Response VDMS::CSVParserUtil::parse_row(std::vector &row) +{ + VDMS::Response result; + + VDMS::CSVParserUtil::commandType queryType = get_query_type(_columnNames[0]); + switch (queryType) + { + case commandType::AddEntity: + { + EntityQueryParser entityquery; + result = entityquery.ParseAddEntity(row, _columnNames); + } + + break; + case commandType::AddConnection: + { + + ConnectionQueryParser connectionquery; + result = connectionquery.ParseAddConnection(row, _columnNames); + } + break; + + case commandType::AddImage: + { + ImageQueryParser imagequery; + result = imagequery.ParseAddImage(row, _columnNames); + } + break; + case commandType::AddVideo: + { + VideoQueryParser videoquery; + result = videoquery.ParseAddVideo(row, _columnNames); + } + break; + case commandType::AddDescriptorSet: + { + DescriptorSetQueryParser descriptorsetquery; + + result = descriptorsetquery.ParseAddDescriptorSet(row, _columnNames); + } + break; + case commandType::AddDescriptor: + { + DescriptorQueryParser descriptorquery; + result = descriptorquery.ParseAddDescriptor(row, _columnNames, id); + } + break; + case commandType::AddBoundingBox: + { + BoundingBoxQueryParser boundingboxquery; + result = boundingboxquery.ParseAddBoundingBox(row, _columnNames); + } + break; + // case commandType::UpdateEntity:{ + // EntityQueryParser update_entityquery; + // update_entityquery.ParseUpdateEntity(row, _columnNames); + // } + // break; + // case commandType::UpdateConnection:{ + // ConnectionQueryParser update_connectionquery; + // update_connectionquery.ParseUpdateConnection(row,allquery,i+1,_columnNames); + // } + // break; + // case commandType::UpdateImage:{ + // ImageQueryParser update_image_query; + // update_image_query.ParseUpdateImage(row,allquery,i+1,_columnNames); + // } + // break; + // case commandType::UpdateBoundingBox:{ + // BoundingBoxQueryParser update_boundingboxquery; + // update_boundingboxquery.ParseUpdateBoundingBox(row,allquery,i+1,_columnNames); + // } + // break; + case commandType::UNKNOWN:{ + Json::Value results; + results["status"] = -1; + results["info"] = "Command does not exist"; + result.json = results.toStyledString(); + } + break; + } + return result; +} + +int VDMS::CSVParserUtil::isBool(const std::string &s) +{ + std::string lower = s; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + if (lower == "true") + return 1; + else if (lower == "false") + return 2; + return 0; +} + +bool VDMS::CSVParserUtil::isFloat(const std::string &s) +{ + std::istringstream ss(s); + float x; + char c; + return (ss >> x) && !(ss >> c); +} + +bool VDMS::CSVParserUtil::isInt(const std::string &s) +{ + std::istringstream ss(s); + int x; + char c; + return (ss >> x) && (floor(x)) && !(ss >> c); +} + +VDMS::CSVParserUtil::commandType VDMS::CSVParserUtil::get_query_type(const string &str) +{ + CSVParserUtil::commandType querytype = commandType::UNKNOWN; + + std::lock_guard lock(CSVParserUtil::querytype_mutex); + std::map::iterator iter; + iter = commands.find(str); + if (iter != commands.end()) { + switch (commands[str]) + { + case EntityClass: + querytype = commandType::AddEntity; + break; + case ConnectionClass: + querytype = commandType::AddConnection; + break; + case ImagePath: + querytype = commandType::AddImage; + break; + case VideoPath: + querytype = commandType::AddVideo; + break; + case DescriptorType: + querytype = commandType::AddDescriptorSet; + break; + case DescriptorClass: + querytype = commandType::AddDescriptor; + break; + case RectangleBound: + querytype = commandType::AddBoundingBox; + break; + } + // std::cout << " I executed queryType "<< querytype << std::endl; + // return querytype; + } + + return querytype; +} + +VDMS::CSVParserUtil::DATATYPE VDMS::CSVParserUtil::getDataType(const string &str, const string &propname) +{ + if (propname.substr(0, 5) == "date:") + return DATATYPE::DATE; + else if (isInt(str)) + return DATATYPE::INTEGER; + else if (isFloat(str)) + return DATATYPE::FLOAT; + else if (isBool(str) == 1) + return DATATYPE::TRUE; + else if (isBool(str) == 2) + return DATATYPE::FALSE; + else + return DATATYPE::STRING; +} + +void VDMS::CSVParserUtil::parseProperty(const string &columnNames, const string &row, const string &queryType, Json::Value &aquery) +{ + std::lock_guard lock(CSVParserUtil::aquery_mutex); + string propname = columnNames.substr(5, string::npos); + // std::cout << "Inside parseProp " << propname < lock(CSVParserUtil::cons_mutex); + vector consvals = spiltrow(row); + string consname = consvals[0]; + if (consname.substr(0, 5) == "date:") + { + for (int z = 1; z < consvals.size(); z++) + { + if (z % 2 == 1) + { + aquery[queryType]["constraints"][consname.substr(5, string::npos)].append(consvals[z]); + } + else + { + Json::Value date; + date["_date"] = consvals[z]; + aquery[queryType]["constraints"][consname.substr(5, string::npos)].append(date); + } + } + } + else + { + for (int z = 1; z < consvals.size(); z++) + { + CSVParserUtil::DATATYPE dtype = getDataType(consvals[z], consname); + if (dtype == DATATYPE::TRUE) + { + aquery[queryType]["constraints"][consname].append(true); + } + else if (dtype == DATATYPE::FALSE) + { + aquery[queryType]["constraints"][consname].append(false); + } + else if (dtype == DATATYPE::INTEGER) + { + aquery[queryType]["constraints"][consname].append(stoi(consvals[z])); + } + else if (dtype == DATATYPE::FLOAT) + { + aquery[queryType]["constraints"][consname].append(stof(consvals[z])); + } + else + { + aquery[queryType]["constraints"][consname].append(consvals[z]); + } + } + } +} + +vector VDMS::CSVParserUtil::spiltrow(const string &str) +{ + string row = str; + vector rowv; + int start = 0; + for (int i = 0; i < row.size(); i++) + { + if ((row[i] == '<') || (row[i] == '>') || (row[i] == '=')) + { + if (row[i + 1] == '=') + { + rowv.push_back(row.substr(start, i - start)); + rowv.push_back(row.substr(i, 2)); + i++; + } + else + { + rowv.push_back(row.substr(start, i - start)); + rowv.push_back(row.substr(i, 1)); + } + start = i + 1; + } + else if (row[i] == ',') + { + row.erase(i, 1); + i--; + } + } + rowv.push_back(row.substr(start, string::npos)); + return rowv; +} +void VDMS::CSVParserUtil::parseOperations(string columnNames, string row, string queryType, Json::Value &aquery) +{ + + std::lock_guard lock(CSVParserUtil::ops_mutex); + string type = columnNames.substr(4, string::npos); + Json::Value opsjson; + vector opsKeys; + if (isValidOpsType(type)) + opsjson["type"] = type; + else + throw "invalid operation command name"; + vector rowvec; + int c; + splitRowOnComma(row, rowvec); + if (type == "crop") + { + if (rowvec.size() <= 3 || rowvec.size() > 4) + throw "For crop data should be of size 4"; + opsKeys = {"x", "y", "width", "height"}; + for (c = 0; c < rowvec.size(); c++) + { + string substr = rowvec[c]; + DATATYPE dType = isValidDataType(substr, 2); + + if (dType == DATATYPE::INTEGER) + opsjson[opsKeys[c]] = stoi(substr); + else if (dType == DATATYPE::FLOAT) + opsjson[opsKeys[c]] = stof(substr); + + else + { + throw "Numeric data is required for crop command"; + } + } + } + + else if (type == "threshold") + { + DATATYPE dType = isValidDataType(row, 2); + if (dType == DATATYPE::INTEGER) + opsjson["value"] = stoi(row); + else if (dType == DATATYPE::FLOAT) + opsjson["value"] = stof(row); + + else + { + throw "Numeric data is required for resize command"; + } + } + + else if (type == "resize") + { + if (rowvec.size() <= 1 || rowvec.size() > 2) + throw "For resize data should be of size 2"; + opsKeys = {"width", "height"}; + for (c = 0; c < rowvec.size(); c++) + { + string substr = rowvec[c]; + DATATYPE dType = isValidDataType(substr, 2); + + if (dType == DATATYPE::INTEGER) + opsjson[opsKeys[c]] = stoi(substr); + else if (dType == DATATYPE::FLOAT) + opsjson[opsKeys[c]] = stof(substr); + + else + { + throw "Numeric data is required for resize command"; + } + } + } + + else if (type == "flip") + { + DATATYPE dType = isValidDataType(row, 2); + if (dType == DATATYPE::INTEGER) + { + opsjson["code"] = stoi(row); + } + else + { + throw "Numeric data is required for flip command"; + } + } + + else if (type == "rotate") + { + if (rowvec.size() <= 1 || rowvec.size() > 2) + throw "For rotate data should be of size 2"; + opsKeys = {"angle", "resize"}; + for (c = 0; c < rowvec.size(); c++) + { + string substr = rowvec[c]; + if (c == 0) + { + DATATYPE dType = isValidDataType(substr, 2); + if (dType == DATATYPE::INTEGER) + opsjson[opsKeys[c]] = stoi(substr); + else if (dType == DATATYPE::FLOAT) + opsjson[opsKeys[c]] = stof(substr); + + else + { + throw "Numeric data is required for resize command"; + } + } + else if (c == 1) + { + DATATYPE dType = isValidDataType(substr, 1); + if (dType == DATATYPE::TRUE) + opsjson[opsKeys[c]] = true; + else if (dType == DATATYPE::FALSE) + opsjson[opsKeys[c]] = false; + + else + { + throw "Boolean data is required for rotate resize"; + } + } + } + } + + else if (type == "interval") + { + if (rowvec.size() <= 2 || rowvec.size() > 3) + throw "interval operation has 3 values to specify"; + opsKeys = {"start", "stop", "step"}; + for (c = 0; c < rowvec.size(); c++) + { + string substr = rowvec[c]; + DATATYPE dType = isValidDataType(substr, 2); + if (dType == DATATYPE::INTEGER) + { + opsjson[opsKeys[c]] = stoi(substr); + } + else + { + throw "Numeric datatype is required for the interval"; + } + } + } + + aquery[queryType]["operations"].append(opsjson); +} + +void VDMS::CSVParserUtil::splitRowOnComma(const std::string &row, std::vector &rowvec) +{ + std::string::size_type start = 0; + while (start != std::string::npos) + { + std::string::size_type end = row.find(',', start); + if (end == std::string::npos) + { + if (start < row.size()) + { + rowvec.emplace_back(std::move(row.substr(start))); + } + break; + } + if (end > start) + { + rowvec.emplace_back(std::move(row.substr(start, end - start))); + } + start = end + 1; + } +} + +VDMS::CSVParserUtil::DATATYPE VDMS::CSVParserUtil ::isValidDataType(string data, int type) +{ + CSVParserUtil::DATATYPE actualtype = getDataType(data, ""); + return actualtype; + + // if(type==2 && (actualt=3 || acype=tualtype==4))//2 is for num + // return actualtype; + // else if(type==1 && (actualtype==1 || actualtype==2))//1 is for bool + // return actualtype; + // else + // return -1; +} + +bool VDMS::CSVParserUtil::isValidOpsType(string &type) +{ + if (type == "resize" || type == "threshold" || type == "flip" || type == "rotate" || type == "interval" || type == "crop") + return true; + return false; +} + +void VDMS::CSVParserUtil::parseBlobFile(const std::string &filename, std::string **descriptor_ptr) +{ + std::vector v; + + std::ifstream input(filename); + if (!input.is_open()) + { + // handle error if file cannot be opened + std::cerr << "Error: Could not open file " << filename << std::endl; + *descriptor_ptr = nullptr; + return; + } + + std::string str; + input >> str; + + std::stringstream ss(str); + while (ss.good()) + { + std::string substr; + getline(ss, substr, ';'); + v.push_back(std::stof(substr)); + } + + input.close(); + + // Convert vector to array of bytes + const size_t byteSize = v.size() * sizeof(float); + unsigned char *bytes = new unsigned char[byteSize]; + std::memcpy(bytes, v.data(), byteSize); + + // Copy bytes to dynamically allocated string + *descriptor_ptr = new std::string(reinterpret_cast(bytes), byteSize); + + // Clean up dynamically-allocated memory + delete[] bytes; +} + +void VDMS::CSVParserUtil::videoToString(const std::string &filename, std::string **video_data_ptr) +{ + // Open the video file in binary mode + std::ifstream file(filename, std::ios::binary); + + if (!file) + { + std::cerr << "Failed to open file: " << filename << std::endl; + *video_data_ptr = nullptr; + } + else + { + // Read the entire content of the file into a string + std::stringstream buffer; + buffer << file.rdbuf(); + std::string video_data = buffer.str(); + + *video_data_ptr = new std::string(video_data); + } + + // Close the file + file.close(); +} + +void VDMS::CSVParserUtil::read_blob_image(const std::string &filename, std::string **image_data_ptr) +{ + std::ifstream file(filename, std::ios::binary); + + if (file.is_open()) + { + + // Get the size of the file + file.seekg(0, std::ios::end); + size_t size = file.tellg(); + file.seekg(0, std::ios::beg); + + // Allocate a buffer to hold the file data + char *buffer = new char[size]; + + // Read the file data into the buffer + file.read(buffer, size); + + if (file.gcount() != size) + { + std::cerr << "Error: Failed to read entire file." << std::endl; + delete[] buffer; + *image_data_ptr = nullptr; + return; + } + + // Close the file + file.close(); + + // Allocate a new std::string to hold the image data + std::string *image_data = new std::string; + image_data->assign(buffer, size); + + // Free the buffer + delete[] buffer; + + // Assign the std::string pointer to the image_data_ptr + *image_data_ptr = image_data; + } + else + { + std::cerr << "Error: Failed to open file." << std::endl; + *image_data_ptr = nullptr; + } +} +VDMS::Response VDMS::CSVParserUtil::send_to_vdms(const Json::Value &query, + const std::vector blobs) +{ + Json::StyledWriter _fastwriter; + + return vdms_client->query(_fastwriter.write(query), blobs); +} diff --git a/client/cpp/CSVParserUtil.h b/client/cpp/CSVParserUtil.h new file mode 100644 index 00000000..ad30bd9d --- /dev/null +++ b/client/cpp/CSVParserUtil.h @@ -0,0 +1,102 @@ +#pragma once +#include +#include +#include +#include "rapidcsv.h" +#include +#include +#include +#include +#include +#include "VDMSClient.h" + +using namespace std; +using namespace std::chrono; + +namespace VDMS { +class CSVParserUtil { + + enum QueryType { EntityClass, + ConnectionClass, + ImagePath, + VideoPath, + DescriptorType, + DescriptorClass, + RectangleBound, + EntityUpdate, + ConnectionUpdate, + ImageUpdate, + RectangleUpdate + }; + enum class commandType { + AddEntity, + AddConnection, + AddImage, + AddVideo, + AddDescriptorSet, + AddDescriptor, + AddBoundingBox, + UpdateEntity, + UpdateConnection, + UpdateImage, + UpdateBoundingBox, + UNKNOWN + + }; + enum class DATATYPE{ + TRUE, + FALSE, + INTEGER, + FLOAT, + STRING, + DATE, + WRONG + + }; + std::map commands; + std::map command_list; + + + + public: + CSVParserUtil(); + CSVParserUtil(const std::string&, int port, const std::vector, int id ); + void initCommandsMap(); + int isBool( const string& data); + bool isFloat(const string &); + bool isInt(const string &); + VDMS::Response parse_row(std::vector& fields); + commandType get_query_type(const string &data); + CSVParserUtil::DATATYPE getDataType(const string& data,const string& colname); + void parseProperty(const string& columnNames,const string& row,const string& queryType, Json::Value &aquery); + void parseConstraints(const string &columnNames,const string& row, string& queryType, Json::Value &aquery); + void parseOperations(const string columnNames,string row,string queryType,Json::Value &aquery); + + // void parseCSVdata(int startrowcount,int endcount,rapidcsv::Document &doc, int &); + vector spiltrow(const string& row); + bool isValidOpsType(string& type); + void splitRowOnComma(const std::string& row, std::vector& rowvec); + + string function_accessing_columnNames(int i); + DATATYPE isValidDataType(string data,int type); + void read_blob_image(const std::string& filename, std::string** image_data_ptr); + void videoToString(const std::string& filename, std::string** video_data); + void parseBlobFile(const std::string& filename, std::string** descriptor_ptr); + + VDMS::Response send_to_vdms(const Json::Value &json_query, const std::vector blobs = {}); + + public: + std::string vdms_server; + int vdms_port; + std::vector _columnNames; + std::mutex querytype_mutex; + std::mutex aquery_mutex; + std::mutex cons_mutex; + std::mutex ops_mutex; + int id; + std::unique_ptr vdms_client; + + }; +}; + + diff --git a/client/cpp/ConnectionQueryParser.h b/client/cpp/ConnectionQueryParser.h new file mode 100644 index 00000000..afd8394d --- /dev/null +++ b/client/cpp/ConnectionQueryParser.h @@ -0,0 +1,115 @@ +#include "CSVParserUtil.h" +namespace VDMS { +class ConnectionQueryParser : public CSVParserUtil{ + public: + VDMS::Response ParseAddConnection(vector row, vector & cols); + // VDMS::Response ParseUpdateConnection(vector row, vector & cols); +}; +}; + +VDMS::Response VDMS::ConnectionQueryParser::ParseAddConnection(vector row, vector & columnNames){ + Json::Value aquery; + Json::Value allquery; + Json::Value find_query1, find_query2,find_query; + + if (row[0].empty()) { + std::cerr << "Error: Connection Class not provided\n"; + +} + +// Set command name and connection class +const string command_name = "AddConnection"; +const string connection_class = row[0]; +int ref1=1, ref2=3; +aquery["AddConnection"]["class"] = connection_class; + +// Parse class1 and class2 columns +for (int i = 1; i < columnNames.size(); i++) { + string column_name = columnNames[i]; + string column_value = row[i]; + string column_type = column_name.substr(0, 5); + string command="FindEntity"; + + + if (column_value.empty()) { + continue; + } + + + + if (column_name.find('@') != std::string::npos) { + std::size_t at_pos = column_name.find('@'); + + if (at_pos != std::string::npos) { + // Extract the name and id substrings. + std::string class1 = column_name.substr(0, at_pos); + std::string class_prop = column_name.substr(at_pos + 1); + + find_query["FindEntity"]["class"]=class1; + find_query["FindEntity"]["_ref"]=ref1++; + find_query["FindEntity"]["constraints"][class_prop][0] = "=="; + find_query["FindEntity"]["constraints"][class_prop][1]=column_value; + } + allquery.append(find_query); + + } + + + + // if (column_type == "cons_") { + // parseConstraints(column_name, column_value, command, find_query1); + // parseConstraints(column_name, column_value, command, find_query2); + // } + // if (column_type == "prop_") { + // parseProperty(column_name, column_value, command_name, aquery); + // } + find_query.clear(); + + +} + +// Set connection references +aquery["AddConnection"]["ref1"] = allquery[0]["FindEntity"]["_ref"]; +aquery["AddConnection"]["ref2"] = allquery[1]["FindEntity"]["_ref"]; + + + +allquery.append(aquery); + + +return send_to_vdms(allquery); +} + +// VDMS::Response VDMS::ConnectionQueryParser::ParseUpdateConnection(vector row, vector & columnNames){ +// Json::Value aquery; +// Json::Value aqueryf; +// Json::Value allquery; +// if(row[0]=="") +// throw "Connection Class not provided"; +// std::string command_name="UpdateConnection"; +// aquery["UpdateConnection"]["class"]=row[0]; +// aquery["UpdateConnection"]["_ref"]=96; +// aqueryf["FindConnection"]["class"]=row[0]; +// aqueryf["FindConnection"]["_ref"]=96; +// for(int j=1;j rowvec; +// splitRowOnComma(row[j],rowvec); +// for(int v=0;v row, vector& columnNames, int id); + +}; +}; + +VDMS::Response VDMS::DescriptorQueryParser::ParseAddDescriptor(vector row, vector& columnNames, int id){ + + if(row[0]==""){ + throw "Set not provided"; + } + Json::Value aquery; + Json::Value fullquery; + std::vector blobs; + std::string* descriptor; + std::string command_name="AddDescriptor"; + aquery["AddDescriptor"]["set"]=row[0]; + aquery["AddDescriptor"]["_ref"]=id+3; + for(int j=1;j row, vector& columnNames); + bool isValidMetric(string& metric); + bool isValidEngine(string& engine); +}; +}; +VDMS::Response VDMS::DescriptorSetQueryParser::ParseAddDescriptorSet(vector row, vector& columnNames){ + if(row[0]==""){ + throw "Descriptor Name not provided"; + } + Json::Value aquery; + Json::Value fullquery; + std::string command_name="AddDescriptorSet"; + aquery["AddDescriptorSet"]["name"]=row[0]; + + + for(int j=1;j + +namespace VDMS +{ + + class EntityQueryParser : public CSVParserUtil + { + public: + VDMS::Response ParseAddEntity(vector row, vector &cols); + // VDMS::Response ParseUpdateEntity(vector row, vector & cols); + }; +}; + +VDMS::Response VDMS::EntityQueryParser::ParseAddEntity(vector row, vector &cols) +{ + Json::Value aquery; + Json::Value fullquery; + + std::string command_name = "AddEntity"; + // std::cout << command_name << columnNames.size() < row, vector & cols){ +// Json:: Value aquery; +// Json::Value all_query; +// Json::Value find_query; +// std::string command_name="UpdateEntity"; +// if(row[0]==""){ +// throw "Entity Class not specified"; +// } +// aquery["UpdateEntity"]["class"]=row[0]; +// int ref=10; +// std::cout << _columnNames[0] < rowvec; +// splitRowOnComma(row[j],rowvec); +// for(int v=0;v row, vector columnNames); + // VDMS::Response ParseUpdateImage(vector row, vector columnNames); + bool ValidImageFormat(string data); + + +}; +}; + + + +VDMS::Response VDMS::ImageQueryParser::ParseAddImage(vector row, vector columnNames){ + Json::Value aquery; + Json::Value fullquery; + std::vector blobs; + // + if(row[0].empty()) + throw "Image path is not specified"; + if (columnNames.size() == 0) { + throw std::invalid_argument("Error: Column names vector is empty."); + } + + std::string command_name="AddImage"; + + aquery["AddImage"]["_ref"]=11; + + std::string name =row[0]; + + std::string* image_data_ptr = nullptr; + + read_blob_image(name, &image_data_ptr); + + // std::cout << *image_data_ptr << std::endl; + if(image_data_ptr!=nullptr){ + blobs.push_back(image_data_ptr); + // std::cout <<*blobs[0] < row, vector columnNames){ +// Json :: Value aquery; + +// Json::Value fullquery; + +// std::string command_name="UpdateIamge"; +// aquery["UpdateImage"]["_ref"]=12; +// for(int j=1;j rowvec; +// splitRowOnComma(row[j],rowvec); +// for(int v=0;v(end - start); +// cout << "duaration in ms is "< blobs) { diff --git a/client/cpp/VDMSClient.h b/client/cpp/VDMSClient.h index 54ca8f9e..ced571d7 100644 --- a/client/cpp/VDMSClient.h +++ b/client/cpp/VDMSClient.h @@ -32,6 +32,7 @@ #include #include #include "comm/Connection.h" +// #include "CSVParser.h" namespace VDMS { @@ -39,6 +40,7 @@ namespace VDMS { std::string json; std::vector blobs; }; + class VDMSClient { static const int VDMS_PORT = 55555; @@ -49,6 +51,7 @@ namespace VDMS { // will leave the functioning like that. If the client has a need to // disconnect and connect specifically, then we can add explicit calls. comm::ConnClient _conn; + public: VDMSClient(std::string addr = "localhost", int port = VDMS_PORT); @@ -56,6 +59,7 @@ namespace VDMS { // Blocking call VDMS::Response query(const std::string &json_query, const std::vector blobs = {}); - + // void parse_csv_file(std::string filename, std::string , int); + }; }; diff --git a/client/cpp/VideoQueryParser.h b/client/cpp/VideoQueryParser.h new file mode 100644 index 00000000..3c89758b --- /dev/null +++ b/client/cpp/VideoQueryParser.h @@ -0,0 +1,84 @@ +#pragma once +#include "CSVParserUtil.h" +namespace VDMS { +class VideoQueryParser : public CSVParserUtil{ + public: + VDMS::Response ParseAddVideo(vector row, vector& columnNames); + bool isValidCodec(string& row); + bool isValidContainer(string& row); + +}; +} +VDMS::Response VDMS::VideoQueryParser::ParseAddVideo(vector row, vector& columnNames){ + Json::Value aquery; + Json::Value fullquery; + std::vector blobs; + if(row[0]=="") + throw "Video not provided"; + std::string command_name="AddVideo"; + + std::string video_name=row[0]; + try { + std::string* video_data_ptr; + CSVParserUtil::videoToString(video_name, &video_data_ptr); + + if(video_data_ptr!=nullptr){ + blobs.push_back(video_data_ptr); + // std::cout <<*blobs[0] < +#include +#include +#ifdef HAS_CODECVT +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + +namespace rapidcsv +{ +#if defined(_MSC_VER) + static const bool sPlatformHasCR = true; +#else + static const bool sPlatformHasCR = false; +#endif + + /** + * @brief Datastructure holding parameters controlling how invalid numbers (including + * empty strings) should be handled. + */ + struct ConverterParams + { + /** + * @brief Constructor + * @param pHasDefaultConverter specifies if conversion of non-numerical strings shall be + * converted to a default numerical value, instead of causing + * an exception to be thrown (default). + * @param pDefaultFloat floating-point default value to represent invalid numbers. + * @param pDefaultInteger integer default value to represent invalid numbers. + */ + explicit ConverterParams(const bool pHasDefaultConverter = false, + const long double pDefaultFloat = std::numeric_limits::signaling_NaN(), + const long long pDefaultInteger = 0) + : mHasDefaultConverter(pHasDefaultConverter) + , mDefaultFloat(pDefaultFloat) + , mDefaultInteger(pDefaultInteger) + { + } + + /** + * @brief specifies if conversion of non-numerical strings shall be converted to a default + * numerical value, instead of causing an exception to be thrown (default). + */ + bool mHasDefaultConverter; + + /** + * @brief floating-point default value to represent invalid numbers. + */ + long double mDefaultFloat; + + /** + * @brief integer default value to represent invalid numbers. + */ + long long mDefaultInteger; + }; + + /** + * @brief Exception thrown when attempting to access Document data in a datatype which + * is not supported by the Converter class. + */ + class no_converter : public std::exception + { + /** + * @brief Provides details about the exception + * @returns an explanatory string + */ + virtual const char* what() const throw() + { + return "unsupported conversion datatype"; + } + }; + + /** + * @brief Class providing conversion to/from numerical datatypes and strings. Only + * intended for rapidcsv internal usage, but exposed externally to allow + * specialization for custom datatype conversions. + */ + template + class Converter + { + public: + /** + * @brief Constructor + * @param pConverterParams specifies how conversion of non-numerical values to + * numerical datatype shall be handled. + */ + Converter(const ConverterParams& pConverterParams) + : mConverterParams(pConverterParams) + { + } + + /** + * @brief Converts numerical value to string representation. + * @param pVal numerical value + * @param pStr output string + */ + void ToStr(const T& pVal, std::string& pStr) const + { + if (typeid(T) == typeid(int) || + typeid(T) == typeid(long) || + typeid(T) == typeid(long long) || + typeid(T) == typeid(unsigned) || + typeid(T) == typeid(unsigned long) || + typeid(T) == typeid(unsigned long long) || + typeid(T) == typeid(float) || + typeid(T) == typeid(double) || + typeid(T) == typeid(long double) || + typeid(T) == typeid(char)) + { + std::ostringstream out; + out << pVal; + pStr = out.str(); + } + else + { + throw no_converter(); + } + } + + /** + * @brief Converts string holding a numerical value to numerical datatype representation. + * @param pVal numerical value + * @param pStr output string + */ + void ToVal(const std::string& pStr, T& pVal) const + { + try + { + if (typeid(T) == typeid(int)) + { + pVal = static_cast(std::stoi(pStr)); + return; + } + else if (typeid(T) == typeid(long)) + { + pVal = static_cast(std::stol(pStr)); + return; + } + else if (typeid(T) == typeid(long long)) + { + pVal = static_cast(std::stoll(pStr)); + return; + } + else if (typeid(T) == typeid(unsigned)) + { + pVal = static_cast(std::stoul(pStr)); + return; + } + else if (typeid(T) == typeid(unsigned long)) + { + pVal = static_cast(std::stoul(pStr)); + return; + } + else if (typeid(T) == typeid(unsigned long long)) + { + pVal = static_cast(std::stoull(pStr)); + return; + } + } + catch (...) + { + if (!mConverterParams.mHasDefaultConverter) + { + throw; + } + else + { + pVal = static_cast(mConverterParams.mDefaultInteger); + return; + } + } + + try + { + if (typeid(T) == typeid(float)) + { + pVal = static_cast(std::stof(pStr)); + return; + } + else if (typeid(T) == typeid(double)) + { + pVal = static_cast(std::stod(pStr)); + return; + } + else if (typeid(T) == typeid(long double)) + { + pVal = static_cast(std::stold(pStr)); + return; + } + } + catch (...) + { + if (!mConverterParams.mHasDefaultConverter) + { + throw; + } + else + { + pVal = static_cast(mConverterParams.mDefaultFloat); + return; + } + } + + if (typeid(T) == typeid(char)) + { + pVal = static_cast(pStr[0]); + return; + } + else + { + throw no_converter(); + } + } + + private: + const ConverterParams& mConverterParams; + }; + + /** + * @brief Specialized implementation handling string to string conversion. + * @param pVal string + * @param pStr string + */ + template<> + inline void Converter::ToStr(const std::string& pVal, std::string& pStr) const + { + pStr = pVal; + } + + /** + * @brief Specialized implementation handling string to string conversion. + * @param pVal string + * @param pStr string + */ + template<> + inline void Converter::ToVal(const std::string& pStr, std::string& pVal) const + { + pVal = pStr; + } + + template + using ConvFunc = std::function; + + /** + * @brief Datastructure holding parameters controlling which row and column should be + * treated as labels. + */ + struct LabelParams + { + /** + * @brief Constructor + * @param pColumnNameIdx specifies the zero-based row index of the column labels, setting + * it to -1 prevents column lookup by label name, and gives access + * to all rows as document data. Default: 0 + * @param pRowNameIdx specifies the zero-based column index of the row labels, setting + * it to -1 prevents row lookup by label name, and gives access + * to all columns as document data. Default: -1 + */ + explicit LabelParams(const int pColumnNameIdx = 0, const int pRowNameIdx = -1) + : mColumnNameIdx(pColumnNameIdx) + , mRowNameIdx(pRowNameIdx) + { + } + + /** + * @brief specifies the zero-based row index of the column labels. + */ + int mColumnNameIdx; + + /** + * @brief specifies the zero-based column index of the row labels. + */ + int mRowNameIdx; + }; + + /** + * @brief Datastructure holding parameters controlling how the CSV data fields are separated. + */ + struct SeparatorParams + { + /** + * @brief Constructor + * @param pSeparator specifies the column separator (default ','). + * @param pTrim specifies whether to trim leading and trailing spaces from + * cells read (default false). + * @param pHasCR specifies whether a new document (i.e. not an existing document read) + * should use CR/LF instead of only LF (default is to use standard + * behavior of underlying platforms - CR/LF for Win, and LF for others). + * @param pQuotedLinebreaks specifies whether to allow line breaks in quoted text (default false) + * @param pAutoQuote specifies whether to automatically dequote data during read, and add + * quotes during write (default true). + */ + explicit SeparatorParams(const char pSeparator = ',', const bool pTrim = false, + const bool pHasCR = sPlatformHasCR, const bool pQuotedLinebreaks = false, + const bool pAutoQuote = true) + : mSeparator(pSeparator) + , mTrim(pTrim) + , mHasCR(pHasCR) + , mQuotedLinebreaks(pQuotedLinebreaks) + , mAutoQuote(pAutoQuote) + { + } + + /** + * @brief specifies the column separator. + */ + char mSeparator; + + /** + * @brief specifies whether to trim leading and trailing spaces from cells read. + */ + bool mTrim; + + /** + * @brief specifies whether new documents should use CR/LF instead of LF. + */ + bool mHasCR; + + /** + * @brief specifies whether to allow line breaks in quoted text. + */ + bool mQuotedLinebreaks; + + /** + * @brief specifies whether to automatically dequote cell data. + */ + bool mAutoQuote; + }; + + /** + * @brief Datastructure holding parameters controlling how special line formats should be + * treated. + */ + struct LineReaderParams + { + /** + * @brief Constructor + * @param pSkipCommentLines specifies whether to skip lines prefixed with + * mCommentPrefix. Default: false + * @param pCommentPrefix specifies which prefix character to indicate a comment + * line. Default: # + * @param pSkipEmptyLines specifies whether to skip empty lines. Default: false + */ + explicit LineReaderParams(const bool pSkipCommentLines = false, + const char pCommentPrefix = '#', + const bool pSkipEmptyLines = false) + : mSkipCommentLines(pSkipCommentLines) + , mCommentPrefix(pCommentPrefix) + , mSkipEmptyLines(pSkipEmptyLines) + { + } + + /** + * @brief specifies whether to skip lines prefixed with mCommentPrefix. + */ + bool mSkipCommentLines; + + /** + * @brief specifies which prefix character to indicate a comment line. + */ + char mCommentPrefix; + + /** + * @brief specifies whether to skip empty lines. + */ + bool mSkipEmptyLines; + }; + + /** + * @brief Class representing a CSV document. + */ + class Document + { + public: + /** + * @brief Constructor + * @param pPath specifies the path of an existing CSV-file to populate the Document + * data with. + * @param pLabelParams specifies which row and column should be treated as labels. + * @param pSeparatorParams specifies which field and row separators should be used. + * @param pConverterParams specifies how invalid numbers (including empty strings) should be + * handled. + * @param pLineReaderParams specifies how special line formats should be treated. + */ + explicit Document(const std::string& pPath = std::string(), + const LabelParams& pLabelParams = LabelParams(), + const SeparatorParams& pSeparatorParams = SeparatorParams(), + const ConverterParams& pConverterParams = ConverterParams(), + const LineReaderParams& pLineReaderParams = LineReaderParams()) + : mPath(pPath) + , mLabelParams(pLabelParams) + , mSeparatorParams(pSeparatorParams) + , mConverterParams(pConverterParams) + , mLineReaderParams(pLineReaderParams) + { + if (!mPath.empty()) + { + ReadCsv(); + } + } + + /** + * @brief Constructor + * @param pStream specifies an input stream to read CSV data from. + * @param pLabelParams specifies which row and column should be treated as labels. + * @param pSeparatorParams specifies which field and row separators should be used. + * @param pConverterParams specifies how invalid numbers (including empty strings) should be + * handled. + * @param pLineReaderParams specifies how special line formats should be treated. + */ + explicit Document(std::istream& pStream, + const LabelParams& pLabelParams = LabelParams(), + const SeparatorParams& pSeparatorParams = SeparatorParams(), + const ConverterParams& pConverterParams = ConverterParams(), + const LineReaderParams& pLineReaderParams = LineReaderParams()) + : mPath() + , mLabelParams(pLabelParams) + , mSeparatorParams(pSeparatorParams) + , mConverterParams(pConverterParams) + , mLineReaderParams(pLineReaderParams) + { + ReadCsv(pStream); + } + + /** + * @brief Read Document data from file. + * @param pPath specifies the path of an existing CSV-file to populate the Document + * data with. + * @param pLabelParams specifies which row and column should be treated as labels. + * @param pSeparatorParams specifies which field and row separators should be used. + * @param pConverterParams specifies how invalid numbers (including empty strings) should be + * handled. + * @param pLineReaderParams specifies how special line formats should be treated. + */ + void Load(const std::string& pPath, + const LabelParams& pLabelParams = LabelParams(), + const SeparatorParams& pSeparatorParams = SeparatorParams(), + const ConverterParams& pConverterParams = ConverterParams(), + const LineReaderParams& pLineReaderParams = LineReaderParams()) + { + mPath = pPath; + mLabelParams = pLabelParams; + mSeparatorParams = pSeparatorParams; + mConverterParams = pConverterParams; + mLineReaderParams = pLineReaderParams; + ReadCsv(); + } + + /** + * @brief Read Document data from stream. + * @param pStream specifies an input stream to read CSV data from. + * @param pLabelParams specifies which row and column should be treated as labels. + * @param pSeparatorParams specifies which field and row separators should be used. + * @param pConverterParams specifies how invalid numbers (including empty strings) should be + * handled. + * @param pLineReaderParams specifies how special line formats should be treated. + */ + void Load(std::istream& pStream, + const LabelParams& pLabelParams = LabelParams(), + const SeparatorParams& pSeparatorParams = SeparatorParams(), + const ConverterParams& pConverterParams = ConverterParams(), + const LineReaderParams& pLineReaderParams = LineReaderParams()) + { + mPath = ""; + mLabelParams = pLabelParams; + mSeparatorParams = pSeparatorParams; + mConverterParams = pConverterParams; + mLineReaderParams = pLineReaderParams; + ReadCsv(pStream); + } + + /** + * @brief Write Document data to file. + * @param pPath optionally specifies the path where the CSV-file will be created + * (if not specified, the original path provided when creating or + * loading the Document data will be used). + */ + void Save(const std::string& pPath = std::string()) + { + if (!pPath.empty()) + { + mPath = pPath; + } + WriteCsv(); + } + + /** + * @brief Write Document data to stream. + * @param pStream specifies an output stream to write the data to. + */ + void Save(std::ostream& pStream) + { + WriteCsv(pStream); + } + + /** + * @brief Clears loaded Document data. + * + */ + void Clear() + { + mData.clear(); + mColumnNames.clear(); + mRowNames.clear(); +#ifdef HAS_CODECVT + mIsUtf16 = false; + mIsLE = false; +#endif + } + + /** + * @brief Get column index by name. + * @param pColumnName column label name. + * @returns zero-based column index. + */ + ssize_t GetColumnIdx(const std::string& pColumnName) const + { + if (mLabelParams.mColumnNameIdx >= 0) + { + if (mColumnNames.find(pColumnName) != mColumnNames.end()) + { + return mColumnNames.at(pColumnName) - (mLabelParams.mRowNameIdx + 1); + } + } + return -1; + } + + /** + * @brief Get column by index. + * @param pColumnIdx zero-based column index. + * @returns vector of column data. + */ + template + std::vector GetColumn(const size_t pColumnIdx) const + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + std::vector column; + Converter converter(mConverterParams); + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + if (std::distance(mData.begin(), itRow) > mLabelParams.mColumnNameIdx) + { + if (columnIdx < static_cast(itRow->size())) + { + T val; + converter.ToVal(itRow->at(columnIdx), val); + column.push_back(val); + } + else + { + const std::string errStr = "requested column index " + + std::to_string(columnIdx - (mLabelParams.mRowNameIdx + 1)) + " >= " + + std::to_string(itRow->size() - (mLabelParams.mRowNameIdx + 1)) + + " (number of columns on row index " + + std::to_string(std::distance(mData.begin(), itRow) - + (mLabelParams.mColumnNameIdx + 1)) + ")"; + throw std::out_of_range(errStr); + } + } + } + return column; + } + + /** + * @brief Get column by index. + * @param pColumnIdx zero-based column index. + * @param pToVal conversion function. + * @returns vector of column data. + */ + template + std::vector GetColumn(const size_t pColumnIdx, ConvFunc pToVal) const + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + std::vector column; + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + if (std::distance(mData.begin(), itRow) > mLabelParams.mColumnNameIdx) + { + T val; + pToVal(itRow->at(columnIdx), val); + column.push_back(val); + } + } + return column; + } + + /** + * @brief Get column by name. + * @param pColumnName column label name. + * @returns vector of column data. + */ + template + std::vector GetColumn(const std::string& pColumnName) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + return GetColumn(columnIdx); + } + + /** + * @brief Get column by name. + * @param pColumnName column label name. + * @param pToVal conversion function. + * @returns vector of column data. + */ + template + std::vector GetColumn(const std::string& pColumnName, ConvFunc pToVal) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + return GetColumn(columnIdx, pToVal); + } + + /** + * @brief Set column by index. + * @param pColumnIdx zero-based column index. + * @param pColumn vector of column data. + */ + template + void SetColumn(const size_t pColumnIdx, const std::vector& pColumn) + { + const size_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + + while (pColumn.size() + (mLabelParams.mColumnNameIdx + 1) > GetDataRowCount()) + { + std::vector row; + row.resize(GetDataColumnCount()); + mData.push_back(row); + } + + if ((columnIdx + 1) > GetDataColumnCount()) + { + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + itRow->resize(columnIdx + 1 + (mLabelParams.mRowNameIdx + 1)); + } + } + + Converter converter(mConverterParams); + for (auto itRow = pColumn.begin(); itRow != pColumn.end(); ++itRow) + { + std::string str; + converter.ToStr(*itRow, str); + mData.at(std::distance(pColumn.begin(), itRow) + (mLabelParams.mColumnNameIdx + 1)).at(columnIdx) = str; + } + } + + /** + * @brief Set column by name. + * @param pColumnName column label name. + * @param pColumn vector of column data. + */ + template + void SetColumn(const std::string& pColumnName, const std::vector& pColumn) + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + SetColumn(columnIdx, pColumn); + } + + /** + * @brief Remove column by index. + * @param pColumnIdx zero-based column index. + */ + void RemoveColumn(const size_t pColumnIdx) + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + itRow->erase(itRow->begin() + columnIdx); + } + } + + /** + * @brief Remove column by name. + * @param pColumnName column label name. + */ + void RemoveColumn(const std::string& pColumnName) + { + ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + RemoveColumn(columnIdx); + } + + /** + * @brief Insert column at specified index. + * @param pColumnIdx zero-based column index. + * @param pColumn vector of column data (optional argument). + * @param pColumnName column label name (optional argument). + */ + template + void InsertColumn(const size_t pColumnIdx, const std::vector& pColumn = std::vector(), + const std::string& pColumnName = std::string()) + { + const size_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + + std::vector column; + if (pColumn.empty()) + { + column.resize(GetDataRowCount()); + } + else + { + column.resize(pColumn.size() + (mLabelParams.mColumnNameIdx + 1)); + Converter converter(mConverterParams); + for (auto itRow = pColumn.begin(); itRow != pColumn.end(); ++itRow) + { + std::string str; + converter.ToStr(*itRow, str); + const size_t rowIdx = std::distance(pColumn.begin(), itRow) + (mLabelParams.mColumnNameIdx + 1); + column.at(rowIdx) = str; + } + } + + while (column.size() > GetDataRowCount()) + { + std::vector row; + const size_t columnCount = std::max(static_cast(mLabelParams.mColumnNameIdx + 1), + GetDataColumnCount()); + row.resize(columnCount); + mData.push_back(row); + } + + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + const size_t rowIdx = std::distance(mData.begin(), itRow); + itRow->insert(itRow->begin() + columnIdx, column.at(rowIdx)); + } + + if (!pColumnName.empty()) + { + SetColumnName(pColumnIdx, pColumnName); + } + } + + /** + * @brief Get number of data columns (excluding label columns). + * @returns column count. + */ + size_t GetColumnCount() const + { + const ssize_t count = static_cast((mData.size() > 0) ? mData.at(0).size() : 0) - + (mLabelParams.mRowNameIdx + 1); + return (count >= 0) ? count : 0; + } + + /** + * @brief Get row index by name. + * @param pRowName row label name. + * @returns zero-based row index. + */ + ssize_t GetRowIdx(const std::string& pRowName) const + { + if (mLabelParams.mRowNameIdx >= 0) + { + if (mRowNames.find(pRowName) != mRowNames.end()) + { + return mRowNames.at(pRowName) - (mLabelParams.mColumnNameIdx + 1); + } + } + return -1; + } + + /** + * @brief Get row by index. + * @param pRowIdx zero-based row index. + * @returns vector of row data. + */ + template + std::vector GetRow(const size_t pRowIdx) const + { + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + std::vector row; + Converter converter(mConverterParams); + for (auto itCol = mData.at(rowIdx).begin(); itCol != mData.at(rowIdx).end(); ++itCol) + { + if (std::distance(mData.at(rowIdx).begin(), itCol) > mLabelParams.mRowNameIdx) + { + T val; + converter.ToVal(*itCol, val); + row.push_back(val); + } + } + return row; + } + + /** + * @brief Get row by index. + * @param pRowIdx zero-based row index. + * @param pToVal conversion function. + * @returns vector of row data. + */ + template + std::vector GetRow(const size_t pRowIdx, ConvFunc pToVal) const + { + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + std::vector row; + Converter converter(mConverterParams); + for (auto itCol = mData.at(rowIdx).begin(); itCol != mData.at(rowIdx).end(); ++itCol) + { + if (std::distance(mData.at(rowIdx).begin(), itCol) > mLabelParams.mRowNameIdx) + { + T val; + pToVal(*itCol, val); + row.push_back(val); + } + } + return row; + } + + /** + * @brief Get row by name. + * @param pRowName row label name. + * @returns vector of row data. + */ + template + std::vector GetRow(const std::string& pRowName) const + { + ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + return GetRow(rowIdx); + } + + /** + * @brief Get row by name. + * @param pRowName row label name. + * @param pToVal conversion function. + * @returns vector of row data. + */ + template + std::vector GetRow(const std::string& pRowName, ConvFunc pToVal) const + { + ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + return GetRow(rowIdx, pToVal); + } + + /** + * @brief Set row by index. + * @param pRowIdx zero-based row index. + * @param pRow vector of row data. + */ + template + void SetRow(const size_t pRowIdx, const std::vector& pRow) + { + const size_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + + while ((rowIdx + 1) > GetDataRowCount()) + { + std::vector row; + row.resize(GetDataColumnCount()); + mData.push_back(row); + } + + if (pRow.size() > GetDataColumnCount()) + { + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + itRow->resize(pRow.size() + (mLabelParams.mRowNameIdx + 1)); + } + } + + Converter converter(mConverterParams); + for (auto itCol = pRow.begin(); itCol != pRow.end(); ++itCol) + { + std::string str; + converter.ToStr(*itCol, str); + mData.at(rowIdx).at(std::distance(pRow.begin(), itCol) + (mLabelParams.mRowNameIdx + 1)) = str; + } + } + + /** + * @brief Set row by name. + * @param pRowName row label name. + * @param pRow vector of row data. + */ + template + void SetRow(const std::string& pRowName, const std::vector& pRow) + { + ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + return SetRow(rowIdx, pRow); + } + + /** + * @brief Remove row by index. + * @param pRowIdx zero-based row index. + */ + void RemoveRow(const size_t pRowIdx) + { + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + mData.erase(mData.begin() + rowIdx); + } + + /** + * @brief Remove row by name. + * @param pRowName row label name. + */ + void RemoveRow(const std::string& pRowName) + { + ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + RemoveRow(rowIdx); + } + + /** + * @brief Insert row at specified index. + * @param pRowIdx zero-based row index. + * @param pRow vector of row data (optional argument). + * @param pRowName row label name (optional argument). + */ + template + void InsertRow(const size_t pRowIdx, const std::vector& pRow = std::vector(), + const std::string& pRowName = std::string()) + { + const size_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + + std::vector row; + if (pRow.empty()) + { + row.resize(GetDataColumnCount()); + } + else + { + row.resize(pRow.size() + (mLabelParams.mRowNameIdx + 1)); + Converter converter(mConverterParams); + for (auto itCol = pRow.begin(); itCol != pRow.end(); ++itCol) + { + std::string str; + converter.ToStr(*itCol, str); + row.at(std::distance(pRow.begin(), itCol) + (mLabelParams.mRowNameIdx + 1)) = str; + } + } + + while (rowIdx > GetDataRowCount()) + { + std::vector tempRow; + tempRow.resize(GetDataColumnCount()); + mData.push_back(tempRow); + } + + mData.insert(mData.begin() + rowIdx, row); + + if (!pRowName.empty()) + { + SetRowName(pRowIdx, pRowName); + } + } + + /** + * @brief Get number of data rows (excluding label rows). + * @returns row count. + */ + size_t GetRowCount() const + { + const ssize_t count = static_cast(mData.size()) - (mLabelParams.mColumnNameIdx + 1); + return (count >= 0) ? count : 0; + } + + /** + * @brief Get cell by index. + * @param pColumnIdx zero-based column index. + * @param pRowIdx zero-based row index. + * @returns cell data. + */ + template + T GetCell(const size_t pColumnIdx, const size_t pRowIdx) const + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + + T val; + Converter converter(mConverterParams); + converter.ToVal(mData.at(rowIdx).at(columnIdx), val); + return val; + } + + /** + * @brief Get cell by index. + * @param pColumnIdx zero-based column index. + * @param pRowIdx zero-based row index. + * @param pToVal conversion function. + * @returns cell data. + */ + template + T GetCell(const size_t pColumnIdx, const size_t pRowIdx, ConvFunc pToVal) const + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + + T val; + pToVal(mData.at(rowIdx).at(columnIdx), val); + return val; + } + + /** + * @brief Get cell by name. + * @param pColumnName column label name. + * @param pRowName row label name. + * @returns cell data. + */ + template + T GetCell(const std::string& pColumnName, const std::string& pRowName) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + const ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + return GetCell(columnIdx, rowIdx); + } + + /** + * @brief Get cell by name. + * @param pColumnName column label name. + * @param pRowName row label name. + * @param pToVal conversion function. + * @returns cell data. + */ + template + T GetCell(const std::string& pColumnName, const std::string& pRowName, ConvFunc pToVal) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + const ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + return GetCell(columnIdx, rowIdx, pToVal); + } + + /** + * @brief Get cell by column name and row index. + * @param pColumnName column label name. + * @param pRowIdx zero-based row index. + * @returns cell data. + */ + template + T GetCell(const std::string& pColumnName, const size_t pRowIdx) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + return GetCell(columnIdx, pRowIdx); + } + + /** + * @brief Get cell by column name and row index. + * @param pColumnName column label name. + * @param pRowIdx zero-based row index. + * @param pToVal conversion function. + * @returns cell data. + */ + template + T GetCell(const std::string& pColumnName, const size_t pRowIdx, ConvFunc pToVal) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + return GetCell(columnIdx, pRowIdx, pToVal); + } + + /** + * @brief Get cell by column index and row name. + * @param pColumnIdx zero-based column index. + * @param pRowName row label name. + * @returns cell data. + */ + template + T GetCell(const size_t pColumnIdx, const std::string& pRowName) const + { + const ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + return GetCell(pColumnIdx, rowIdx); + } + + /** + * @brief Get cell by column index and row name. + * @param pColumnIdx zero-based column index. + * @param pRowName row label name. + * @param pToVal conversion function. + * @returns cell data. + */ + template + T GetCell(const size_t pColumnIdx, const std::string& pRowName, ConvFunc pToVal) const + { + const ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + return GetCell(pColumnIdx, rowIdx, pToVal); + } + + /** + * @brief Set cell by index. + * @param pRowIdx zero-based row index. + * @param pColumnIdx zero-based column index. + * @param pCell cell data. + */ + template + void SetCell(const size_t pColumnIdx, const size_t pRowIdx, const T& pCell) + { + const size_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + const size_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + + while ((rowIdx + 1) > GetDataRowCount()) + { + std::vector row; + row.resize(GetDataColumnCount()); + mData.push_back(row); + } + + if ((columnIdx + 1) > GetDataColumnCount()) + { + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + itRow->resize(columnIdx + 1); + } + } + + std::string str; + Converter converter(mConverterParams); + converter.ToStr(pCell, str); + mData.at(rowIdx).at(columnIdx) = str; + } + + /** + * @brief Set cell by name. + * @param pColumnName column label name. + * @param pRowName row label name. + * @param pCell cell data. + */ + template + void SetCell(const std::string& pColumnName, const std::string& pRowName, const T& pCell) + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + const ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + SetCell(columnIdx, rowIdx, pCell); + } + + /** + * @brief Get column name + * @param pColumnIdx zero-based column index. + * @returns column name. + */ + std::string GetColumnName(const ssize_t pColumnIdx) + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + if (mLabelParams.mColumnNameIdx < 0) + { + throw std::out_of_range("column name row index < 0: " + std::to_string(mLabelParams.mColumnNameIdx)); + } + + return mData.at(mLabelParams.mColumnNameIdx).at(columnIdx); + } + + /** + * @brief Set column name + * @param pColumnIdx zero-based column index. + * @param pColumnName column name. + */ + void SetColumnName(size_t pColumnIdx, const std::string& pColumnName) + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + mColumnNames[pColumnName] = columnIdx; + if (mLabelParams.mColumnNameIdx < 0) + { + throw std::out_of_range("column name row index < 0: " + std::to_string(mLabelParams.mColumnNameIdx)); + } + + // increase table size if necessary: + const int rowIdx = mLabelParams.mColumnNameIdx; + if (rowIdx >= static_cast(mData.size())) + { + mData.resize(rowIdx + 1); + } + auto& row = mData[rowIdx]; + if (columnIdx >= static_cast(row.size())) + { + row.resize(columnIdx + 1); + } + + mData.at(mLabelParams.mColumnNameIdx).at(columnIdx) = pColumnName; + } + + /** + * @brief Get column names + * @returns vector of column names. + */ + std::vector GetColumnNames() + { + if (mLabelParams.mColumnNameIdx >= 0) + { + return std::vector(mData.at(mLabelParams.mColumnNameIdx).begin() + + (mLabelParams.mRowNameIdx + 1), + mData.at(mLabelParams.mColumnNameIdx).end()); + } + + return std::vector(); + } + + /** + * @brief Get row name + * @param pRowIdx zero-based column index. + * @returns row name. + */ + std::string GetRowName(const ssize_t pRowIdx) + { + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + if (mLabelParams.mRowNameIdx < 0) + { + throw std::out_of_range("row name column index < 0: " + std::to_string(mLabelParams.mRowNameIdx)); + } + + return mData.at(rowIdx).at(mLabelParams.mRowNameIdx); + } + + /** + * @brief Set row name + * @param pRowIdx zero-based row index. + * @param pRowName row name. + */ + void SetRowName(size_t pRowIdx, const std::string& pRowName) + { + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + mRowNames[pRowName] = rowIdx; + if (mLabelParams.mRowNameIdx < 0) + { + throw std::out_of_range("row name column index < 0: " + std::to_string(mLabelParams.mRowNameIdx)); + } + + // increase table size if necessary: + if (rowIdx >= static_cast(mData.size())) + { + mData.resize(rowIdx + 1); + } + auto& row = mData[rowIdx]; + if (mLabelParams.mRowNameIdx >= static_cast(row.size())) + { + row.resize(mLabelParams.mRowNameIdx + 1); + } + + mData.at(rowIdx).at(mLabelParams.mRowNameIdx) = pRowName; + } + + /** + * @brief Get row names + * @returns vector of row names. + */ + std::vector GetRowNames() + { + std::vector rownames; + if (mLabelParams.mRowNameIdx >= 0) + { + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + if (std::distance(mData.begin(), itRow) > mLabelParams.mColumnNameIdx) + { + rownames.push_back(itRow->at(mLabelParams.mRowNameIdx)); + } + } + } + return rownames; + } + + private: + void ReadCsv() + { + std::ifstream stream; + stream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + stream.open(mPath, std::ios::binary); + ReadCsv(stream); + } + + void ReadCsv(std::istream& pStream) + { + Clear(); + pStream.seekg(0, std::ios::end); + std::streamsize length = pStream.tellg(); + pStream.seekg(0, std::ios::beg); + +#ifdef HAS_CODECVT + std::vector bom2b(2, '\0'); + if (length >= 2) + { + pStream.read(bom2b.data(), 2); + pStream.seekg(0, std::ios::beg); + } + + static const std::vector bomU16le = { '\xff', '\xfe' }; + static const std::vector bomU16be = { '\xfe', '\xff' }; + if ((bom2b == bomU16le) || (bom2b == bomU16be)) + { + mIsUtf16 = true; + mIsLE = (bom2b == bomU16le); + + std::wifstream wstream; + wstream.exceptions(std::wifstream::failbit | std::wifstream::badbit); + wstream.open(mPath, std::ios::binary); + if (mIsLE) + { + wstream.imbue(std::locale(wstream.getloc(), + new std::codecvt_utf16(std::consume_header | + std::little_endian)>)); + } + else + { + wstream.imbue(std::locale(wstream.getloc(), + new std::codecvt_utf16)); + } + std::wstringstream wss; + wss << wstream.rdbuf(); + std::string utf8 = ToString(wss.str()); + std::stringstream ss(utf8); + ParseCsv(ss, utf8.size()); + } + else +#endif + { + // check for UTF-8 Byte order mark and skip it when found + if (length >= 3) + { + std::vector bom3b(3, '\0'); + pStream.read(bom3b.data(), 3); + static const std::vector bomU8 = { '\xef', '\xbb', '\xbf' }; + if (bom3b != bomU8) + { + // file does not start with a UTF-8 Byte order mark + pStream.seekg(0, std::ios::beg); + } + else + { + // file did start with a UTF-8 Byte order mark, simply skip it + length -= 3; + } + } + + ParseCsv(pStream, length); + } + } + + void ParseCsv(std::istream& pStream, std::streamsize p_FileLength) + { + const std::streamsize bufLength = 64 * 1024; + std::vector buffer(bufLength); + std::vector row; + std::string cell; + bool quoted = false; + int cr = 0; + int lf = 0; + + while (p_FileLength > 0) + { + std::streamsize readLength = std::min(p_FileLength, bufLength); + pStream.read(buffer.data(), readLength); + for (int i = 0; i < readLength; ++i) + { + if (buffer[i] == '"') + { + if (cell.empty() || cell[0] == '"') + { + quoted = !quoted; + } + cell += buffer[i]; + } + else if (buffer[i] == mSeparatorParams.mSeparator) + { + if (!quoted) + { + row.push_back(Unquote(Trim(cell))); + cell.clear(); + } + else + { + cell += buffer[i]; + } + } + else if (buffer[i] == '\r') + { + if (mSeparatorParams.mQuotedLinebreaks && quoted) + { + cell += buffer[i]; + } + else + { + ++cr; + } + } + else if (buffer[i] == '\n') + { + if (mSeparatorParams.mQuotedLinebreaks && quoted) + { + cell += buffer[i]; + } + else + { + ++lf; + if (mLineReaderParams.mSkipEmptyLines && row.empty() && cell.empty()) + { + // skip empty line + } + else + { + row.push_back(Unquote(Trim(cell))); + + if (mLineReaderParams.mSkipCommentLines && !row.at(0).empty() && + (row.at(0)[0] == mLineReaderParams.mCommentPrefix)) + { + // skip comment line + } + else + { + mData.push_back(row); + } + + cell.clear(); + row.clear(); + quoted = false; + } + } + } + else + { + cell += buffer[i]; + } + } + p_FileLength -= readLength; + } + + // Handle last line without linebreak + if (!cell.empty() || !row.empty()) + { + row.push_back(Unquote(Trim(cell))); + cell.clear(); + mData.push_back(row); + row.clear(); + } + + // Assume CR/LF if at least half the linebreaks have CR + mSeparatorParams.mHasCR = (cr > (lf / 2)); + + // Set up column labels + if ((mLabelParams.mColumnNameIdx >= 0) && + (static_cast(mData.size()) > mLabelParams.mColumnNameIdx)) + { + int i = 0; + for (auto& columnName : mData[mLabelParams.mColumnNameIdx]) + { + mColumnNames[columnName] = i++; + } + } + + // Set up row labels + if ((mLabelParams.mRowNameIdx >= 0) && + (static_cast(mData.size()) > + (mLabelParams.mColumnNameIdx + 1))) + { + int i = 0; + for (auto& dataRow : mData) + { + if (static_cast(dataRow.size()) > mLabelParams.mRowNameIdx) + { + mRowNames[dataRow[mLabelParams.mRowNameIdx]] = i++; + } + } + } + } + + void WriteCsv() const + { +#ifdef HAS_CODECVT + if (mIsUtf16) + { + std::stringstream ss; + WriteCsv(ss); + std::string utf8 = ss.str(); + std::wstring wstr = ToWString(utf8); + + std::wofstream wstream; + wstream.exceptions(std::wofstream::failbit | std::wofstream::badbit); + wstream.open(mPath, std::ios::binary | std::ios::trunc); + + if (mIsLE) + { + wstream.imbue(std::locale(wstream.getloc(), + new std::codecvt_utf16(std::little_endian)>)); + } + else + { + wstream.imbue(std::locale(wstream.getloc(), + new std::codecvt_utf16)); + } + + wstream << static_cast(0xfeff); + wstream << wstr; + } + else +#endif + { + std::ofstream stream; + stream.exceptions(std::ofstream::failbit | std::ofstream::badbit); + stream.open(mPath, std::ios::binary | std::ios::trunc); + WriteCsv(stream); + } + } + + void WriteCsv(std::ostream& pStream) const + { + for (auto itr = mData.begin(); itr != mData.end(); ++itr) + { + for (auto itc = itr->begin(); itc != itr->end(); ++itc) + { + if (mSeparatorParams.mAutoQuote && + ((itc->find(mSeparatorParams.mSeparator) != std::string::npos) || + (itc->find(' ') != std::string::npos))) + { + // escape quotes in string + std::string str = *itc; + ReplaceString(str, "\"", "\"\""); + + pStream << "\"" << str << "\""; + } + else + { + pStream << *itc; + } + + if (std::distance(itc, itr->end()) > 1) + { + pStream << mSeparatorParams.mSeparator; + } + } + pStream << (mSeparatorParams.mHasCR ? "\r\n" : "\n"); + } + } + + size_t GetDataRowCount() const + { + return mData.size(); + } + + size_t GetDataColumnCount() const + { + return (mData.size() > 0) ? mData.at(0).size() : 0; + } + + std::string Trim(const std::string& pStr) + { + if (mSeparatorParams.mTrim) + { + std::string str = pStr; + + // ltrim + str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](int ch) { return !isspace(ch); })); + + // rtrim + str.erase(std::find_if(str.rbegin(), str.rend(), [](int ch) { return !isspace(ch); }).base(), str.end()); + + return str; + } + else + { + return pStr; + } + } + + std::string Unquote(const std::string& pStr) + { + if (mSeparatorParams.mAutoQuote && (pStr.size() >= 2) && (pStr.front() == '"') && (pStr.back() == '"')) + { + // remove start/end quotes + std::string str = pStr.substr(1, pStr.size() - 2); + + // unescape quotes in string + ReplaceString(str, "\"\"", "\""); + + return str; + } + else + { + return pStr; + } + } + +#ifdef HAS_CODECVT +#if defined(_MSC_VER) +#pragma warning (disable: 4996) +#endif + static std::string ToString(const std::wstring& pWStr) + { + return std::wstring_convert, wchar_t>{ }.to_bytes(pWStr); + } + + static std::wstring ToWString(const std::string& pStr) + { + return std::wstring_convert, wchar_t>{ }.from_bytes(pStr); + } +#if defined(_MSC_VER) +#pragma warning (default: 4996) +#endif +#endif + + static void ReplaceString(std::string& pStr, const std::string& pSearch, const std::string& pReplace) + { + size_t pos = 0; + + while ((pos = pStr.find(pSearch, pos)) != std::string::npos) + { + pStr.replace(pos, pSearch.size(), pReplace); + pos += pReplace.size(); + } + } + + private: + std::string mPath; + LabelParams mLabelParams; + SeparatorParams mSeparatorParams; + ConverterParams mConverterParams; + LineReaderParams mLineReaderParams; + std::vector> mData; + std::map mColumnNames; + std::map mRowNames; +#ifdef HAS_CODECVT + bool mIsUtf16 = false; + bool mIsLE = false; +#endif + }; +} \ No newline at end of file diff --git a/client/python/README.md b/client/python/README.md new file mode 100644 index 00000000..1e62c2b3 --- /dev/null +++ b/client/python/README.md @@ -0,0 +1,4 @@ +# VDMS Client Python Module + +This is the client module for VDMS. +For more information, visit github.com/IntelLabs/vdms diff --git a/client/python/setup.py b/client/python/setup.py new file mode 100644 index 00000000..8bc86ae7 --- /dev/null +++ b/client/python/setup.py @@ -0,0 +1,24 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="vdms", + version="0.0.17", + author="Chaunté W. Lacewell", + author_email="chaunte.w.lacewell@intel.com", + description="VDMS Client Module", + install_requires=['protobuf'], + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/IntelLabs/vdms", + license="MIT", + packages=setuptools.find_packages(), + python_requires='>=2.6, !=3.0.*, !=3.1.*, !=3.2.*, <4', + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], +) diff --git a/client/python/vdms/queryMessage_pb2.py b/client/python/vdms/queryMessage_pb2.py new file mode 100644 index 00000000..79134502 --- /dev/null +++ b/client/python/vdms/queryMessage_pb2.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: queryMessage.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='queryMessage.proto', + package='VDMS.protobufs', + syntax='proto3', + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x12queryMessage.proto\x12\x0eVDMS.protobufs\"+\n\x0cqueryMessage\x12\x0c\n\x04json\x18\x01 \x01(\t\x12\r\n\x05\x62lobs\x18\x02 \x03(\x0c\x62\x06proto3' +) + + + + +_QUERYMESSAGE = _descriptor.Descriptor( + name='queryMessage', + full_name='VDMS.protobufs.queryMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='json', full_name='VDMS.protobufs.queryMessage.json', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='blobs', full_name='VDMS.protobufs.queryMessage.blobs', index=1, + number=2, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=38, + serialized_end=81, +) + +DESCRIPTOR.message_types_by_name['queryMessage'] = _QUERYMESSAGE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +queryMessage = _reflection.GeneratedProtocolMessageType('queryMessage', (_message.Message,), { + 'DESCRIPTOR' : _QUERYMESSAGE, + '__module__' : 'queryMessage_pb2' + # @@protoc_insertion_point(class_scope:VDMS.protobufs.queryMessage) + }) +_sym_db.RegisterMessage(queryMessage) + + +# @@protoc_insertion_point(module_scope) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index ab63b138..7dd6111b 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -4,7 +4,6 @@ ARG UBUNTU_VERSION=20.04 ARG UBUNTU_NAME=focal ARG BUILD_THREADS=-j16 -ARG MAVEN_OPTS='-Dhttps.nonProxyHosts="localhost|127.0.0.1"' #1 FROM ubuntu:${UBUNTU_VERSION} @@ -12,78 +11,64 @@ FROM ubuntu:${UBUNTU_VERSION} # Dockerfile limitations force a repetition of global args ARG UBUNTU_VERSION ARG UBUNTU_NAME -ARG MAVEN_OPTS # Install Packages -RUN apt-get update && apt-get -y install software-properties-common && \ +RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \ add-apt-repository "deb http://security.ubuntu.com/ubuntu ${UBUNTU_NAME}-security main" && \ - apt-get -y install apt-transport-https autoconf automake bison build-essential \ - bzip2 ca-certificates curl ed flex g++ git gnupg-agent javacc libarchive-tools \ + apt-get install -y --no-install-recommends apt-transport-https autoconf automake bison build-essential \ + bzip2 ca-certificates curl=7.68.0-1ubuntu2.18 ed flex g++ git gnupg-agent javacc libarchive-tools \ libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \ libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \ libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \ libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \ libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \ - libtbb2 libtiff-dev libtiff5-dev libtool maven mpich openjdk-11-jdk-headless \ - pkg-config python python-dev python3-pip unzip wget + libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \ + pkg-config python3-dev python3-pip unzip && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ + pip3 install --no-cache-dir "numpy>=1.23.2" "setuptools>=65.5.1" -RUN pip3 install numpy - -# Pull Dependencies -RUN git clone --branch v1.40.0 https://github.com/grpc/grpc.git && \ +# Pull and Install Dependencies +WORKDIR /dependencies +RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ git clone --branch v4.0.2 https://github.com/swig/swig.git && \ - git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ - git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ - git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \ git clone https://github.com/tonyzhang617/FLINNG.git && \ - curl http://zlib.net/zlib-1.2.12.tar.gz -o zlib-1.2.12.tar.gz && \ - curl https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar -o /usr/share/java/json-simple-1.1.1.jar && \ - wget https://github.com/TileDB-Inc/TileDB/archive/1.3.1.tar.gz - -# Install Dependencies -RUN cd /CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ - cd /swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ - cd /faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ - cd /FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ - cd /grpc && git submodule update --init --recursive && cd third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd ../../zlib/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd /grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ - -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ - cd / && gunzip zlib-1.2.12.tar.gz && tar -xvf zlib-1.2.12.tar && cd zlib-1.2.12 && ./configure && make ${BUILD_THREADS} && make install && \ - cd / && rm -rf zlib-1.2.12.tar zlib-1.2.12 - -# Google Test & OpenCV -RUN cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ - cd /opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install - -# TileDB -RUN cd / && tar xf 1.3.1.tar.gz && rm 1.3.1.tar.gz && \ - cd TileDB-1.3.1 && mkdir build && cd build && \ - ../bootstrap --prefix=/usr/local/ && make $BUILD_THREADS && make install-tiledb && \ - rm -rf /TileDB-1.3.1 - -# Maven -RUN ln -s /grpc/third_party/protobuf/cmake/build/protoc /grpc/third_party/protobuf/src/protoc && \ - cd /grpc/third_party/protobuf/java/core && mvn package && \ - cp $(ls target/protobuf-java*.jar) /usr/share/java/protobuf.jar + git clone --recurse-submodules -b v1.40.0 https://github.com/grpc/grpc.git && \ + git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ + git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ + curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ + curl -L -o /dependencies/1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ + curl -L -o /dependencies/zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ + cd /dependencies/CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ + cd /dependencies/swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ + cd /dependencies/faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/grpc && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ + cd tools/distrib/python/grpcio_tools && python ../make_grpcio_tools.py && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ + cd /dependencies/grpc/third_party/zlib && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -Dprotobuf_BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ + cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ + cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/ && tar -xvzf zlib-1.2.13.tar.gz && cd zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ + cd /dependencies/ && tar -xvf 1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \ + ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \ + cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ + cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \ + rm -rf /dependencies -# Valijson -RUN cd /valijson && cp -r include/* /usr/local/include/ && \ - cd / && rm -rf valijson && rm -rf faiss && \ - rm -rf grpc && rm -rf opencv && rm -rf swig && rm -rf CMake # VDMS RUN git clone https://github.com/IntelLabs/vdms.git && cd vdms && \ git checkout develop && git submodule update --init --recursive && \ mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && \ - cp /vdms/config-vdms.json /vdms/build/ - -RUN echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \ + cp /vdms/config-vdms.json /vdms/build/ && \ + echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \ echo './vdms' >> /start.sh && chmod 755 /start.sh CMD ["/start.sh"] diff --git a/ext/custom_vcl/custom_vcl_process.cc b/ext/custom_vcl/custom_vcl_process.cc index 42baface..55d04b04 100644 --- a/ext/custom_vcl/custom_vcl_process.cc +++ b/ext/custom_vcl/custom_vcl_process.cc @@ -9,6 +9,12 @@ int main(int argc, char* argv[]) key_ctl_host_remote = ftok("../../vdms", 60); int msgid_ctl_host_remote = msgget(key_ctl_host_remote, 0666 | IPC_CREAT); data_message message_ctl_host_remote; + //need size of data message excluding message_type field for msgsnd and msgrcv + size_t data_message_size = sizeof(message_ctl_host_remote.data_rows) + + sizeof(message_ctl_host_remote.data_cols) + + sizeof(message_ctl_host_remote.data_type) + + sizeof(message_ctl_host_remote.data_image_size) + + sizeof(message_ctl_host_remote.data_json_size); key_t key_data_host_remote; key_data_host_remote = ftok("../../vdms", 61); @@ -22,17 +28,18 @@ int main(int argc, char* argv[]) heartbeat_message message_hb_host_remote; heartbeat_message message_hb_remote_host; + size_t heartbeat_message_size = sizeof(message_hb_host_remote.status); while(true) { //Handle handshake to indicate remote process is alive - int in_alive_msg_status = msgrcv(msgid_ctl_host_remote, &message_hb_host_remote,sizeof(heartbeat_message) , (long) vcl_message_type::VCL_MESSAGE_HEARTBEAT, 0); + int in_alive_msg_status = msgrcv(msgid_ctl_host_remote, &message_hb_host_remote, heartbeat_message_size, (long) vcl_message_type::VCL_MESSAGE_HEARTBEAT, 0); message_hb_remote_host.message_type = (long) vcl_message_type::VCL_MESSAGE_HEARTBEAT; message_hb_remote_host.status = 0; - int msg_send_result = msgsnd(msgid_ctl_remote_host, &message_hb_remote_host, sizeof(heartbeat_message), 0); + int msg_send_result = msgsnd(msgid_ctl_remote_host, &message_hb_remote_host, heartbeat_message_size, 0); - int msg_status = msgrcv(msgid_ctl_host_remote, &message_ctl_host_remote,sizeof(data_message) , (long) vcl_message_type::VCL_MESSAGE_DATA, 0); + int msg_status = msgrcv(msgid_ctl_host_remote, &message_ctl_host_remote, data_message_size, (long) vcl_message_type::VCL_MESSAGE_DATA, 0); if(msg_status > 0) { //Read image from shared memory @@ -78,7 +85,7 @@ int main(int argc, char* argv[]) } } - int msg_send_result = msgsnd(msgid_ctl_remote_host, &message_ctl_remote_host, sizeof(data_message), 0); + int msg_send_result = msgsnd(msgid_ctl_remote_host, &message_ctl_remote_host, data_message_size, 0); if(msg_send_result < 0) { } diff --git a/src/BlobCommand.cc b/src/BlobCommand.cc new file mode 100644 index 00000000..76b64b8d --- /dev/null +++ b/src/BlobCommand.cc @@ -0,0 +1,243 @@ +/** + * @file BlobCommand.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2017 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include + +#include "BlobCommand.h" +#include "VDMSConfig.h" +#include "defines.h" + +using namespace VDMS; + +//========= AddBlob definitions ========= + +BlobCommand::BlobCommand(const std::string &cmd_name): + RSCommand(cmd_name) +{ +} + +AddBlob::AddBlob() : BlobCommand("AddBlob") +{ + + _storage_bin = VDMSConfig::instance()->get_path_bin(); +} + +int AddBlob::construct_protobuf(PMGDQuery& query, + const Json::Value& jsoncmd, + const std::string& blob, + int grp_id, + Json::Value& error) +{ + const Json::Value& cmd = jsoncmd[_cmd_name]; + + std::cout << " inside ADDBLOB" <(cmd, "_ref", + query.get_available_reference()); + + + std::string format = "bin"; + char binary_img_flag = 1; + VCL::Image img((void*)blob.data(), blob.size(), binary_img_flag); + + + std::string blob_root = _storage_bin; + VCL::Image::Format blob_format = VCL::Image::Format::BIN; + std::string file_name = VCL::create_unique(blob_root, format); + std::cout << "Blob was added in " <<_storage_bin << "\t"<< file_name << std::endl; + Json::Value props = get_value(cmd, "properties"); + props[VDMS_EN_BLOB_PATH_PROP] = file_name; + + + query.AddNode(node_ref, VDMS_BLOB_TAG, props, Json::Value()); + + img.store(file_name, blob_format); + + + error["Blob_added"] = file_name; + + if (cmd.isMember("link")) { + add_link(query, cmd["link"], node_ref, VDMS_BLOB_EDGE_TAG); + } + + return 0; +} + +//========= UpdateBLOB definitions ========= + +UpdateBlob::UpdateBlob() : BlobCommand("UpdateBlob") +{ +} + +int UpdateBlob::construct_protobuf(PMGDQuery& query, + const Json::Value& jsoncmd, + const std::string& blob, + int grp_id, + Json::Value& error) +{ + const Json::Value& cmd = jsoncmd[_cmd_name]; + + // Update Image node + query.UpdateNode(get_value(cmd, "_ref", -1), + VDMS_BLOB_TAG, + cmd["properties"], + cmd["remove_props"], + cmd["constraints"], + get_value(cmd, "unique", false)); + + return 0; +} + +//========= FindBLOB definitions ========= + +FindBlob::FindBlob() : BlobCommand("FindBlob") +{ +} + +int FindBlob::construct_protobuf( + PMGDQuery& query, + const Json::Value& jsoncmd, + const std::string& blob, + int grp_id, + Json::Value& error) +{ + const Json::Value& cmd = jsoncmd[_cmd_name]; + + Json::Value results = get_value(cmd, "results"); + + // Unless otherwhis specified, we return the blob. + if (get_value(results, "blob", true)){ + results["list"].append(VDMS_EN_BLOB_PATH_PROP); + } + + query.QueryNode( + get_value(cmd, "_ref", -1), + VDMS_BLOB_TAG, + cmd["link"], + cmd["constraints"], + results, + get_value(cmd, "unique", false) + ); + + return 0; +} + +Json::Value FindBlob::construct_responses( + Json::Value& responses, + const Json::Value& json, + protobufs::queryMessage &query_res, + const std::string &blob) +{ + const Json::Value& cmd = json[_cmd_name]; + + Json::Value ret; + + auto error = [&](Json::Value& res) + { + ret[_cmd_name] = res; + return ret; + }; + + if (responses.size() != 1) { + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "PMGD Response Bad Size"; + return error(return_error); + } + + Json::Value& findBlob = responses[0]; + + if (findBlob["status"] != 0) { + findBlob["status"] = RSCommand::Error; + // Uses PMGD info error. + return error(findBlob); + } + + Json::Value results = get_value(cmd, "results"); + + bool flag_empty = false; + + // Check if blob (image) must be returned + if (get_value(results, "blob", true)) { + + for (auto& ent : findBlob["entities"]) { + + assert(ent.isMember(VDMS_EN_BLOB_PATH_PROP)); + + std::string blob_path = ent[VDMS_EN_BLOB_PATH_PROP].asString(); + ent.removeMember(VDMS_EN_BLOB_PATH_PROP); + + if (ent.getMemberNames().size() == 0) { + flag_empty = true; + } + + try { + VCL::Image blob_im(blob_path); + + + // We will return the image in the format the user + // request, or on its format in disk, except for the case + // of .tdb, where we will encode as png. + VCL::Image::Format format =VCL::Image::Format::BIN; + + std::vector blob_buffer; + blob_buffer = blob_im.get_encoded_image(format); + + if (!blob_buffer.empty()) { + + std::string* blob_str = query_res.add_blobs(); + blob_str->resize(blob_buffer.size()); + std::memcpy((void*)blob_str->data(), + (void*)blob_buffer.data(), + blob_buffer.size()); + } + else { + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "Blob Data not found"; + return error(return_error); + } + } catch (VCL::Exception e) { + print_exception(e); + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "VCL Exception"; + return error(return_error); + } + } + } + + if (flag_empty) { + findBlob.removeMember("entities"); + } + + ret[_cmd_name].swap(findBlob); + return ret; +} diff --git a/src/BlobCommand.h b/src/BlobCommand.h new file mode 100644 index 00000000..c211a162 --- /dev/null +++ b/src/BlobCommand.h @@ -0,0 +1,112 @@ +/** + * @file BlobCommand.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2017 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#pragma once +#include +#include +#include +#include "vcl/Image.h" +#include "vcl/CustomVCL.h" + +#include "RSCommand.h" +#include "ExceptionsCommand.h" + +namespace VDMS { + +// Helper classes for handling various JSON commands. + + class BlobCommand: public RSCommand + { + public: + + BlobCommand(const std::string &cmd_name); + + virtual int construct_protobuf(PMGDQuery& tx, + const Json::Value& root, + const std::string& blob, + int grp_id, + Json::Value& error) = 0; + + virtual bool need_blob(const Json::Value& cmd) { return false; } + + + + }; + + class AddBlob: public BlobCommand + { + + std::string _storage_bin; + + public: + AddBlob(); + + int construct_protobuf(PMGDQuery& tx, + const Json::Value& root, + const std::string& blob, + int grp_id, + Json::Value& error); + + bool need_blob(const Json::Value& cmd) { return true; } + }; + + class UpdateBlob: public BlobCommand + { + public: + UpdateBlob(); + + int construct_protobuf(PMGDQuery& tx, + const Json::Value& root, + const std::string& blob, + int grp_id, + Json::Value& error); + + + }; + + class FindBlob: public BlobCommand + { + public: + FindBlob(); + int construct_protobuf(PMGDQuery& tx, + const Json::Value& root, + const std::string& blob, + int grp_id, + Json::Value& error); + + Json::Value construct_responses( + Json::Value &json_responses, + const Json::Value &json, + protobufs::queryMessage &response, + const std::string &blob); + }; + +}; // namespace VDMS diff --git a/src/DescriptorsCommand.cc b/src/DescriptorsCommand.cc index ceedae7f..86e19a12 100644 --- a/src/DescriptorsCommand.cc +++ b/src/DescriptorsCommand.cc @@ -212,8 +212,8 @@ Json::Value AddDescriptorSet::construct_responses( // We can probably set up a mechanism // to fix a broken link when detected later, same with images. try { - VCL::DescriptorParams* param = new VCL::DescriptorParams(_flinng_num_rows, _flinng_cells_per_row, _flinng_num_hash_tables,_flinng_hashes_per_table); - VCL::DescriptorSet desc_set(desc_set_path, dimensions, eng, metric, param); + VCL::DescriptorParams param(_flinng_num_rows, _flinng_cells_per_row, _flinng_num_hash_tables,_flinng_hashes_per_table); + VCL::DescriptorSet desc_set(desc_set_path, dimensions, eng, metric, ¶m); desc_set.store(); } catch (VCL::Exception e) { diff --git a/src/PMGDIterators.cc b/src/PMGDIterators.cc index 2cd40c7c..6d88eab4 100644 --- a/src/PMGDIterators.cc +++ b/src/PMGDIterators.cc @@ -97,10 +97,10 @@ bool PMGDQueryHandler::NodeEdgeIteratorImpl::next() bool PMGDQueryHandler::NodeEdgeIteratorImpl::_next() { while (_src_ni != NULL && bool(*_src_ni)) { - delete _edge_it; + // delete _edge_it; _src_ni->next(); if (bool(*_src_ni)) { - _edge_it = new PMGD::EdgeIterator((*_src_ni)->get_edges(_dir, _expr.tag())); + _edge_it.reset( new PMGD::EdgeIterator((*_src_ni)->get_edges(_dir, _expr.tag()))); while (_edge_it != NULL && bool(*_edge_it)) { if (check_predicates()) return true; diff --git a/src/PMGDIterators.h b/src/PMGDIterators.h index 3dcfa80f..7bf1fd92 100644 --- a/src/PMGDIterators.h +++ b/src/PMGDIterators.h @@ -206,7 +206,8 @@ namespace VDMS { PMGD::Direction _dir; bool _check_dest; - PMGD::EdgeIterator *_edge_it; + // PMGD::EdgeIterator *_edge_it; + std::unique_ptr _edge_it; bool _next(); bool check_predicates(); @@ -232,6 +233,8 @@ namespace VDMS { PMGD::PropertyPredicate pp; if (_num_predicates > 0) pp = _expr.get_node_predicate(0); + else + pp = PMGD::PropertyPredicate(); return _expr.db().get_edges(_expr.tag(), pp); } else { @@ -245,9 +248,10 @@ namespace VDMS { ReusableNodeIterator *dest_ni = NULL) : _expr(expr), _num_predicates(_expr.num_node_predicates()), _src_ni(src_ni), _dest_ni(dest_ni), - _pred_start(0), _check_dest(false), - _edge_it(new PMGD::EdgeIterator(return_iterator())) + _pred_start(0), _check_dest(false) + { + _edge_it.reset(new PMGD::EdgeIterator(return_iterator())); // If the first criteria did not return any edges, // there is no node checking on either side. if (!bool(*_edge_it)) diff --git a/src/QueryHandler.cc b/src/QueryHandler.cc index f8a249b5..229a2ab9 100644 --- a/src/QueryHandler.cc +++ b/src/QueryHandler.cc @@ -38,6 +38,7 @@ #include "DescriptorsCommand.h" #include "BoundingBoxCommand.h" #include "VideoCommand.h" +#include "BlobCommand.h" #include "ExceptionsCommand.h" @@ -88,6 +89,10 @@ void QueryHandler::init() _rs_cmds["FindVideo"] = new FindVideo(); _rs_cmds["FindFrames"] = new FindFrames(); + _rs_cmds["AddBlob"] = new AddBlob(); + _rs_cmds["UpdateBlob"] = new UpdateBlob(); + _rs_cmds["FindBlob"] = new FindBlob(); + // Load the string containing the schema (api_schema/APISchema.h) Json::Reader reader; Json::Value api_schema; diff --git a/src/Server.cc b/src/Server.cc index 3c5185de..0bd08f42 100644 --- a/src/Server.cc +++ b/src/Server.cc @@ -60,16 +60,16 @@ Server::Server(std::string config_file) _autodelete_interval = VDMSConfig::instance() ->get_int_value("autodelete_interval_s", DEFAULT_AUTODELETE_INTERVAL); _backup_flag = VDMSConfig::instance() - ->get_string_value("backup_flag", DEFAULT_AUTOREPLICATE_FLAG) ; + ->get_string_value("backup_flag", DEFAULT_AUTOREPLICATE_FLAG) ; _autoreplecate_interval = VDMSConfig::instance() ->get_int_value("autoreplicate_interval", DEFAULT_AUTOREPLICATE_INTERVAL); _replication_unit = VDMSConfig::instance() - ->get_string_value("unit", DEFAULT_AUTOREPLICATE_UNIT); + ->get_string_value("unit", DEFAULT_AUTOREPLICATE_UNIT); _backup_path = VDMSConfig::instance() - ->get_string_value("backup_path", DEFAULT_BACKUP_PATH); + ->get_string_value("backup_path", DEFAULT_BACKUP_PATH); _db_path = VDMSConfig::instance() - ->get_string_value("db_root_path", DEFAULT_DB_ROOT); + ->get_string_value("db_root_path", DEFAULT_DB_ROOT); PMGDQueryHandler::init(); QueryHandler::init(); @@ -108,7 +108,7 @@ void Server::process_requests() new comm::Connection(server->accept()); _cm->add_connection(conn_server); - + } catch (comm::ExceptionComm e) { print_exception(e); @@ -118,43 +118,43 @@ void Server::process_requests() delete server; } void Server::untar_data(std::string& name){ - - + + std::string command="tar -xvSf" + name; system(command.c_str()); - + } void Server::auto_replicate_data(){ - - long replication_period; + + long replication_period = 0; QueryHandler qh; - if(_backup_flag =="true"){ + if(_backup_flag =="true"){ if (_autoreplecate_interval >0 ){ if (_replication_unit.compare("h") == 0){ replication_period =_autoreplecate_interval*60*60; } else if (_replication_unit.compare("m") == 0) replication_period =_autoreplecate_interval*60; - - else - replication_period= _autoreplecate_interval; - } - + + else + replication_period= _autoreplecate_interval; + } + if(_backup_path.empty()){ _backup_path=_db_path; //set the defualt path to be db - } - - - if(replication_period > 0) //check to ensure valid autodelete_interval - { - - while(!shutdown) + } + + + if(replication_period > 0) //check to ensure valid autodelete_interval { - sleep(replication_period); - qh.regualar_run_autoreplicate(_backup_path, _db_path, _server_port); + + while(!shutdown) + { + sleep(replication_period); + qh.regualar_run_autoreplicate(_backup_path, _db_path, _server_port); + } } - } - } + } } void Server::autodelete_expired_data() diff --git a/src/defines.h b/src/defines.h index ecc7df08..0a76b97a 100644 --- a/src/defines.h +++ b/src/defines.h @@ -45,6 +45,8 @@ // Entities #define VDMS_EN_BLOB_PATH_PROP "VD:blobPath" +#define VDMS_BLOB_TAG "VD:BLOB" +#define VDMS_BLOB_EDGE_TAG "VD:BLOBLINK" // Images diff --git a/src/vcl/CustomVCL.cc b/src/vcl/CustomVCL.cc index 2f46911a..6ba37533 100644 --- a/src/vcl/CustomVCL.cc +++ b/src/vcl/CustomVCL.cc @@ -6,8 +6,15 @@ int custom_vcl_function(VCL::Image& img, const Json::Value& ops) //create IPC structures for communicating between processes key_t key_ctl_host_remote; key_ctl_host_remote = ftok("vdms", 60); + int msgid_ctl_host_remote = msgget(key_ctl_host_remote, 0666 | IPC_CREAT); data_message message_ctl_host_remote; + //need size of data message excluding message_type field for msgsnd and msgrcv + size_t data_message_size = sizeof(message_ctl_host_remote.data_rows) + + sizeof(message_ctl_host_remote.data_cols) + + sizeof(message_ctl_host_remote.data_type) + + sizeof(message_ctl_host_remote.data_image_size) + + sizeof(message_ctl_host_remote.data_json_size); key_t key_data_host_remote; key_data_host_remote = ftok("vdms", 61); @@ -21,11 +28,12 @@ int custom_vcl_function(VCL::Image& img, const Json::Value& ops) heartbeat_message message_hb_host_remote; heartbeat_message message_hb_remote_host; + size_t heartbeat_message_size = sizeof(message_hb_host_remote.status); //Pass messages to ensure the remote process is functional message_hb_host_remote.message_type = (long) vcl_message_type::VCL_MESSAGE_HEARTBEAT; message_hb_host_remote.status = 0; - int out_alive_msg_status = msgsnd(msgid_ctl_host_remote, &message_hb_host_remote, sizeof(heartbeat_message), 0); + int out_alive_msg_status = msgsnd(msgid_ctl_host_remote, &message_hb_host_remote, heartbeat_message_size, 0); int hb_count = 0; int in_alive_msg_status = -1; @@ -33,7 +41,7 @@ int custom_vcl_function(VCL::Image& img, const Json::Value& ops) //try 10 times to determine if process is running while(hb_count < 10 && in_alive_msg_status < 0) { - in_alive_msg_status = msgrcv(msgid_ctl_remote_host, &message_hb_remote_host,sizeof(heartbeat_message) , (long) vcl_message_type::VCL_MESSAGE_HEARTBEAT, IPC_NOWAIT); + in_alive_msg_status = msgrcv(msgid_ctl_remote_host, &message_hb_remote_host, heartbeat_message_size, (long) vcl_message_type::VCL_MESSAGE_HEARTBEAT, IPC_NOWAIT); hb_count++; } @@ -54,14 +62,17 @@ int custom_vcl_function(VCL::Image& img, const Json::Value& ops) std::string* json_string = new std::string(ops.toStyledString()); message_ctl_host_remote.data_json_size = json_string->size(); - //image size corresponds with first byte after the image memcpy(&(image_buffer[in_image_size]), json_string->c_str(), json_string->size()); - int msg_send_result = msgsnd(msgid_ctl_host_remote, &message_ctl_host_remote, sizeof(data_message), 0); + int msg_send_result = msgsnd(msgid_ctl_host_remote, &message_ctl_host_remote, data_message_size, 0); if(msg_send_result < 0) - {} + { + delete json_string; + return -1; + } + + int msg_recv_result = msgrcv(msgid_ctl_remote_host, &message_ctl_remote_host, data_message_size, (long)vcl_message_type::VCL_MESSAGE_DATA, 0); - int msg_recv_result = msgrcv(msgid_ctl_remote_host, &message_ctl_remote_host, sizeof(data_message), (long)vcl_message_type::VCL_MESSAGE_DATA, 0); if(msg_recv_result < 0) {} @@ -87,5 +98,4 @@ int custom_vcl_function(VCL::Image& img, const Json::Value& ops) } return return_value; - -} \ No newline at end of file +} diff --git a/src/vcl/DescriptorParams.h b/src/vcl/DescriptorParams.h index 698a2fed..ac5498fd 100644 --- a/src/vcl/DescriptorParams.h +++ b/src/vcl/DescriptorParams.h @@ -70,7 +70,5 @@ namespace VCL { this->sub_hash_bits = subhashbits; this->cut_off= cutoff; } - - ~DescriptorParams(); }; }; diff --git a/src/vcl/DescriptorSetData.h b/src/vcl/DescriptorSetData.h index cd66008b..a43f4635 100644 --- a/src/vcl/DescriptorSetData.h +++ b/src/vcl/DescriptorSetData.h @@ -74,7 +74,10 @@ namespace VCL { inline bool dir_exist(const std::string& dir_name) { DIR* dir = opendir(dir_name.c_str()); if (dir) + { + closedir(dir); return true; + } return false; } @@ -117,7 +120,7 @@ namespace VCL { */ DescriptorSetData(const std::string &filename, unsigned dim); - ~DescriptorSetData(); + virtual ~DescriptorSetData(); DescriptorSetData(const DescriptorSetData&) = delete; @@ -147,7 +150,7 @@ namespace VCL { */ virtual long add(float* descriptors, unsigned n_descriptors, long* labels = NULL) = 0; - + virtual long add_and_store(float* descriptors, unsigned n_descriptors, long* labels = NULL) {return 0;} @@ -163,7 +166,7 @@ namespace VCL { */ virtual void search(float* query, unsigned n, unsigned k, long* descriptors, float* distances) = 0; - + virtual void search(float* query, unsigned n, unsigned k, long* descriptors) {} diff --git a/src/vcl/Image.cc b/src/vcl/Image.cc index bbf03b68..dcb9dad2 100644 --- a/src/vcl/Image.cc +++ b/src/vcl/Image.cc @@ -494,6 +494,8 @@ Image::~Image() _operations.clear(); _operations.shrink_to_fit(); delete _tdb; + if(_bin) + free(_bin); } /* *********************** */ diff --git a/src/vcl/TDBDescriptorSet.cc b/src/vcl/TDBDescriptorSet.cc index 1237ce3f..3dfdc8db 100644 --- a/src/vcl/TDBDescriptorSet.cc +++ b/src/vcl/TDBDescriptorSet.cc @@ -145,6 +145,8 @@ void TDBDescriptorSet::classify(float* descriptors, unsigned n, } labels[j] = winner; } + delete[] distances; + delete[] ids_aux; } void TDBDescriptorSet::get_labels(long* ids, unsigned n, long* labels) diff --git a/src/vcl/TDBDescriptorSet.h b/src/vcl/TDBDescriptorSet.h index 8eb10331..edb16ae1 100644 --- a/src/vcl/TDBDescriptorSet.h +++ b/src/vcl/TDBDescriptorSet.h @@ -121,7 +121,7 @@ namespace VCL { TDBDenseDescriptorSet(const std::string &collection_path, unsigned dim, DistanceMetric metric); - ~TDBDenseDescriptorSet(); + ~TDBDenseDescriptorSet() {}; long add(float* descriptors, unsigned n_descriptors, long* classes); @@ -152,7 +152,7 @@ namespace VCL { TDBSparseDescriptorSet(const std::string &collection_path, unsigned dim, DistanceMetric metric); - ~TDBSparseDescriptorSet(); + ~TDBSparseDescriptorSet() {}; long add(float* descriptors, unsigned n_descriptors, long* classes); diff --git a/src/vdms.cc b/src/vdms.cc index f85b11f2..f50027d0 100644 --- a/src/vdms.cc +++ b/src/vdms.cc @@ -40,7 +40,7 @@ void printUsage() { std::cout << "Usage: vdms -cfg config-file.json" << std::endl; - + std::cout << "Usage: vdms -restore db.tar.gz" << std::endl; exit(0); } @@ -54,6 +54,7 @@ static void* start_request_thread(void* server) } static void* start_replication_thread(void* server){ ((VDMS::Server*)(server))->auto_replicate_data(); + return NULL; } @@ -79,36 +80,36 @@ int main(int argc, char **argv) if (argc == 3){ std::string option(argv[1]); - + if (option != "-cfg" && option!="-restore" && option!="-backup") printUsage(); if(option =="-cfg") config_file = std::string (argv[2]); - - - + + + else if (option=="-restore" ){ void* server; - + std::string db_name(argv[2]); size_t file_ext1 = db_name.find_last_of("."); - + std::string temp_name_1= db_name.substr(0,file_ext1); - + size_t file_ext2 = temp_name_1.find_last_of("."); std::string temp_name_2= temp_name_1.substr(0,file_ext2); - + ((VDMS::Server*)(server))->untar_data(db_name); - + config_file = temp_name_2+".json"; - + } } - + printf("Server will start processing requests... \n"); VDMS::Server server(config_file); @@ -116,13 +117,13 @@ int main(int argc, char **argv) request_thread_flag = pthread_create(&request_thread, NULL, start_request_thread, (void*)( &server ) ); autodelete_thread_flag = pthread_create(&autodelete_thread, NULL, start_autodelete_thread, (void*)( &server ) ); auto_replcation_flag = pthread_create(&auto_replicate_thread, NULL, start_replication_thread, (void*)( &server ) ); - + pthread_join(request_thread, NULL); pthread_join(autodelete_thread, NULL); pthread_join(auto_replicate_thread, NULL); - + printf("Server shutting down... \n"); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index dd323a2d..06c72f71 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -40,12 +40,14 @@ add_executable(unit_tests unit_tests/DescriptorSetReadFS_test.cc unit_tests/DescriptorSetStore_test.cc unit_tests/client_add_entity.cc + unit_tests/client_csv.cc unit_tests/meta_data.cc unit_tests/client_find_entities.cc unit_tests/client_image.cc unit_tests/client_bounding_box.cc unit_tests/client_descriptors.cc unit_tests/client_videos.cc + unit_tests/client_blob.cc ) target_link_libraries(unit_tests diff --git a/tests/cleandbs.sh b/tests/cleandbs.sh index 88e4676c..8e2bf9f8 100755 --- a/tests/cleandbs.sh +++ b/tests/cleandbs.sh @@ -1,12 +1,13 @@ -rm -r jsongraph qhgraph simpleAdd_db simpleAddx10_db simpleUpdate_db entitycheck_db datatypecheck_db db_backup test_db_1 +rm -r jsongraph qhgraph simpleAdd_db simpleAddx10_db simpleUpdate_db +rm -r entitycheck_db datatypecheck_db db_backup test_db_1 rm -r tests_log.log tests_screen.log rm -r tdb -rm -r dbs +rm -r db dbs test_db_client rm -r temp rm -r videos_tests rm -r vdms -rm images/tdb_to_jpg.jpg -rm images/tdb_to_png.png -rm images/test_image.jpg +rm test_images/tdb_to_jpg.jpg +rm test_images/tdb_to_png.png +rm test_images/test_image.jpg rm -r backups diff --git a/tests/csv_samples/CSVformat100.csv b/tests/csv_samples/CSVformat100.csv new file mode 100644 index 00000000..3df655d9 --- /dev/null +++ b/tests/csv_samples/CSVformat100.csv @@ -0,0 +1,96 @@ +EntityClass,prop_name,prop_middlename,prop_lastname,prop_id,prop_date:dob,prop_height,prop_weight,prop_age,prop_has_dog,prop_Gender,prop_email,prop_Address,prop_City,cons_1 +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,False,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,date:dob==1968-07-22T12:45:12-08:00 +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,id==1234 +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,weight==70.5 +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur diff --git a/tests/csv_samples/Descriptor.csv b/tests/csv_samples/Descriptor.csv new file mode 100644 index 00000000..2ef43646 --- /dev/null +++ b/tests/csv_samples/Descriptor.csv @@ -0,0 +1,6 @@ +DescriptorClass,label,prop_age,prop_gender,inputdata +Test_14096,Rocky,34,M,../tests/csv_samples/blob_1.txt +Test_14096,Rocky,34,M,../tests/csv_samples/blob_1.txt +Test_14096,Rocky,34,M,../tests/csv_samples/blob_1.txt +Test_14096,Rocky,34,M,../tests/csv_samples/blob_1.txt +Test_14096,Rocky,34,M,../tests/csv_samples/blob_1.txt diff --git a/tests/csv_samples/DescriptorSet.csv b/tests/csv_samples/DescriptorSet.csv new file mode 100644 index 00000000..cf9a3f5b --- /dev/null +++ b/tests/csv_samples/DescriptorSet.csv @@ -0,0 +1,7 @@ +DescriptorType,dimensions,distancemetric,searchengine +Test1024,1024,L2,FaissFlat +Test_14096,1024,L2,FaissFlat +Test1000,1000,L2,FaissFlat +Test100,100,L2,FaissFlat +Test128,128,IP,FaissIVFFlat +Test512,512,L2,TileDBDense diff --git a/tests/csv_samples/Image.csv b/tests/csv_samples/Image.csv new file mode 100644 index 00000000..37723900 --- /dev/null +++ b/tests/csv_samples/Image.csv @@ -0,0 +1,11 @@ +ImagePath,ops_threshold,ops_crop,ops_resize,ops_flip,ops_rotate,prop_type,prop_part,format,cons_1 +../tests/test_images/large1.jpg,350,,,,,,image1,jpg,part==image1 +../tests/test_images/large1.jpg,350,"0,0,224,224",,,,,image2,jpg, +../tests/test_images/large1.jpg,350,,"224,224",,,,image3,jpg, +../tests/test_images/large1.jpg,350,,,1,,,image4,jpg, +../tests/test_images/large1.jpg,350,,,,"45,false",,image5,jpg, +../tests/test_images/large1.jpg,350,,,,"75.5,false",,image6,jpg, +../tests/test_images/large1.jpg,350,,,,,,image7,png, +../tests/test_images/large1.jpg,,,,,,,image8,bin, +../tests/test_images/large1.jpg,350,,,,,,image9,png, +../tests/test_images/large1.jpg,,,,,,,image10,bin, diff --git a/tests/csv_samples/Rectangle.csv b/tests/csv_samples/Rectangle.csv new file mode 100644 index 00000000..cc0fec24 --- /dev/null +++ b/tests/csv_samples/Rectangle.csv @@ -0,0 +1,13 @@ +RectangleBound,prop_name +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 \ No newline at end of file diff --git a/tests/csv_samples/Video.csv b/tests/csv_samples/Video.csv new file mode 100644 index 00000000..a9a8f4f4 --- /dev/null +++ b/tests/csv_samples/Video.csv @@ -0,0 +1,6 @@ +VideoPath,format,compressto,prop_name,ops_resize,ops_interval +../tests/test_videos/Megamind.avi,avi,h264,Good,"200,175", +../tests/test_videos/Megamind.avi,avi,h264,Good,,"10,50,2" +../tests/test_videos/Megamind.avi,avi,h264,Good,, +../tests/test_videos/Megamind.avi,avi,h264,Good,, +../tests/test_videos/Megamind.avi,avi,h264,Good,, diff --git a/tests/csv_samples/blob_1.txt b/tests/csv_samples/blob_1.txt new file mode 100644 index 00000000..5bde9ca8 --- /dev/null +++ b/tests/csv_samples/blob_1.txt @@ -0,0 +1 @@ +0.840188;0.394383;0.783099;0.79844;0.911647;0.197551;0.335223;0.76823;0.277775;0.55397;0.477397;0.628871;0.364784;0.513401;0.95223;0.916195;0.635712;0.717297;0.141603;0.606969;0.0163006;0.242887;0.137232;0.804177;0.156679;0.400944;0.12979;0.108809;0.998924;0.218257;0.512932;0.839112;0.61264;0.296032;0.637552;0.524287;0.493583;0.972775;0.292517;0.771358;0.526745;0.769914;0.400229;0.891529;0.283315;0.352458;0.807725;0.919026;0.0697553;0.949327;0.525995;0.0860558;0.192214;0.663227;0.890233;0.348893;0.0641713;0.020023;0.457702;0.0630958;0.23828;0.970634;0.902208;0.85092;0.266666;0.53976;0.375207;0.760249;0.512535;0.667724;0.531606;0.0392803;0.437638;0.931835;0.93081;0.720952;0.284293;0.738534;0.639979;0.354049;0.687861;0.165974;0.440105;0.880075;0.829201;0.330337;0.228968;0.893372;0.35036;0.68667;0.956468;0.58864;0.657304;0.858676;0.43956;0.92397;0.398437;0.814767;0.684219;0.910972;0.482491;0.215825;0.950252;0.920128;0.14766;0.881062;0.641081;0.431953;0.619596;0.281059;0.786002;0.307458;0.447034;0.226107;0.187533;0.276235;0.556444;0.416501;0.169607;0.906804;0.103171;0.126075;0.495444;0.760475;0.984752;0.935004;0.684445;0.383188;0.749771;0.368664;0.29416;0.232262;0.584489;0.244413;0.15239;0.732149;0.125475;0.79347;0.164102;0.745071;0.0745298;0.950104;0.0525293;0.521563;0.176211;0.240062;0.797798;0.732654;0.656564;0.967405;0.639458;0.759735;0.0934805;0.134902;0.52021;0.0782321;0.0699064;0.204655;0.46142;0.819677;0.573319;0.755581;0.0519388;0.157807;0.999994;0.204329;0.889956;0.125468;0.997799;0.0540576;0.87054;0.0723288;0.00416161;0.923069;0.593892;0.180372;0.163132;0.39169;0.913027;0.819695;0.359095;0.552485;0.57943;0.452576;0.687387;0.0996401;0.530808;0.757294;0.304295;0.992228;0.576971;0.877614;0.747809;0.62891;0.0354209;0.747803;0.833239;0.925377;0.873271;0.831038;0.979434;0.743811;0.903366;0.983596;0.66688;0.497259;0.163968;0.830012;0.888949;0.0769947;0.649707;0.248044;0.62948;0.229137;0.70062;0.316867;0.328777;0.231428;0.074161;0.633072;0.223656;0.651132;0.510686;0.971466;0.280042;0.546107;0.719269;0.113281;0.471483;0.59254;0.944318;0.450918;0.336351;0.847684;0.434513;0.00323146;0.344943;0.598481;0.833243;0.233892;0.675476;0.48295;0.481936;0.304956;0.712087;0.182556;0.621823;0.0408643;0.413984;0.695984;0.673936;0.63764;0.347116;0.184622;0.609106;0.627158;0.730729;0.328374;0.740438;0.202213;0.920914;0.684757;0.65313;0.257265;0.532441;0.0876436;0.260497;0.877384;0.686125;0.0937402;0.111276;0.361601;0.57669;0.593211;0.666557;0.288778;0.775767;0.288379;0.329642;0.189751;0.984363;0.00357857;0.827391;0.331479;0.188201;0.436497;0.958637;0.91893;0.764871;0.699075;0.121143;0.685786;0.383832;0.774274;0.943051;0.916273;0.861917;0.203548;0.793657;0.548042;0.297288;0.904932;0.909643;0.873979;0.498144;0.5762;0.162757;0.273911;0.864579;0.492399;0.463662;0.848942;0.495977;0.291053;0.180421;0.684178;0.72755;0.139058;0.603109;0.492422;0.838134;0.724252;0.178208;0.221966;0.498525;0.121259;0.138238;0.360443;0.324807;0.931895;0.908485;0.622095;0.836828;0.818128;0.496074;0.334972;0.394327;0.658831;0.608883;0.258906;0.15123;0.072545;0.107848;0.647207;0.363598;0.28827;0.331386;0.0911486;0.427328;0.934495;0.58357;0.265461;0.658747;0.761778;0.487427;0.157272;0.883037;0.625665;0.517715;0.207844;0.557561;0.426199;0.829939;0.394388;0.244327;0.326013;0.72936;0.638654;0.984845;0.338243;0.89756;0.136075;0.410788;0.00540855;0.783282;0.774386;0.293678;0.114668;0.865535;0.721006;0.0491625;0.449105;0.986467;0.707909;0.210883;0.473894;0.865181;0.0939195;0.0995593;0.382896;0.301763;0.65712;0.809095;0.131702;0.0515083;0.0534223;0.457716;0.780868;0.692076;0.44256;0.119111;0.589637;0.578635;0.529899;0.595045;0.361917;0.304285;0.888723;0.476585;0.16982;0.609729;0.525747;0.618925;0.596196;0.233656;0.829808;0.0700902;0.0988374;0.923728;0.169649;0.481733;0.225491;0.826769;0.290829;0.357193;0.878278;0.344251;0.814909;0.659146;0.0363274;0.257469;0.778257;0.625964;0.836104;0.308157;0.221009;0.198021;0.612442;0.109733;0.674605;0.782262;0.719462;0.200352;0.401188;0.315658;0.434009;0.230996;0.385748;0.532846;0.154724;0.555398;0.0145793;0.380215;0.382167;0.305408;0.737408;0.260445;0.649659;0.552316;0.919591;0.685986;0.809785;0.697848;0.31195;0.645889;0.00600477;0.53296;0.84391;0.618447;0.642693;0.518515;0.400709;0.362154;0.718867;0.801897;0.677812;0.152876;0.0328927;0.0635606;0.685722;0.187616;0.618958;0.700301;0.567831;0.00112548;0.00570914;0.305239;0.26157;0.655368;0.857555;0.181161;0.341354;0.667341;0.879009;0.653305;0.31323;0.885014;0.186265;0.157139;0.503461;0.828957;0.675654;0.90417;0.191112;0.394521;0.706067;0.868924;0.547397;0.738959;0.932485;0.233119;0.926576;0.551443;0.93342;0.494407;0.552568;0.939129;0.799646;0.814139;0.594497;0.657201;0.9953;0.935852;0.324541;0.874309;0.589157;0.637771;0.759324;0.775421;0.79491;0.262785;0.604379;0.470564;0.166955;0.79549;0.865085;0.873021;0.664414;0.412483;0.611981;0.596899;0.645602;0.538557;0.148342;0.579022;0.0329634;0.70091;0.518151;0.832609;0.515049;0.112648;0.48981;0.510349;0.0484997;0.814351;0.384658;0.637656;0.452122;0.143982;0.413078;0.247033;0.406767;0.0174566;0.717597;0.573721;0.812947;0.582682;0.446743;0.477361;0.995165;0.0587232;0.0742604;0.640766;0.59728;0.222602;0.219788;0.630243;0.923513;0.737939;0.462852;0.438562;0.850586;0.952662;0.948911;0.899086;0.767014;0.333569;0.536743;0.219136;0.477551;0.94982;0.466169;0.884318;0.967277;0.183765;0.458039;0.780224;0.766448;0.904782;0.257585;0.761612;0.963505;0.331846;0.402379;0.560785;0.554448;0.622167;0.191028;0.477961;0.360105;0.65388;0.916523;0.210692;0.606542;0.865434;0.109778;0.373556;0.199003;0.64652;0.592692;0.676554;0.596341;0.0588605;0.560872;0.563617;0.242626;0.0189108;0.343841;0.00907344;0.923692;0.601427;0.770686;0.887197;0.933273;0.173065;0.447982;0.487721;0.795231;0.639009;0.965682;0.155336;0.292889;0.882204;0.366028;0.899431;0.747638;0.475806;0.272987;0.94664;0.122326;0.865679;0.623194;0.718666;0.92454;0.184066;0.282284;0.167165;0.202977;0.626125;0.176239;0.126669;0.227552;0.946925;0.0138663;0.160824;0.119989;0.461848;0.648545;0.915221;0.100857;0.614227;0.070557;0.393746;0.496431;0.436585;0.293177;0.244069;0.912391;0.566164;0.190709;0.0347164;0.431844;0.813904;0.753383;0.356383;0.99797;0.0356664;0.523548;0.200947;0.661792;0.699787;0.327616;0.889343;0.646712;0.341482;0.0501679;0.766701;0.80333;0.698713;0.681922;0.904187;0.31294;0.752479;0.297933;0.809371;0.189064;0.591111;0.0534394;0.101454;0.157275;0.244149;0.136171;0.589119;0.0580523;0.889553;0.945502;0.0560222;0.92522;0.46905;0.256969;0.587011;0.168837;0.584585;0.476355;0.815549;0.926068;0.526523;0.58225;0.729398;0.225236;0.264172;0.633585;0.538175;0.0166506;0.931518;0.347546;0.205714;0.522629;0.400985;0.307168;0.679904;0.645134;0.443339;0.269022;0.703186;0.332892;0.214524;0.759208;0.258112;0.683574;0.0161775;0.845123;0.852411;0.600763;0.321478;0.66796;0.52683;0.848;0.25021;0.256228;0.0732357;0.514382;0.889813;0.611411;0.531033;0.821331;0.958957;0.736747;0.343959;0.359942;0.0439153;0.0238632;0.0050762;0.487254;0.292886;0.708262;0.820146;0.50741;0.467471;0.0782579;0.190984;0.483648;0.923381;0.0433947;0.084411;0.244858;0.711355;0.611241;0.0928584;0.961565;0.867469;0.166094;0.475947;0.757282;0.777505;0.00698012;0.578613;0.736462;0.743727;0.922572;0.0964041;0.787642;0.946435;0.10148;0.274897;0.239321;0.809743;0.0950428;0.74673;0.277214;0.173301;0.937714;0.760862;0.0966814;0.981109;0.845273;0.34154;0.692463;0.456514;0.434398;0.654029;0.323983;0.600492;0.129976;0.081265;0.377997;0.136956;0.659878;0.114459;0.880683;0.58245;0.210863;0.668326;0.528885;0.312343;0.943222;0.768206;0.122086;0.0382648;0.514936;0.3993;0.211565;0.45265;0.160162;0.308247;0.433758;0.00543489;0.649787;0.126222;0.461949;0.0841846;0.78025;0.785932;0.684677;0.910227;0.867197;0.0626739;0.0471826;0.527075;0.177133;0.927866;0.109525;0.387996;0.596191;0.638409;0.70034;0.539413;0.406615;0.822426;0.577678;0.921551;0.221726;0.789244;0.374201;0.381888;0.0974906;0.807959;0.387323;0.747277;0.934181;0.849272;0.831462;0.714432;0.635204;0.516139;0.624658;0.502401;0.578813;0.671841;0.0294762;0.755946;0.599707;0.139001;0.143942;0.195898;0.77741;0.844281;0.735311;0.184025;0.666707;0.31299;0.105576;0.888433;0.102233;0.479777;0.270321;0.199724;0.287736;0.657643;0.947001;0.221918;0.506915;0.778463;0.936349;0.142119;0.294601;0.561007;0.64452;0.873414;0.232848;0.673996;0.629359;0.832555;0.812997;0.773301;0.0284525;0.590407;0.617582;0.763764;0.774432;0.284289;0.0767534;0.880009;0.172722;0.178987;0.359786;0.443043;0.37871;0.647522;0.100686;0.325711;0.86944;0.6076;0.104174;0.805789;0.749719;0.398775;0.366796;0.394239;0.272189;0.599644;0.0682348;0.901549;0.432199;0.881232;0.67485;0.460652;0.471639;0.292432;0.224415;0.246071;0.576721;0.301169;0.12608;0.749443;0.480155;0.485866;0.192486;0.858866;0.133388;0.293171;0.184577;0.00282779;0.900772;0.288752;0.808617;0.650491;0.687527;0.175413;0.0447295;0.959716;0.775058;0.112964;0.861265;0.207257;0.994196;0.536115;0.667908;0.465835;0.828546;0.892324;0.711906;0.405267;0.193493;0.837986;0.154711;0.673648;0.323852;0.347196;0.532514;0.45724;0.640368;0.717092;0.460067;0.54114;0.00584319;0.268684;0.19163;0.69337;0.444097;0.23636;0.653087;0.219155;0.349324;0.514352;0.426412;0.34352;0.0504663;0.0943199;0.809355;0.879013;0.986644;0.521261;0.28428 diff --git a/tests/csv_samples/connection.csv b/tests/csv_samples/connection.csv new file mode 100644 index 00000000..571d2210 --- /dev/null +++ b/tests/csv_samples/connection.csv @@ -0,0 +1,5 @@ +ConnectionClass,Person@id,Person@id,prop_type +BloodRelation,1,2,brother +BloodRelation,14,16,sister +BloodRelation,14,15,mother +BloodRelation,14,13,father \ No newline at end of file diff --git a/tests/csv_samples/person.csv b/tests/csv_samples/person.csv new file mode 100644 index 00000000..2ab3a370 --- /dev/null +++ b/tests/csv_samples/person.csv @@ -0,0 +1,6 @@ +EntityClass,prop_name,prop_lastname,prop_id,prop_age +Person,Ali,Hum,1,2 +Person,Hamzah,Hom,2,3 +Person,Tala,Ali,16,45 +Person,Soha,Khalid,14,12 +Person,Shah,Hum,15,40 diff --git a/tests/python/TestCommand.py b/tests/python/TestCommand.py index c151a53b..37a4b23f 100644 --- a/tests/python/TestCommand.py +++ b/tests/python/TestCommand.py @@ -24,14 +24,11 @@ # THE SOFTWARE. # -import sys -import os -import urllib import time -import json import unittest import vdms + class TestCommand(unittest.TestCase): def __init__(self, *args, **kwargs): @@ -39,7 +36,7 @@ def __init__(self, *args, **kwargs): # VDMS Server Info self.hostname = "localhost" - self.port = 55557 + self.port = 55565 db_up = False attempts = 0 diff --git a/tests/python/TestFindDescriptors.py b/tests/python/TestFindDescriptors.py index 35a81c3a..4db55ea2 100644 --- a/tests/python/TestFindDescriptors.py +++ b/tests/python/TestFindDescriptors.py @@ -53,7 +53,7 @@ def create_set_and_insert(self, set_name, dims, total, labels=True): descriptor_blob = [] class_counter = -1 - for i in range(0,total-1): + for i in range(0,total): if ((i % 4) == 0): class_counter += 1 @@ -80,16 +80,16 @@ def create_set_and_insert(self, set_name, dims, total, labels=True): response, img_array = db.query(all_queries, [descriptor_blob]) # Check success - for x in range(0,total-1): + for x in range(0,total): self.assertEqual(response[x]["AddDescriptor"]["status"], 0) - @unittest.skip("Skipping class until fixed") + # @unittest.skip("Skipping class until fixed") def test_findDescByConstraints(self): # Add Set set_name = "features_128d_4_findbyConst" dims = 128 - total = 2 + total = 5 self.create_set_and_insert(set_name, dims, total) db = self.create_connection() @@ -100,7 +100,7 @@ def test_findDescByConstraints(self): finddescriptor["set"] = set_name constraints = {} - constraints["myid"] = ["==", 205] + constraints["myid"] = ["==", 202] finddescriptor["constraints"] = constraints results = {} @@ -119,15 +119,15 @@ def test_findDescByConstraints(self): self.assertEqual(response[0]["FindDescriptor"]["status"], 0) self.assertEqual(response[0]["FindDescriptor"]["returned"], 1) self.assertEqual(response[0]["FindDescriptor"] - ["entities"][0]["myid"], 205) + ["entities"][0]["myid"], 202) - @unittest.skip("Skipping class until fixed") + # @unittest.skip("Skipping class until fixed") def test_findDescUnusedRef(self): # Add Set set_name = "features_128d_4_findunusedRef" dims = 128 - total = 2 + total = 5 self.create_set_and_insert(set_name, dims, total) db = self.create_connection() @@ -138,7 +138,7 @@ def test_findDescUnusedRef(self): finddescriptor["set"] = set_name constraints = {} - constraints["myid"] = ["==", 205] + constraints["myid"] = ["==", 202] finddescriptor["constraints"] = constraints results = {} @@ -156,16 +156,15 @@ def test_findDescUnusedRef(self): # Check success self.assertEqual(response[0]["FindDescriptor"]["status"], 0) self.assertEqual(response[0]["FindDescriptor"]["returned"], 1) - self.assertEqual(response[0]["FindDescriptor"] - ["entities"][0]["myid"], 205) + self.assertEqual(response[0]["FindDescriptor"]["entities"][0]["myid"], 202) - @unittest.skip("Skipping class until fixed") + # @unittest.skip("Skipping class until fixed") def test_findDescByConst_get_id(self): # Add Set set_name = "features_128d_4_findDescriptors_id" dims = 128 - total = 2 + total = 5 self.create_set_and_insert(set_name, dims, total) db = self.create_connection() @@ -176,7 +175,7 @@ def test_findDescByConst_get_id(self): finddescriptor["set"] = set_name constraints = {} - constraints["myid"] = ["==", 205] + constraints["myid"] = ["==", 202] finddescriptor["constraints"] = constraints results = {} @@ -195,15 +194,15 @@ def test_findDescByConst_get_id(self): self.assertEqual(response[0]["FindDescriptor"]["status"], 0) self.assertEqual(response[0]["FindDescriptor"]["returned"], 1) self.assertEqual(response[0]["FindDescriptor"] - ["entities"][0]["myid"], 205) + ["entities"][0]["myid"], 202) - @unittest.skip("Skipping class until fixed") + # @unittest.skip("Skipping class until fixed") def test_findDescByConst_blobTrue(self): # Add Set set_name = "features_128d_4_findDescriptors_id_blob" dims = 128 - total = 2 + total = 5 self.create_set_and_insert(set_name, dims, total) db = self.create_connection() @@ -214,7 +213,7 @@ def test_findDescByConst_blobTrue(self): finddescriptor["set"] = set_name constraints = {} - constraints["myid"] = ["==", 205] + constraints["myid"] = ["==", 202] finddescriptor["constraints"] = constraints results = {} @@ -234,17 +233,17 @@ def test_findDescByConst_blobTrue(self): self.assertEqual(response[0]["FindDescriptor"]["status"], 0) self.assertEqual(response[0]["FindDescriptor"]["returned"], 1) self.assertEqual(response[0]["FindDescriptor"] - ["entities"][0]["myid"], 205) + ["entities"][0]["myid"], 202) self.assertEqual(len(fv_array), 1) self.assertEqual(len(fv_array[0]), dims*4) - - @unittest.skip("Skipping class until fixed") + + # @unittest.skip("Skipping class until fixed") def test_findDescByConst_multiple_blobTrue(self): # Add Set set_name = "features_128d_4_findDescriptors_m_blob" dims = 128 - total = 2 + total = 5 self.create_set_and_insert(set_name, dims, total) db = self.create_connection() @@ -255,11 +254,12 @@ def test_findDescByConst_multiple_blobTrue(self): finddescriptor["set"] = set_name constraints = {} - constraints["myid"] = ["<=", 205] + constraints["myid"] = ["<=", 202] finddescriptor["constraints"] = constraints results = {} results["list"] = ["myid"] + results["sort"] = "myid" results["blob"] = True finddescriptor["results"] = results @@ -273,19 +273,18 @@ def test_findDescByConst_multiple_blobTrue(self): # Check success self.assertEqual(response[0]["FindDescriptor"]["status"], 0) - self.assertEqual(response[0]["FindDescriptor"]["returned"], 6) - self.assertEqual(response[0]["FindDescriptor"] - ["entities"][5]["myid"], 200) - self.assertEqual(len(fv_array), 6) + self.assertEqual(response[0]["FindDescriptor"]["returned"], 3) + self.assertEqual(response[0]["FindDescriptor"]["entities"][1]["myid"], 201) + self.assertEqual(len(fv_array), 3) self.assertEqual(len(fv_array[0]), dims*4) - @unittest.skip("Skipping class until fixed") + # @unittest.skip("Skipping class until fixed") def test_findDescByBlob(self): # Add Set set_name = "findwith_blob" dims = 128 - total = 2 + total = 5 self.create_set_and_insert(set_name, dims, total) db = self.create_connection() @@ -311,7 +310,7 @@ def test_findDescByBlob(self): descriptor_blob = [] x = np.ones(dims) - x[2] = 2.34 + 30*20 + x[2] = x[2] = 2.34 + 1*20 #2.34 + 1*20 x = x.astype('float32') descriptor_blob.append(x.tobytes()) @@ -323,7 +322,6 @@ def test_findDescByBlob(self): # Check success self.assertEqual(response[0]["FindDescriptor"]["status"], 0) self.assertEqual(response[0]["FindDescriptor"]["returned"], kn) - self.assertEqual(response[0]["FindDescriptor"] ["entities"][0]["_distance"], 0) self.assertEqual(response[0]["FindDescriptor"] @@ -331,13 +329,13 @@ def test_findDescByBlob(self): self.assertEqual(response[0]["FindDescriptor"] ["entities"][2]["_distance"], 400) - @unittest.skip("Skipping class until fixed") + # @unittest.skip("Skipping class until fixed") def test_findDescByBlobNoLabels(self): # Add Set set_name = "findwith_blob_no_labels" dims = 128 - total = 2 + total = 5 self.create_set_and_insert(set_name, dims, total, labels=False) db = self.create_connection() @@ -363,7 +361,7 @@ def test_findDescByBlobNoLabels(self): descriptor_blob = [] x = np.ones(dims) - x[2] = 2.34 + 30*20 + x[2] = 2.34 + 1*20 x = x.astype('float32') descriptor_blob.append(x.tobytes()) @@ -376,13 +374,13 @@ def test_findDescByBlobNoLabels(self): self.assertEqual(response[0]["FindDescriptor"]["status"], 0) self.assertEqual(response[0]["FindDescriptor"]["returned"], kn) - @unittest.skip("Skipping class until fixed") + # @unittest.skip("Skipping class until fixed") def test_findDescByBlobNoResults(self): # Add Set set_name = "findwith_blobNoResults" dims = 128 - total = 1 + total = 0 self.create_set_and_insert(set_name, dims, total) db = self.create_connection() @@ -415,17 +413,17 @@ def test_findDescByBlobNoResults(self): # Check success self.assertEqual(response[0]["FindDescriptor"]["status"], 0) - self.assertEqual(response[0]["FindDescriptor"]["returned"], kn) - self.assertEqual(len(blob_array), kn) - self.assertEqual(descriptor_blob[0], blob_array[0]) + self.assertEqual(response[0]["FindDescriptor"]["returned"], 0) + # self.assertEqual(len(blob_array), kn) + # self.assertEqual(descriptor_blob[0], blob_array[0]) - @unittest.skip("Skipping class until fixed") + # @unittest.skip("Skipping class until fixed") def test_findDescByBlobUnusedRef(self): # Add Set set_name = "findwith_blobUnusedRef" dims = 50 - total = 1 + total = 3 self.create_set_and_insert(set_name, dims, total) db = self.create_connection() @@ -451,7 +449,7 @@ def test_findDescByBlobUnusedRef(self): descriptor_blob = [] x = np.ones(dims) - x[2] = 2.34 + 30*20 + x[2] = 2.34 + 1*20 x = x.astype('float32') descriptor_blob.append(x.tobytes()) @@ -463,71 +461,65 @@ def test_findDescByBlobUnusedRef(self): self.assertEqual(len(blob_array), kn) self.assertEqual(descriptor_blob[0], blob_array[0]) - # This Test is not passing: - # It should do knn and filter by constraints. - # def test_findDescByBlobAndConstraints(self): + # @unittest.skip("Skipping class until fixed") + def test_findDescByBlobAndConstraints(self): - # # Add Set - # set_name = "findwith_blob_const" - # dims = 128 - # total = 100 - # self.create_set_and_insert(set_name, dims, total) + # Add Set + set_name = "findwith_blob_const" + dims = 128 + total = 5 + self.create_set_and_insert(set_name, dims, total) - # db = vdms.vdms() - # db.connect(hostname, port) + db = self.create_connection() - # kn = 3 + kn = 3 - # all_queries = [] + all_queries = [] - # finddescriptor = {} - # finddescriptor["set"] = set_name - # finddescriptor["k_neighbors"] = kn + finddescriptor = {} + finddescriptor["set"] = set_name + finddescriptor["k_neighbors"] = kn - # results = {} - # results["list"] = ["myid", "_id", "_distance"] - # results["blob"] = True - # finddescriptor["results"] = results + results = {} + results["list"] = ["myid", "_id", "_distance"] + results["blob"] = True + finddescriptor["results"] = results - # constraints = {} - # constraints["myid"] = ["==", 205] - # finddescriptor["constraints"] = constraints + constraints = {} + constraints["myid"] = ["==", 202] + finddescriptor["constraints"] = constraints - # query = {} - # query["FindDescriptor"] = finddescriptor + query = {} + query["FindDescriptor"] = finddescriptor - # all_queries = [] - # all_queries.append(query) + all_queries = [] + all_queries.append(query) - # descriptor_blob = [] - # x = np.ones(dims) - # x[2] = 2.34 + 30*20 - # x = x.astype('float32') - # descriptor_blob.append(x.tobytes()) + descriptor_blob = [] + x = np.ones(dims) + x[2] = 2.34 + 2*20 + x = x.astype('float32') + descriptor_blob.append(x.tobytes()) - # response, blob_array = db.query(all_queries, [descriptor_blob]) + response, blob_array = db.query(all_queries, [descriptor_blob]) - # self.assertEqual(len(blob_array), kn) - # self.assertEqual(descriptor_blob[0], blob_array[0]) + self.assertEqual(len(blob_array), 1) + self.assertEqual(descriptor_blob[0], blob_array[0]) - # # Check success - # self.assertEqual(response[0]["FindDescriptor"]["status"], 0) - # self.assertEqual(response[0]["FindDescriptor"]["returned"], kn) + # Check success + self.assertEqual(response[0]["FindDescriptor"]["status"], 0) + self.assertEqual(response[0]["FindDescriptor"]["returned"], 1) - # self.assertEqual(response[0]["FindDescriptor"] - # ["entities"][0]["_distance"], 0) - # self.assertEqual(response[0]["FindDescriptor"] - # ["entities"][1]["_distance"], 400) - # self.assertEqual(response[0]["FindDescriptor"] - # ["entities"][2]["_distance"], 400) + self.assertEqual(response[0]["FindDescriptor"] + ["entities"][0]["_distance"], 0) - @unittest.skip("Skipping class until fixed") + # @unittest.skip("Skipping class until fixed") def test_findDescByBlobWithLink(self): # Add Set set_name = "findwith_blob_link" dims = 128 - total = 1 + total = 3 db = self.create_connection() @@ -550,7 +542,7 @@ def test_findDescByBlobWithLink(self): descriptor_blob = [] class_counter = -1 - for i in range(0,total-1): + for i in range(0,total): #-1): if ((i % 4) == 0): class_counter += 1 @@ -620,12 +612,13 @@ def test_findDescByBlobWithLink(self): descriptor_blob = [] x = np.ones(dims) - x[2] = 2.34 + 30*20 + x[2] = 2.34 + 1*20 x = x.astype('float32') descriptor_blob.append(x.tobytes()) results = {} results["list"] = ["entity_prop"] + results["sort"] = "entity_prop" link = {} link["ref"] = reference @@ -662,8 +655,8 @@ def test_findDescByBlobWithLink(self): self.assertEqual(response[1]["FindEntity"]["returned"], kn) self.assertEqual(response[1]["FindEntity"] - ["entities"][0]["entity_prop"], 231) + ["entities"][0]["entity_prop"], 200) self.assertEqual(response[1]["FindEntity"] - ["entities"][1]["entity_prop"], 230) + ["entities"][1]["entity_prop"], 201) self.assertEqual(response[1]["FindEntity"] - ["entities"][2]["entity_prop"], 229) + ["entities"][2]["entity_prop"], 202) diff --git a/tests/python/TestRetail.py b/tests/python/TestRetail.py index 55217393..d879697f 100644 --- a/tests/python/TestRetail.py +++ b/tests/python/TestRetail.py @@ -28,6 +28,7 @@ import TestCommand import longquery import numpy as np +import unittest n_cameras = 15 dim = 1000 @@ -206,7 +207,7 @@ def single(self, thID, db, results): results[thID] = 0 - + @unittest.skip("Skipping class until fixed") def test_concurrent(self): self.build_store() diff --git a/tests/python/TestVideos.py b/tests/python/TestVideos.py index b08bcd96..efe5fc46 100644 --- a/tests/python/TestVideos.py +++ b/tests/python/TestVideos.py @@ -25,6 +25,7 @@ # import TestCommand +import unittest class TestVideos(TestCommand.TestCommand): @@ -128,6 +129,7 @@ def test_addVideoFromLocalFile_file_not_found(self): response, obj_array = db.query([query], [[]]) self.assertEqual(response[0]["status"], -1) + @unittest.skip("Skipping class until fixed") def test_addVideoFromLocalFile_success(self): db = self.create_connection() diff --git a/tests/python/config-tests.json b/tests/python/config-tests.json index fb1d3077..30141207 100644 --- a/tests/python/config-tests.json +++ b/tests/python/config-tests.json @@ -3,7 +3,7 @@ // Sets database paths and other parameters { // Network - "port": 55557, + "port": 55565, "db_root_path": "test_db", "more-info": "github.com/IntelLabs/vdms" diff --git a/tests/python/run_python_tests.sh b/tests/python/run_python_tests.sh index 2363db92..b5e34dbb 100755 --- a/tests/python/run_python_tests.sh +++ b/tests/python/run_python_tests.sh @@ -24,15 +24,25 @@ # THE SOFTWARE. # -rm log.log screen.log -rm -r test_db +TEST_DIR=${PWD} +base_dir=$(dirname $(dirname $PWD)) +client_path=${base_dir}/client/python +export PYTHONPATH=$client_path:${PYTHONPATH} -./../../build/vdms -cfg config-tests.json > screen.log 2> log.log & -python3 -m unittest discover --pattern=Test*.py -v -# coverage run -m unittest discover --pattern=Test*.py -v -# coverage report -m -# coverage xml +# Uncomment to re-generate queryMessage_pb2.py +# python3 -m grpc_tools.protoc -I=${base_dir}/utils/src/protobuf --python_out=${client_path}/vdms ${base_dir}/utils/src/protobuf/queryMessage.proto + +cd ${TEST_DIR} +rm -rf test_db log.log screen.log +mkdir -p test_db +./../../build/vdms -cfg config-tests.json > screen.log 2> log.log & +py_unittest_pid=$! sleep 1 -pkill vdms + +echo 'Running Python tests...' +python3 -m coverage run --include="../../*" --omit="../*" -m unittest discover --pattern=Test*.py -v + +rm -rf test_db log.log screen.log +kill -9 $py_unittest_pid diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 1e448940..2ee2f92e 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -1,5 +1,5 @@ sh cleandbs.sh - +mkdir test_db_client mkdir dbs # necessary for Descriptors mkdir temp # necessary for Videos mkdir videos_tests @@ -8,14 +8,12 @@ mkdir backups # Start server for client test ./../build/vdms -cfg unit_tests/config-tests.json > tests_screen.log 2> tests_log.log & +./../build/vdms -cfg unit_tests/config-client-tests.json > tests_screen.log 2> tests_log.log & + echo 'not the vdms application - this file is needed for shared key' > vdms -# Gets coverage for files in ../src and ../include -# OMIT Descriptors_Add.add_1by1_and_search_1k due to duration echo 'Running C++ tests...' ./../build/tests/unit_tests \ --gtest_filter=-ImageTest.CreateNameTDB:ImageTest.NoMetadata:VideoTest.CreateUnique:Descriptors_Add.add_1by1_and_search_1k -# echo 'Running Python tests...' -# cd python -# sh run_python_tests.sh +# kill -9 $cpp_unittest_pid $client_test_pid diff --git a/tests/server/AddFindUpdate_blob.json b/tests/server/AddFindUpdate_blob.json new file mode 100644 index 00000000..664acda0 --- /dev/null +++ b/tests/server/AddFindUpdate_blob.json @@ -0,0 +1,39 @@ +[ + { + "AddBlob": + { + + "_ref": 12, + + "properties": { + "Name":"blob-sample-1", + "colored": "true", + "file":"audio" + } + } + + }, + { + "UpdateBlob" : { + + "constraints": { + "Name" : [ "==", "blob-sample-1" ] + }, + + "properties": { + "colored" : "false", + "length" : 200 + } + } + }, + { + "FindBlob" : { + "constraints" : { + "Name" : [ "==", "blob-sample-1" ] + }, + "results" : { + "list" : [ "Name" ] + } + } + } +] diff --git a/tests/server/config-add10-tests.json b/tests/server/config-add10-tests.json index 0fa2bb15..acdee217 100644 --- a/tests/server/config-add10-tests.json +++ b/tests/server/config-add10-tests.json @@ -3,7 +3,7 @@ // Sets database paths and other parameters { // Network - "port": 55555, + "port": 55557, // Database paths "pmgd_path": "simpleAddx10_db" diff --git a/tests/server/config-addfind-tests.json b/tests/server/config-addfind-tests.json index e243bcec..8452e1ab 100644 --- a/tests/server/config-addfind-tests.json +++ b/tests/server/config-addfind-tests.json @@ -3,7 +3,7 @@ // Sets database paths and other parameters { // Network - "port": 55555, + "port": 55557, // Database paths "pmgd_path": "jsongraph" diff --git a/tests/server/config-auto-replicate-tests.json b/tests/server/config-auto-replicate-tests.json index d37dfcff..9d283df1 100755 --- a/tests/server/config-auto-replicate-tests.json +++ b/tests/server/config-auto-replicate-tests.json @@ -1,5 +1,5 @@ { - "port": 55555, + "port": 55557, "autoreplicate_interval":5, "unit":"s", "max_simultaneous_clients": 100, diff --git a/tests/server/config-datatype-tests.json b/tests/server/config-datatype-tests.json index 69f2762a..5373514d 100644 --- a/tests/server/config-datatype-tests.json +++ b/tests/server/config-datatype-tests.json @@ -3,7 +3,7 @@ // Sets database paths and other parameters { // Network - "port": 55555, + "port": 55557, // Database paths "pmgd_path": "datatypecheck_db" diff --git a/tests/server/config-emptyresult-tests.json b/tests/server/config-emptyresult-tests.json index e52ceb4c..e66ff24c 100644 --- a/tests/server/config-emptyresult-tests.json +++ b/tests/server/config-emptyresult-tests.json @@ -3,7 +3,7 @@ // Sets database paths and other parameters { // Network - "port": 55555, + "port": 55557, // Database paths "pmgd_path": "entitycheck_db" diff --git a/tests/server/config-tests.json b/tests/server/config-tests.json index 2d158362..cf28e646 100644 --- a/tests/server/config-tests.json +++ b/tests/server/config-tests.json @@ -3,7 +3,7 @@ // Sets database paths and other parameters { // Network - "port": 55555, + "port": 55557, // Database paths "pmgd_path": "simpleAdd_db", diff --git a/tests/server/config-update-tests.json b/tests/server/config-update-tests.json index 8765c8c4..189607ec 100644 --- a/tests/server/config-update-tests.json +++ b/tests/server/config-update-tests.json @@ -3,7 +3,7 @@ // Sets database paths and other parameters { // Network - "port": 55555, + "port": 55557, // Database paths "pmgd_path": "simpleUpdate_db", diff --git a/tests/server/json_queries.cc b/tests/server/json_queries.cc index 6132119b..d20b2f29 100644 --- a/tests/server/json_queries.cc +++ b/tests/server/json_queries.cc @@ -63,25 +63,25 @@ std::string singleAddImage(" \ "); TEST( AutoReplicate, default_replicate) { - + VDMSConfig::init("server/config-auto-replicate-tests.json"); PMGDQueryHandler::init(); QueryHandler::init(); std::string backup_path ="backups"; - std::string db_path="db_backup"; - int port =55555; + std::string db_path="db_backup"; + int port =55557; QueryHandler qh_base; qh_base.regualar_run_autoreplicate(backup_path, db_path, port); // set flag to show autodelete queue has been initialized QueryHandlerTester query_handler(qh_base); - + VDMSConfig::destroy(); PMGDQueryHandler::destroy(); -} +} TEST(AddImage, simpleAdd) @@ -413,7 +413,7 @@ TEST(QueryHandler, EmptyResultCheck) PMGDQueryHandler::init(); QueryHandler::init(); - QueryHandler qh_base; + QueryHandler qh_base; qh_base.reset_autodelete_init_flag(); // set flag to show autodelete queue has been initialized QueryHandlerTester query_handler(qh_base); @@ -633,3 +633,67 @@ TEST(QueryHandler, CustomFunctionNoProcess) VDMSConfig::destroy(); PMGDQueryHandler::destroy(); } + + +TEST(QueryHandler, AddUpdateFind_Blob) +{ + + Json::StyledWriter writer; + + std::ifstream ifile; + int fsize; + char * inBuf; + ifile.open("server/AddFindUpdate_blob.json", std::ifstream::in); + ifile.seekg(0, std::ios::end); + fsize = (int)ifile.tellg(); + ifile.seekg(0, std::ios::beg); + inBuf = new char[fsize]; + ifile.read(inBuf, fsize); + std::string json_query = std::string(inBuf); + ifile.close(); + delete[] inBuf; + + Json::Reader reader; + Json::Value root; + Json::Value parsed; + + VDMSConfig::init("unit_tests/config-tests.json"); + PMGDQueryHandler::init(); + QueryHandler::init(); + + QueryHandler qh_base; + qh_base.reset_autodelete_init_flag(); // set flag to show autodelete queue has been initialized + QueryHandlerTester query_handler(qh_base); + + VDMS::protobufs::queryMessage proto_query; + proto_query.set_json(json_query); + + std::string image; + std::ifstream file("test_images/brain.png", + std::ios::in | std::ios::binary | std::ios::ate); + + image.resize(file.tellg()); + + file.seekg(0, std::ios::beg); + if( !file.read(&image[ 0 ], image.size())) + std::cout << "error" << std::endl; + + proto_query.add_blobs(image); + VDMS::protobufs::queryMessage response; + + query_handler.pq(proto_query, response ); + + reader.parse(response.json().c_str(), parsed); + // std::cout << writer.write(parsed) << std::endl; + + // Verify results returned. + for (int j = 0; j < parsed.size(); j++) { + const Json::Value& query = parsed[j]; + ASSERT_EQ(query.getMemberNames().size(), 1); + std::string cmd = query.getMemberNames()[0]; + EXPECT_EQ(query[cmd]["status"].asInt(), 0); + } + + VDMSConfig::destroy(); + PMGDQueryHandler::destroy(); +} diff --git a/tests/images/large1.jpg b/tests/test_images/large1.jpg similarity index 100% rename from tests/images/large1.jpg rename to tests/test_images/large1.jpg diff --git a/tests/unit_tests/Image_test.cc b/tests/unit_tests/Image_test.cc index fcc8cef8..c48be40e 100644 --- a/tests/unit_tests/Image_test.cc +++ b/tests/unit_tests/Image_test.cc @@ -44,7 +44,7 @@ class ImageTest : public ::testing::Test { protected: virtual void SetUp() { - img_ = "images/large1.jpg"; + img_ = "test_images/large1.jpg"; tdb_img_ = "tdb/test_image.tdb"; cv_img_ = cv::imread(img_, -1); @@ -215,7 +215,7 @@ TEST_F(ImageTest, MatConstructor) TEST_F(ImageTest, EncodedBufferConstructor) { - std::fstream jpgimage("images/large1.jpg"); + std::fstream jpgimage("test_images/large1.jpg"); jpgimage.seekg(0, jpgimage.end); int length = jpgimage.tellg(); @@ -538,19 +538,19 @@ TEST_F(ImageTest, Read) VCL::ImageTest img_data; img_data.set_format("jpg"); - ASSERT_THROW(img_data.read("images/.jpg"), VCL::Exception); + ASSERT_THROW(img_data.read("test_images/.jpg"), VCL::Exception); - img_data.read("images/large1"); + img_data.read("test_images/large1"); - EXPECT_EQ("images/large1.jpg", img_data.get_image_id()); + EXPECT_EQ("test_images/large1.jpg", img_data.get_image_id()); } TEST_F(ImageTest, WriteMatToJPG) { VCL::Image img(cv_img_); - img.store("images/test_image", VCL::Image::Format::JPG); + img.store("test_images/test_image", VCL::Image::Format::JPG); - cv::Mat test = cv::imread("images/test_image.jpg"); + cv::Mat test = cv::imread("test_images/test_image.jpg"); EXPECT_FALSE( test.empty() ); } @@ -786,14 +786,14 @@ TEST_F(ImageTest, TDBToPNG) { VCL::Image img(tdb_img_); - img.store("images/tdb_to_png", VCL::Image::Format::PNG); + img.store("test_images/tdb_to_png", VCL::Image::Format::PNG); } TEST_F(ImageTest, TDBToJPG) { VCL::Image img(tdb_img_); - img.store("images/tdb_to_jpg", VCL::Image::Format::JPG); + img.store("test_images/tdb_to_jpg", VCL::Image::Format::JPG); } TEST_F(ImageTest, EncodedImage) diff --git a/tests/unit_tests/TDBImage_test.cc b/tests/unit_tests/TDBImage_test.cc index ed8e31b9..2b9097f2 100644 --- a/tests/unit_tests/TDBImage_test.cc +++ b/tests/unit_tests/TDBImage_test.cc @@ -43,7 +43,7 @@ class TDBImageTest : public ::testing::Test { virtual void SetUp() { tdb_img_ = "tdb/test_image.tdb"; tdb_test_ = "tdb/write_test.tdb"; - cv_img_ = cv::imread("images/large1.jpg", cv::IMREAD_ANYCOLOR); + cv_img_ = cv::imread("test_images/large1.jpg", cv::IMREAD_ANYCOLOR); rect_ = VCL::Rectangle(100, 100, 100, 100); } diff --git a/tests/unit_tests/client_blob.cc b/tests/unit_tests/client_blob.cc new file mode 100644 index 00000000..7af5259d --- /dev/null +++ b/tests/unit_tests/client_blob.cc @@ -0,0 +1,57 @@ +#include "meta_data_helper.h" +#include "CSVParserUtil.h" +TEST(BLOB, add_Blob){ + std::string filename ="../tests/test_images/large1.jpg"; + std::vector blobs; + VDMS::CSVParserUtil csv_util; + std::string* blob_data_ptr = nullptr; + + csv_util.read_blob_image(filename, &blob_data_ptr); + + if(blob_data_ptr!=nullptr){ + blobs.push_back(blob_data_ptr); + // std::cout <<*blobs[0] <read_blob(filename)); + meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); + Json::Value tuple ; + tuple=meta_obj->construct_Blob(); + + + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple), blobs); + Json::Value result; + + meta_obj->_reader.parse(response.json.c_str(), result); + int status1 = result[0]["AddBlob"]["status"].asInt(); + EXPECT_EQ(status1, 0); +} + +TEST(BLOB, update_Blob){ + + Meta_Data* meta_obj=new Meta_Data(); + meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); + Json::Value tuple ; + tuple=meta_obj->construct_updateBlob(); + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple)); + Json::Value result; + + meta_obj->_reader.parse(response.json.c_str(), result); + int status1 = result[0]["status"].asInt(); + + EXPECT_EQ(status1, 0); +} +TEST(BLOB, find_Blob){ + + Meta_Data* meta_obj=new Meta_Data(); + meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); + Json::Value tuple ; + tuple=meta_obj->construct_findBlob(); + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple)); + Json::Value result; + + meta_obj->_reader.parse(response.json.c_str(), result); + int status1 = result[0]["status"].asInt(); + + EXPECT_EQ(status1, 0); +} \ No newline at end of file diff --git a/tests/unit_tests/client_bounding_box.cc b/tests/unit_tests/client_bounding_box.cc index ae85ab86..0bde4d75 100644 --- a/tests/unit_tests/client_bounding_box.cc +++ b/tests/unit_tests/client_bounding_box.cc @@ -14,23 +14,12 @@ TEST(CLIENT_CPP, add_BB){ } TEST(CLIENT_CPP, add_BB_with_image){ - std::fstream jpgimage("../tests/images/large1.jpg"); - jpgimage.seekg(0, jpgimage.end); - int length = jpgimage.tellg(); - // std::cout<<"Length " < blobs; - - std::string *bytes_str = new std::string(buffer); - - blobs.push_back(bytes_str); - Meta_Data* meta_obj=new Meta_Data(); + blobs.push_back(meta_obj->read_blob(filename)); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->constuct_BB(true); @@ -38,7 +27,8 @@ TEST(CLIENT_CPP, add_BB_with_image){ VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple), blobs); Json::Value result; meta_obj->_reader.parse(response.json.c_str(), result); + // std::cout << result < all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddEntity"]["status"].asInt(), 0); + } +} + +// TEST(CLIENT_CPP_CSV, parse_update_csv_entity) +// { + +// std::string filename = "../tests/csv_samples/update_entity.csv"; +// size_t num_threads = 2; +// std::string vdms_server ="localhost"; +// int port = 55558; +// std::vector all_results; +// VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + +// all_results = csv_parser.parse(); +// Json::Value result; +// Json::Reader _reader; +// for (int k=0; k all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddConnection"]["status"].asInt(), 0); + } +} +TEST(CLIENT_CPP_CSV, parse_csv_images) +{ + std::string filename = "../tests/csv_samples/Image.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddImage"]["status"].asInt(), 0); + } +} + +TEST(CLIENT_CPP_CSV, parse_csv_descriptor_set) +{ + std::string filename = "../tests/csv_samples/DescriptorSet.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddDescriptorSet"]["status"].asInt(), 0); + } +} + +TEST(CLIENT_CPP_CSV, parse_csv_descriptor) +{ + std::string filename = "../tests/csv_samples/Descriptor.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddDescriptor"]["status"].asInt(), 0); + } +} +TEST(CLIENT_CPP_CSV, parse_csv_bb) +{ + std::string filename = "../tests/csv_samples/Rectangle.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddBoundingBox"]["status"].asInt(), 0); + } +} +TEST(CLIENT_CPP_CSV, parse_csv_video) +{ + std::string filename = "../tests/csv_samples/Video.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddVideo"]["status"].asInt(), 0); + } +} + +TEST(CLIENT_CPP_CSV, parse_csv_invalid_entity) +{ + std::string filename = "../tests/csv_samples/invalid.csv"; + std::ofstream csv_file; + csv_file.open(filename); + csv_file << "EntityInvalidTest,prop_name,prop_lastname,prop_id,prop_age\n"; + csv_file << "Person,Ali,Hum,1,2\n"; + csv_file.close(); + + size_t num_threads = 1; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + remove(filename.c_str()); + + Json::Value result; + Json::Reader _reader; + _reader.parse(all_results[0].json.c_str(), result); + EXPECT_EQ(result["status"].asInt(), -1); + EXPECT_EQ(result["info"].asString(), "Command does not exist"); +} + +TEST(CLIENT_CPP_CSV, parse_csv_invalid_image) +{ + std::string filename = "../tests/csv_samples/invalid_file.csv"; + std::ofstream csv_file; + csv_file.open(filename); + csv_file << "ImagePath,ops_threshold,ops_crop,ops_resize,ops_flip,ops_rotate,prop_type,prop_part,format,cons_1\n"; + csv_file << "../tests/test_images/large1_invalid.jpg,350,,,,,,image1,jpg,part==image1\n"; + csv_file.close(); + + size_t num_threads = 1; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + remove(filename.c_str()); + + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["status"].asInt(), -1); + } +} + +TEST(CLIENT_CPP_CSV, parse_csv_invalid_video) +{ + std::string filename = "../tests/csv_samples/invalid_file.csv"; + std::ofstream csv_file; + csv_file.open(filename); + csv_file << "VideoPath,format,compressto,prop_name,ops_resize,ops_interval\n"; + csv_file << "../tests/test_videos/Megamind_invalid.avi,avi,h264,Good,,\n"; + csv_file.close(); + + size_t num_threads = 1; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + remove(filename.c_str()); + + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["status"].asInt(), -1); + } +} + diff --git a/tests/unit_tests/client_image.cc b/tests/unit_tests/client_image.cc index 9c49a7e7..8ab44d6e 100644 --- a/tests/unit_tests/client_image.cc +++ b/tests/unit_tests/client_image.cc @@ -3,30 +3,19 @@ TEST(CLIENT_CPP, add_image){ - - std::string image; - std::fstream file("../tests/images/large1.jpg", - std::ios::in | std::ios::binary | std::ios::ate); - image.resize(file.tellg()); - file.seekg(0, std::ios::beg); - if( !file.read(&image[ 0 ], image.size())) - std::cout << "error" << std::endl; + std::string filename ="../tests/test_images/large1.jpg"; std::vector blobs; - - std::string *bytes_str = new std::string(image); - - blobs.push_back(bytes_str); - Meta_Data* meta_obj=new Meta_Data(); + blobs.push_back(meta_obj->read_blob(filename)); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->constuct_image(); - + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple), blobs); Json::Value result; meta_obj->_reader.parse(response.json.c_str(), result); @@ -38,10 +27,10 @@ TEST(CLIENT_CPP, add_image){ TEST(CLIENT_CPP, add_image_resize_operation){ - + std::string image; - std::fstream file("../tests/images/large1.jpg", + std::fstream file("../tests/test_images/large1.jpg", std::ios::in | std::ios::binary | std::ios::ate); image.resize(file.tellg()); @@ -51,42 +40,42 @@ TEST(CLIENT_CPP, add_image_resize_operation){ std::cout << "error" << std::endl; std::vector blobs; - + std::string *bytes_str = new std::string(image); - + blobs.push_back(bytes_str); Json::Value op; op["type"]="resize"; op["width"]=100; - op["height"]=100; + op["height"]=100; Meta_Data* meta_obj=new Meta_Data(); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->constuct_image(true, op); - - + + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple), blobs); Json::Value result; meta_obj->_reader.parse(response.json.c_str(), result); - + int status1 = result[0]["AddImage"]["status"].asInt(); EXPECT_EQ(status1, 0); } TEST(CLIENT_CPP, find_image){ - + Meta_Data* meta_obj=new Meta_Data(); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->construct_find_image(); - - + + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple)); Json::Value result; meta_obj->_reader.parse(response.json.c_str(), result); - + int status1 = result[0]["FindImage"]["status"].asInt(); EXPECT_EQ(status1, 0); diff --git a/tests/unit_tests/client_videos.cc b/tests/unit_tests/client_videos.cc index 20c18f39..887ea75f 100644 --- a/tests/unit_tests/client_videos.cc +++ b/tests/unit_tests/client_videos.cc @@ -24,7 +24,7 @@ string readFileIntoString(const string& path) { -TEST(CLIENT_CPP, add_single_video){ +TEST(CLIENT_CPP_Video, add_single_video){ // std::string video; @@ -32,31 +32,15 @@ TEST(CLIENT_CPP, add_single_video){ std::vector blobs; - const char *_video_id ="../tests/videos/Megamind.avi"; - std::ifstream ifile; - ifile.open(_video_id); - - int fsize; - char* inBuf; - ifile.seekg(0, std::ios::end); - fsize = (long)ifile.tellg(); - ifile.seekg(0, std::ios::beg); - inBuf = new char[fsize]; - ifile.read(inBuf, fsize); - std::string blob = (std::string(inBuf)); - ifile.close(); - delete[] inBuf; - - - std::string* bytes_str =new std::string(blob); - blobs.push_back(bytes_str); - Meta_Data* meta_obj=new Meta_Data(); + std::string filename ="../tests/videos/Megamind.avi"; + + + Meta_Data* meta_obj=new Meta_Data(); + blobs.push_back(meta_obj->read_blob(filename)); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->constuct_video(false); - - - std::cout<< "Printing bytes_str " << bytes_str << "\t"<< blobs[0] << std::endl; + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple), blobs); Json::Value result; diff --git a/tests/unit_tests/config-client-tests.json b/tests/unit_tests/config-client-tests.json new file mode 100644 index 00000000..17dcc1bc --- /dev/null +++ b/tests/unit_tests/config-client-tests.json @@ -0,0 +1,10 @@ +// VDMS Config File +// This is the run-time config file +// Sets database paths and other parameters +{ + + "port": 55558, + "db_root_path": "test_db_client", + + "more-info": "github.com/IntelLabs/vdms" +} \ No newline at end of file diff --git a/tests/unit_tests/meta_data.cc b/tests/unit_tests/meta_data.cc index 11edd7bb..e4f51340 100644 --- a/tests/unit_tests/meta_data.cc +++ b/tests/unit_tests/meta_data.cc @@ -1,9 +1,9 @@ #include "meta_data_helper.h" Meta_Data::Meta_Data(){ - - + } + Json::Value Meta_Data::construct_Flinng_Set( std::string& name, int& dim ){ Json::Value descriptor_set; @@ -20,13 +20,12 @@ Json::Value Meta_Data::construct_Flinng_Set( std::string& name, int& dim ){ descriptor_set["flinng_sub_hash_bits"]=2; descriptor_set["flinng_cut_off"]=6; set_query["AddDescriptorSet"] = descriptor_set; - + return set_query; - + } + Json::Value Meta_Data::construct_flinng_descriptor(){ - - Json::Value tuple; std::shared_ptr test_aclient; std::string name="flinng_test_2060"; @@ -35,10 +34,10 @@ Json::Value Meta_Data::construct_flinng_descriptor(){ test_aclient.reset ( new VDMS::VDMSClient(get_server(), get_port())); VDMS::Response response =test_aclient->query(_fastwriter.write(tuple)); Json::Value result; - _reader.parse(response.json.c_str(), result); + _reader.parse(response.json.c_str(), result); Json::Value AddDesc; Json::Value Desc; - + Desc["set"] ="flinng_test_2060"; Desc["label"] ="Person"; Desc["_ref"]=1; @@ -47,9 +46,6 @@ Json::Value Meta_Data::construct_flinng_descriptor(){ AddDesc["AddDescriptor"] = Desc; tuple.append(AddDesc); return tuple; - - - } Json::Value Meta_Data::construct_descriptor(){ @@ -64,10 +60,10 @@ Json::Value Meta_Data::construct_descriptor(){ test_aclient.reset ( new VDMS::VDMSClient(get_server(), get_port())); VDMS::Response response =test_aclient->query(_fastwriter.write(tuple)); Json::Value result; - _reader.parse(response.json.c_str(), result); + _reader.parse(response.json.c_str(), result); Json::Value AddDesc; Json::Value Desc; - + Desc["set"] ="features_vectors_store1"; Desc["label"] ="Person"; Desc["_ref"]=1; @@ -76,49 +72,46 @@ Json::Value Meta_Data::construct_descriptor(){ AddDesc["AddDescriptor"] = Desc; tuple.append(AddDesc); return tuple; - - } Json::Value Meta_Data::construct_find_descriptor() { - Json::Value FindDesc; - Json::Value Desc; - Json::Value tuple; -// Desc["results"]["count"] = ""; - // Desc["constraints"]["id"][0] =">="; - // Desc["constraints"]["id"][1] =100; - Desc["results"]["list"][0] = "_distance"; - Desc["results"]["list"][1] = "id"; - Desc["set"]= "features_vectors_store1"; - Desc["k_neighbors"]=5; -// Desc["blob"] =true; - FindDesc["FindDescriptor"] = Desc; - tuple.append(FindDesc); - FindDesc.clear(); - Desc.clear(); -return tuple; + Json::Value FindDesc; + Json::Value Desc; + Json::Value tuple; + // Desc["results"]["count"] = ""; + // Desc["constraints"]["id"][0] =">="; + // Desc["constraints"]["id"][1] =100; + Desc["results"]["list"][0] = "_distance"; + Desc["results"]["list"][1] = "id"; + Desc["set"]= "features_vectors_store1"; + Desc["k_neighbors"]=5; + // Desc["blob"] =true; + FindDesc["FindDescriptor"] = Desc; + tuple.append(FindDesc); + FindDesc.clear(); + Desc.clear(); + return tuple; } Json::Value Meta_Data::construct_find_flinng_descriptor() { - Json::Value FindDesc; - Json::Value Desc; - Json::Value tuple; - Desc["results"]["list"][0] = "_distance"; - Desc["results"]["list"][1] = "id"; - Desc["set"]= "flinng_test_2060"; - Desc["k_neighbors"]=5; -// Desc["blob"] =true; - FindDesc["FindDescriptor"] = Desc; - tuple.append(FindDesc); - FindDesc.clear(); - Desc.clear(); -return tuple; + Json::Value FindDesc; + Json::Value Desc; + Json::Value tuple; + Desc["results"]["list"][0] = "_distance"; + Desc["results"]["list"][1] = "id"; + Desc["set"]= "flinng_test_2060"; + Desc["k_neighbors"]=5; + // Desc["blob"] =true; + FindDesc["FindDescriptor"] = Desc; + tuple.append(FindDesc); + FindDesc.clear(); + Desc.clear(); + return tuple; } - Json::Value Meta_Data::constuct_image(bool add_operation, Json::Value operations){ Json::Value image; @@ -154,28 +147,108 @@ Json::Value Meta_Data::constuct_image(bool add_operation, Json::Value operations add_video["AddVideo"]=video; tuple.append(add_video); return tuple; - } +} - Json::Value Meta_Data::construct_find_image(){ +Json::Value Meta_Data::construct_find_image(){ + Json::Value tuple; + + Json::Value cons; + cons["Name"][0] = "=="; + cons["Name"][1] = "sample-image"; + + Json::Value results; + results["blob"] = false; + results["list"][0] = "Name"; + results["list"][1] = "ID"; Json::Value image; - Json::Value find_image; - Json::Value tuple; - Json::Value result; - - image["constraints"] ["Name"][0] = "=="; - image["constraints"] ["Name"][1] = "sample-image"; image["_ref"]=1; - result["list"][0] = "Name"; - image["results"]=result; + image["constraints"] = cons; + image["results"]=results; + + Json::Value find_image; find_image["FindImage"]=image; + tuple.append(find_image); return tuple; - } -Json::Value Meta_Data::constuct_BB(bool with_image){ +} + +std::string* Meta_Data::read_blob(std::string& fname){ + std::string video; + std::ifstream video_file(fname, + std::ios::in | std::ios::binary | std::ios::ate); + + video.resize(video_file.tellg()); + video_file.seekg(0, std::ios::beg); + if( !video_file.read(&video[ 0 ], video.size())) + std::cout << "error" << std::endl; + std::string* bytes_str =new std::string(video); + // std::cout << *bytes_str < #include #include -#include +#include #include #include @@ -22,20 +22,24 @@ class Meta_Data{ public: std::shared_ptr _aclient; std::string _server_name="localhost"; - int _port =55557; - - Json::FastWriter _fastwriter; + int _port =55558; + + Json::FastWriter _fastwriter; Json::Reader _reader; - Json::Value _result; + Json::Value _result; Meta_Data (); - + Json::Value construct_add_query(int ref, bool const_on, bool experiation); Json::Value construct_add_area(int ref, bool const_on); Json::Value construct_add_connection(int ref1, int ref2, bool const_on); - Json::Value construct_find_entity(bool ,bool); + Json::Value construct_find_entity(bool ,bool ); Json::Value constuct_BB(bool); + Json::Value construct_Blob(); + Json::Value construct_updateBlob(); + Json::Value construct_findBlob(); + std::string* read_blob(std::string&); Json::Value constuct_image(bool =false, Json::Value operations={}); Json::Value constuct_video(bool =false); Json::Value construct_find_image(); @@ -46,9 +50,4 @@ class Meta_Data{ Json::Value construct_Flinng_Set(std::string&, int&); std::string get_server(){return _server_name;} int get_port() {return _port;} - - - - - }; diff --git a/utils/src/api_schema/api_schema.json b/utils/src/api_schema/api_schema.json index 26ea8347..cafcb948 100644 --- a/utils/src/api_schema/api_schema.json +++ b/utils/src/api_schema/api_schema.json @@ -59,7 +59,11 @@ { "$ref": "#/definitions/AddVideoTop" }, { "$ref": "#/definitions/UpdateVideoTop" }, { "$ref": "#/definitions/FindVideoTop" }, - { "$ref": "#/definitions/FindFramesTop" } + { "$ref": "#/definitions/FindFramesTop" }, + + { "$ref": "#/definitions/AddBlobTop" }, + { "$ref": "#/definitions/UpdateBlobTop" }, + { "$ref": "#/definitions/FindBlobTop" } ] }, "uniqueItems": false, @@ -451,6 +455,26 @@ "additionalProperties": false }, + "AddBlobTop": { + "properties": { + "AddBlob" : { "type": "object", "$ref": "#/definitions/AddBlob" } + }, + "additionalProperties": false + }, + "UpdateBlobTop": { + "properties": { + "UpdateBlob" : { "type": "object", "$ref": "#/definitions/UpdateBlob" } + }, + "additionalProperties": false + }, + + "FindBlobTop": { + "properties": { + "FindBlob" : { "type": "object", "$ref": "#/definitions/FindBlob" } + }, + "additionalProperties": false + }, + "AddVideoTop": { "properties": { "AddVideo" : { "type": "object", "$ref": "#/definitions/AddVideo" } @@ -458,6 +482,9 @@ "additionalProperties": false }, + + + "UpdateVideoTop": { "properties": { "UpdateVideo" : { "type": "object", "$ref": "#/definitions/UpdateVideo" } @@ -685,6 +712,41 @@ "additionalProperties": false }, + "AddBlob": { + "properties": { + "_ref": { "$ref": "#/definitions/refInt" }, + "from_server_file": { "type": "string"}, + "link": { "$ref": "#/definitions/blockLink" }, + "properties": { "type": "object" } + + }, + "additionalProperties": false + }, + + "UpdateBlob": { + "properties": { + "_ref": { "$ref": "#/definitions/refInt" }, + "properties": { "type": "object" }, + "remove_props": { "$ref": "#/definitions/stringArray" }, + "constraints": { "type": "object" } + }, + "additionalProperties": false + }, + + "FindBlob": { + "properties": { + "_ref": { "$ref": "#/definitions/refInt" }, + "link": { "$ref": "#/definitions/blockLink" }, + "constraints": { "type": "object" }, + "results": { "$ref": "#/definitions/blockResults" }, + "unique": { "type": "boolean" } + }, + + "additionalProperties": false + }, + + + "AddVideo": { "properties": { "_ref": { "$ref": "#/definitions/refInt" }, @@ -698,7 +760,7 @@ }, "additionalProperties": false }, - + "UpdateVideo": { "properties": { "_ref": { "$ref": "#/definitions/refInt" },