Merge pull request #11 from IntelLabs/master

Merge back master into develop

INSTALL.md

@@ -25,7 +25,7 @@
 * valijson
   * git clone https://github.com/tristanpenman/valijson.git
   * cd valijson
-  * cp include/* /usr/local/include
+  * cp -r include/* /usr/local/include (may need to run as sudo)
   * This is a headers-only library, no compilation/installation necessary
 
 * Persistent Memory Graph Database (PMGD)
@@ -38,14 +38,13 @@
 
 ### Requirement for Python Client
 
-    sudo apt-get install pip
-    pip install protobuf
+    sudo apt-get install python-pip
+    pip install protobuf (may need to run as sudo)
 
     Add VDMS Python module to PYPATH:
-    export PYTHONPATH="${PYTHONPATH}:<path_to_vdms>/python/vdms"
+    export PYTHONPATH="${PYTHONPATH}:<path_to_vdms>/client/python/vdms"
     # Example:
-    export PYTHONPATH="${PYTHONPATH}:/opt/intel/vdms/python/vdms"
-
+    export PYTHONPATH="${PYTHONPATH}:/opt/intel/vdms/client/python/vdms"
 
 ### Compilation
 
@@ -61,3 +60,20 @@ Flag | Explanation
 --timing    | Compiles server with chronos for internal timing.
 -jX         | Compiles in parallel, using X cores
 INTEL_PATH=path  | Path to the root folder containing pmgd and vcl. Default is "./" which is pmgd and vcl inside vdms folder. Example: scons INTEL_PATH=/opt/intel/
+
+### Running The VDMS Server
+
+The config-vdms.json file contains the configuration of the server.
+Some of the parameters include the TCP port that will be use for incoming
+connections, maximun number of simultaneous clients, and paths to the
+folders where data/metadata will be stored.
+
+**Note:** The folders must already exists in the filesystem.
+
+We provide a script (run_server.sh) that will create some default directories,
+corresponding the default values in the config-vdms.json.
+
+To run the server using the default directories and port, simply run:
+
+    sh run_server.sh
+

README.md

@@ -32,6 +32,15 @@ how it enables efficient vision analytics.
 Here is our [ATC HotStorage '17 Position Paper](https://www.usenix.org/system/files/conference/hotstorage17/hotstorage17-paper-gupta-cledat.pdf).
 Also, take a look at our [presentation](https://www.usenix.org/conference/hotstorage17/program/presentation/gupta-cledat).
 
+## Get Started
+
+To get started, take a look at the [INSTALL.md](INSTALL.md) file, where
+you will find instructions on how to install and run the server.
+
+Also, visit our [wiki](https://github.com/IntelLabs/vdms/wiki)
+to learn more about the VDMS API, and take a look at some of
+the examples/tutorials.
+
 ## Cite
 
 Feel free to include our work into your research!

docker/Dockerfile

@@ -0,0 +1,80 @@
+# Pull base image.
+FROM ubuntu:16.04
+
+RUN apt-get -qq update
+RUN apt-get -qq install -y python python-dev python-pip python-virtualenv
+RUN apt-get -qq install -y bzip2
+RUN apt-get install --no-install-recommends -y libjsoncpp-dev build-essential cmake git pkg-config libprotobuf-dev libleveldb-dev libsnappy-dev libhdf5-serial-dev protobuf-compiler libatlas-base-dev libboost-all-dev libgflags-dev libgoogle-glog-dev liblmdb-dev libgflags-dev libgoogle-glog-dev liblmdb-dev
+RUN apt-get install -y scons libjsoncpp-dev flex javacc libbison-dev openjdk-8-jdk
+RUN apt-get install -y autoconf automake libtool curl make g++ unzip
+RUN apt-get install -y cmake wget git zlib1g-dev libbz2-dev libssl-dev liblz4-dev mpich
+RUN apt-get install -y libjpeg8-dev libtiff5-dev libjasper-dev libpng12-dev libgtk-3-dev
+RUN apt-get install -y zlib1g-dev
+RUN apt-get install -y libbz2-dev
+RUN apt-get install -y libssl-dev
+RUN apt-get install -y liblz4-dev
+RUN apt-get install -y autoconf automake libtool curl make g++ unzip
+RUN apt-get install -y libopenmpi-dev
+RUN apt-get install -y libgtest-dev
+RUN apt-get install -y ed
+RUN apt-get install -y libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev
+RUN apt-get install -y libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libdc1394-22-dev
+RUN pip install scons numpy
+RUN apt-get remove -y libprotobuf-dev protobuf-compiler
+RUN cd / && git clone https://github.com/google/protobuf.git
+RUN cd protobuf && git checkout v3.3.0 && ./autogen.sh && ./configure --prefix=/usr/local && make -j $(cat /proc/cpuinfo | wc -l) && make install && ldconfig
+RUN rm -rf /protobuf
+
+# Blosc
+RUN cd / && git clone https://github.com/Blosc/c-blosc.git
+RUN cd c-blosc && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX='/usr' .. && cmake --build . && ctest && cmake --build . --target install
+RUN rm -rf /c-blosc
+
+# Zstd
+RUN cd / && wget https://github.com/facebook/zstd/archive/v1.1.0.tar.gz && tar xf v1.1.0.tar.gz
+RUN cd zstd-1.1.0 && make install PREFIX='/usr'
+RUN cd /usr/src/gtest && cmake . && make && mv libgtest* /usr/lib/
+
+# TileDB
+RUN cd / && wget https://github.com/TileDB-Inc/TileDB/archive/0.6.1.tar.gz && tar xf 0.6.1.tar.gz
+RUN cd TileDB-0.6.1 && mkdir build && cd build && cmake .. && make -j $(cat /proc/cpuinfo | wc -l) && make install
+RUN rm -rf /TileDB-0.6.1
+
+# Valijson
+RUN cd / && git clone https://github.com/tristanpenman/valijson.git
+RUN cd valijson && cp -r include/* /usr/local/include/
+
+# OpenCV
+RUN mkdir /opencv && cd /opencv && wget -O opencv.zip https://github.com/opencv/opencv/archive/3.3.0.zip
+RUN cd /opencv && unzip opencv.zip && cd opencv-3.3.0 && mkdir build && cd build && cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=/usr/local .. && make -j $(cat /proc/cpuinfo | wc -l) && make install && rm -rf /opencv
+
+# VCL install
+RUN cd / && wget https://github.com/intellabs/vcl/archive/v0.1.0.tar.gz && tar xf v0.1.0.tar.gz
+RUN mv vcl-0.1.0 vcl && cd vcl && scons -j16
+
+# PMGD install
+RUN cd / && wget https://github.com/intellabs/pmgd/archive/v1.0.0.tar.gz && tar xf v1.0.0.tar.gz
+RUN mv pmgd-1.0.0 pmgd && cd pmgd && make
+RUN rm -rf /*.tar.gz
+
+# VDMS install
+RUN cd / && wget https://github.com/intellabs/vdms/archive/v1.0.0.tar.gz && tar xf v1.0.0.tar.gz
+RUN mv vdms-1.0.0 vdms && cd vdms && mkdir db && scons -j16 INTEL_PATH=/
+
+RUN rm -rf /*.tar.gz
+RUN rm -rf /var/lib/apt/lists/* /root/.cache
+
+RUN echo '#!/bin/bash' > /start.sh
+RUN echo 'export LD_LIBRARY_PATH=/pmgd/lib:$(find /usr/local/lib/ / -type f -name "*.so" | xargs dirname | sort | uniq | tr "\n" ":")' >> /start.sh
+RUN echo 'cd /vdms' >> /start.sh
+RUN echo 'rm -rf db' >> /start.sh
+RUN echo 'mkdir db' >> /start.sh
+RUN echo 'mkdir db/images' >> /start.sh
+RUN echo 'mkdir db/images/pngs' >> /start.sh
+RUN echo 'mkdir db/images/jpgs' >> /start.sh
+RUN echo 'mkdir db/images/tiledb' >> /start.sh
+RUN echo 'mkdir db/images/tiledb/tdb' >> /start.sh
+RUN echo './vdms' >> /start.sh
+RUN chmod 755 /start.sh
+ENTRYPOINT "/start.sh"
+

docker/README.md

@@ -0,0 +1,24 @@
+# VDMS Docker
+
+Here are the instructions to build a docker image of the VDMS Server.
+
+The build command is: 
+
+    Dockerfile | docker build -t vdms:latest .
+
+If you are under a proxy, use:
+
+    Dockerfile | docker build -t vdms:latest --build-arg=http_proxy --build-arg=https_proxy .
+
+To run the docker image as a container, include the --net flag.
+This flag is needed as the server will be accepting connections on the default VDMS port (55555).
+
+    // Run the image interactively 
+    docker run -it --net=host vdms:latest
+
+    // or
+
+    // Run the image and deattach it from your bash 
+    docker run -d --net=host vdms:latest
+
+

run_server.sh

@@ -1,9 +1,8 @@
-scons -j123
 rm log.log
 rm -r db
 mkdir db
-mkdir db/native_format
-mkdir db/native_format/pngs
-mkdir db/native_format/jpgs
-mkdir db/native_format/descriptors
+mkdir db/images
+mkdir db/images/pngs
+mkdir db/images/jpgs
+
 ./vdms 2> log.log

src/PMGDQuery.cc

@@ -151,7 +151,7 @@ void PMGDQuery::set_value(const std::string& key, const PMGDProp& p,
             break;
 
         case PMGDProp::TimeType:
-            prop[key] = p.string_value();
+            prop[key] = p.time_value();
             break;
 
         case PMGDProp::FloatType:
@@ -192,7 +192,7 @@ void PMGDQuery::set_property(PMGDProp* p, const std::string& key,
         case Json::objectValue:
             if (val.isMember("_date")) {
                 p->set_type(PMGDProp::TimeType);
-                p->set_string_value(val["_date"].asString());
+                p->set_time_value(val["_date"].asString());
             }
             else if (val.isMember("_blob")) {
                 // the blob value is read and stored as a string

src/PMGDQueryHandler.cc

@@ -63,15 +63,10 @@ std::vector<PMGDCmdResponses>
     std::vector<PMGDCmdResponses> responses(num_groups);
 
     assert(_tx == NULL);
+    _dblock->lock();
     for (const auto cmd : cmds) {
         PMGDCmdResponse *response = new PMGDCmdResponse();
-        try {
-            if (process_query(cmd, response) < 0) {
-                error_cleanup(responses, response);
-                break;  // Goto cleanup site.
-            }
-        }
-        catch (Exception e) {
+        if (process_query(cmd, response) < 0) {
             error_cleanup(responses, response);
             break;  // Goto cleanup site.
         }
@@ -90,6 +85,7 @@ std::vector<PMGDCmdResponses>
         _tx = NULL;
     }
 
+    _dblock->unlock();
     return responses;
 }
 
@@ -125,7 +121,6 @@ int PMGDQueryHandler::process_query(const PMGDCmd *cmd,
         switch (code) {
             case PMGDCmd::TxBegin:
             {
-                _dblock->lock();
 
                 // TODO: Needs to distinguish transaction parameters like RO/RW
                 _tx = new Transaction(*_db, Transaction::ReadWrite);
@@ -135,15 +130,11 @@ int PMGDQueryHandler::process_query(const PMGDCmd *cmd,
             case PMGDCmd::TxCommit:
             {
                 _tx->commit();
-                _dblock->unlock();
                 set_response(response, protobufs::TX, PMGDCmdResponse::Success);
                 break;
             }
             case PMGDCmd::TxAbort:
             {
-                delete _tx;
-                _tx = NULL;
-                _dblock->unlock();
                 set_response(response, protobufs::TX, PMGDCmdResponse::Abort,
                                 "Abort called");
                 retval = -1;
@@ -161,12 +152,9 @@ int PMGDQueryHandler::process_query(const PMGDCmd *cmd,
         }
     }
     catch (Exception e) {
-        delete _tx;
-        _tx = NULL;
-        _dblock->unlock();
         set_response(response, PMGDCmdResponse::Exception,
                         e.name + std::string(": ") +  e.msg);
-        throw e;
+        retval = -1;
     }
 
     return retval;

tests/AddAndFind_query.json

@@ -10,7 +10,7 @@
               "Email":"[email protected]",
               "Age": 86,
               "Birthday":{"_date":"Sat Oct 1 17:59:24 PDT 1936"},
-              "name":"Mark",
+              "Name":"Mark",
                "fv": {"_blob":"Raghed----ghjhsglfhwa"}
           }
       }
@@ -27,7 +27,7 @@
               "Age": 80,
               "Birthday":{"_date":"Sat Oct 1 17:59:24 PDT 1936"},
               "fv":{"_blob" :"Ahmad----ghjhsglfhwa"},
-               "name": "Kai"
+               "Name": "Kai"
 
           }
       }
@@ -56,14 +56,14 @@
           "properties": {
               "Email":"[email protected]",
               "Age": 56,
-              "Birthday":{"_date":"Sat Oct 1 17:59:24 PDT 1936"},
+              "Birthday":{"_date":"Sat Oct 7 17:59:24 PDT 1946"},
                "Study": false,
                "avg": 32.4,
-                "name": "Mathias",
+                "Name": "Mathias",
                 "fv": {"_blob":"----ghjhsglfhwa"}
           },
           "constraints":
-          { "name": ["==", "Mathias"]
+          { "Name": ["==", "Mathias"]
           }
       }
 
@@ -80,7 +80,7 @@
                "Birthday":{"_date":"Sat Oct 1 17:59:24 PDT 1936"},
                "Study": true,
                "avg": 21.6,
-               "name": "Alin",
+               "Name": "Alin",
                 "fv": {"_blob":"----ghjhsglfhwa"}
 
           }
@@ -112,7 +112,7 @@
        },
 
        "results": {
-       "list": ["name"]
+       "list": ["Name"]
      }
   }
 },
@@ -129,7 +129,7 @@
        },
 
        "results": {
-       "list": ["name"]
+       "list": ["Name"]
      }
   }
 },
@@ -141,7 +141,7 @@
          "Age":  [">", 0, "<=", 180 ]
       },
             "results": {
-            "list":["name","Age"]
+            "list":["Name","Age"]
          }
   }
 },
@@ -159,7 +159,7 @@
         },
        "results": {
 
-       "list":["Age","name", "Email"],
+       "list":["Age","Name", "Email"],
        "limit" :3
 
        }
@@ -177,7 +177,7 @@
         },
        "results": {
 
-       "list":["Age","name", "Email"],
+       "list":["Age","Name", "Email"],
        "sort":"Email"
       }
   }
@@ -190,7 +190,7 @@
          "Age":  [">", 0, "<=", 100 ]
       },
         "results": {
-            "list":["name","Age","Email"],
+            "list":["Name","Age","Email"],
             "sort" :"Age"
           }
     }
@@ -205,7 +205,7 @@
         },
        "results": {
 
-       "list":["Age","name", "Email"],
+       "list":["Age","Name", "Email", "Birthday"],
        "limit":2
 
 

tests/json_queries.cc

@@ -244,6 +244,13 @@ TEST(QueryHandler, AddAndFind){
         if (cmd =="FindEntity")
             out_query_num++;
 
+        if (j == 12) { // Last FindEntiyu
+            EXPECT_EQ(query["FindEntity"]["entities"][0]["Birthday"].asString(),
+              "1946-10-07T17:59:24-07:00");
+
+            EXPECT_EQ(query["FindEntity"]["entities"][1]["Birthday"].asString(),
+              "1936-10-01T17:59:24-07:00");
+        }
         if ( query[cmd]["status"] == 0)
             success++;