Merge pull request #78 from IntelLabs/release-2.0.0

Release 2.0.0

INSTALL.md

@@ -29,22 +29,20 @@
   * This is a headers-only library, no compilation/installation necessary
 
 * Persistent Memory Graph Database (PMGD)
-  * Download version 1.0.0 from: https://github.com/IntelLabs/pmgd/releases
+  * Download version 2.0.0 from: https://github.com/IntelLabs/pmgd/releases
   * Follow installation instructions
 
 * Visual Compute Library
-  * Download version 0.1.0 from: https://github.com/IntelLabs/vcl/releases
+  * Download version 1.0.0 from: https://github.com/IntelLabs/vcl/releases
   * Follow installation instructions
 
-### Requirement for Python Client
+### Python Client Module
 
-    sudo apt-get install python-pip
-    pip install protobuf (may need to run as sudo)
+VDMS offers the Python Client Module through the pip package manager, 
+and it is compatible with Python 2.7+ and 3.3+. 
+pip (or pip2 and pip3) will automatically install dependencies (protobuf).
 
-    Add VDMS Python module to PYPATH:
-    export PYTHONPATH="${PYTHONPATH}:<path_to_vdms>/client/python/vdms"
-    # Example:
-    export PYTHONPATH="${PYTHONPATH}:/opt/intel/vdms/client/python/vdms"
+    pip install vdms 
 
 ### Compilation
 
@@ -56,8 +54,8 @@
 
 Flag | Explanation
 ------------ | -------------
---no-server | Compiles client libraries (C++/Python) only. (will not compile neither server not tests)
---timing    | Compiles server with chronos for internal timing.
+--no-server | Compiles client libraries (C++/Python) only. (will not compile neither server nor tests)
+--timing    | Compiles server with chronos for internal timing, experimental.
 -jX         | Compiles in parallel, using X cores
 INTEL_PATH=path  | Path to the root folder containing pmgd and vcl. Default is "./" which is pmgd and vcl inside vdms folder. Example: scons INTEL_PATH=/opt/intel/
 
@@ -68,8 +66,6 @@ Some of the parameters include the TCP port that will be use for incoming
 connections, maximun number of simultaneous clients, and paths to the
 folders where data/metadata will be stored.
 
-**Note:** The folders must already exists in the filesystem.
-
 We provide a script (run_server.sh) that will create some default directories,
 corresponding the default values in the config-vdms.json.
 

README.md

@@ -29,9 +29,6 @@ propose a storage architecture designed for efficient visual data access
 that exploits next generation hardware and give preliminary results showing
 how it enables efficient vision analytics.
 
-Here is our [ATC HotStorage '17 Position Paper](https://www.usenix.org/system/files/conference/hotstorage17/hotstorage17-paper-gupta-cledat.pdf).
-Also, take a look at our [presentation](https://www.usenix.org/conference/hotstorage17/program/presentation/gupta-cledat).
-
 ## Get Started
 
 To get started, take a look at the [INSTALL.md](INSTALL.md) file, where
@@ -41,16 +38,9 @@ Also, visit our [wiki](https://github.com/IntelLabs/vdms/wiki)
 to learn more about the VDMS API, and take a look at some of
 the examples/tutorials.
 
-## Cite
-
-Feel free to include our work into your research!
+## Academic Papers
 
-    @inproceedings {203374,
-    author = {Vishakha Gupta-Cledat and Luis Remis and Christina R Strong},
-    title = {Addressing the Dark Side of Vision Research: Storage},
-    booktitle = {9th {USENIX} Workshop on Hot Topics in Storage and File Systems (HotStorage 17)},
-    year = {2017},
-    address = {Santa Clara, CA},
-    url = {https://www.usenix.org/conference/hotstorage17/program/presentation/gupta-cledat},
-    publisher = {{USENIX} Association},
-    }
+Conference | Links, Cite | Description
+------------ | ------------- | -------------
+Learning Systems @ NIPS 2018 | [Paper](https://export.arxiv.org/abs/1810.11832), [Cite](https://dblp.uni-trier.de/rec/bibtex/journals/corr/abs-1810-11832) | Systems for Machine Learning [Workshop](http://learningsys.org/nips18/cfp.html) @ NIPS
+HotStorage @ ATC 2017 | [Paper](https://www.usenix.org/conference/hotstorage17/program/presentation/gupta-cledat), [Presentation](https://www.usenix.org/conference/hotstorage17/program/presentation/gupta-cledat), [Cite](https://www.usenix.org/biblio/export/bibtex/203374)| Positioning Paper at USENIX ATC 2017 Workshop

SConstruct

@@ -18,7 +18,7 @@ def buildServer(env):
               os.path.join(env['INTEL_PATH'], 'vcl/src'),
              ],
     LIBS = [ 'pmgd', 'pmgd-util',
-             'jsoncpp', 'protobuf',
+             'jsoncpp', 'protobuf', 'tbb',
              'vdms-utils', 'vcl', 'pthread',
            ],
     LIBPATH = ['/usr/local/lib/', 'utils/',
@@ -35,24 +35,28 @@ def buildServer(env):
                   'src/QueryHandler.cc',
                   'src/QueryMessage.cc',
                   'src/CommunicationManager.cc',
+                  'src/ExceptionsCommand.cc',
                   'src/PMGDQuery.cc',
                   'src/SearchExpression.cc',
                   'src/PMGDIterators.cc',
                   'src/PMGDQueryHandler.cc',
                   'src/RSCommand.cc',
                   'src/ImageCommand.cc',
-                  'src/ExceptionsCommand.cc',
+                  'src/DescriptorsManager.cc',
+                  'src/DescriptorsCommand.cc',
+                  'src/BoundingBoxCommand.cc',
+                  'src/VideoCommand.cc',
                 ]
 
-  vdms = env.Program('vdms', vdms_server_files)
+  env.Program('vdms', vdms_server_files)
 
 # Set INTEL_PATH. First check arguments, then enviroment, then default
 if ARGUMENTS.get('INTEL_PATH', '') != '':
-  intel_path = ARGUMENTS.get("INTEL_PATH", '')
+    intel_path = ARGUMENTS.get("INTEL_PATH", '')
 elif os.environ.get('INTEL_PATH', '') != '':
-  intel_path = os.environ.get('INTEL_PATH', '')
+    intel_path = os.environ.get('INTEL_PATH', '')
 else:
-  intel_path = os.getcwd()
+    intel_path = os.getcwd()
 
 # Enviroment use by all the builds
 env = Environment(CXXFLAGS="-std=c++11 -O3")
@@ -63,6 +67,6 @@ SConscript(os.path.join('utils', 'SConscript'), exports=['env'])
 SConscript(os.path.join('client/cpp','SConscript'), exports=['env'])
 
 if GetOption('no-server'):
-  buildServer(env)
-  # Build tests only if server is built
-  SConscript(os.path.join('tests', 'SConscript'), exports=['env'])
+    buildServer(env)
+    # Build tests only if server is built
+    SConscript(os.path.join('tests', 'SConscript'), exports=['env'])

client/cpp/VDMSClient.cc

@@ -38,15 +38,15 @@ const string VDMSClient::query(const string &json)
     protobufs::queryMessage cmd;
     cmd.set_json(json);
 
-	std::basic_string<uint8_t> msg(cmd.ByteSize(),0);
-	cmd.SerializeToArray((void*)msg.data(), msg.length());
-	_conn.send_message(msg.data(), msg.length());
+    std::basic_string<uint8_t> msg(cmd.ByteSize(),0);
+    cmd.SerializeToArray((void*)msg.data(), msg.length());
+    _conn.send_message(msg.data(), msg.length());
 
     // Wait now for response
     // TODO: Perhaps add an asynchronous version too.
-	msg = _conn.recv_message();
-	protobufs::queryMessage resp;
-	resp.ParseFromArray((const void*)msg.data(), msg.length());
+    msg = _conn.recv_message();
+    protobufs::queryMessage resp;
+    resp.ParseFromArray((const void*)msg.data(), msg.length());
 
     return resp.json();
 }
@@ -56,19 +56,20 @@ const string VDMSClient::query(const string &json, const vector<string *> blobs)
     protobufs::queryMessage cmd;
     cmd.set_json(json);
 
-    for (auto it : blobs) {
+    for (auto& it : blobs) {
         string *blob = cmd.add_blobs();
-        blob = it;
+        *blob = *it;
     }
-	std::basic_string<uint8_t> msg(cmd.ByteSize(),0);
-	cmd.SerializeToArray((void*)msg.data(), msg.length());
-	_conn.send_message(msg.data(), msg.length());
+
+    std::basic_string<uint8_t> msg(cmd.ByteSize(),0);
+    cmd.SerializeToArray((void*)msg.data(), msg.length());
+    _conn.send_message(msg.data(), msg.length());
 
     // Wait now for response
     // TODO: Perhaps add an asynchronous version too.
-	msg = _conn.recv_message();
-	protobufs::queryMessage resp;
-	resp.ParseFromArray((const void*)msg.data(), msg.length());
+    msg = _conn.recv_message();
+    protobufs::queryMessage resp;
+    resp.ParseFromArray((const void*)msg.data(), msg.length());
 
     return resp.json();
 }

client/python/vdms/__init__.py

@@ -0,0 +1,4 @@
+name = "vdms"
+
+from .vdms import *
+

client/python/vdms/vdms.py

@@ -35,14 +35,18 @@
 import json
 
 # VDMS Protobuf import (autogenerated)
-import queryMessage_pb2
+from . import queryMessage_pb2
 
-class VDMS(object):
+class vdms(object):
 
     def __init__(self):
         self.dataNotUsed = []
         self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.connected = False
+        self.last_response = ''
+
+    def __del__(self):
+        self.conn.close()
 
     def connect(self, host='localhost', port=55555):
         self.conn.connect((host, port))
@@ -81,12 +85,10 @@ def query(self, query, img_array = []):
         # end = time.time()
         # print "ATcomm[ms]:" + str((end - start)*1000)
 
-        # time.sleep(1)
-
         # Recieve response
         recv_len = self.conn.recv(4)
         recv_len = struct.unpack('@I', recv_len)[0]
-        response = ''
+        response = b''
         while len(response) < recv_len:
             packet = self.conn.recv(recv_len - len(response))
             if not packet:
@@ -100,14 +102,12 @@ def query(self, query, img_array = []):
         for b in querRes.blobs:
             img_array.append(b)
 
-        return (querRes.json, img_array)
+        self.last_response = json.loads(querRes.json)
+
+        return (self.last_response, img_array)
 
-# Aux functions for printing JSON queries/responses
-def aux_print_json(data):
-    # Check the query type
-    if isinstance(data, str): # assumes string
-        json_obj = json.loads(data)
-    else:
-        json_obj = data
+    def get_last_response(self):
+        return self.last_response
 
-    print json.dumps(json_obj, indent=4, sort_keys=False)
+    def get_last_response_str(self):
+        return json.dumps(self.last_response, indent=4, sort_keys=False)

config-vdms.json

@@ -3,15 +3,11 @@
 // Sets database paths and other parameters
 {
     // Network
-    "port": 55555, // Default is 55555
-    "max_simultaneous_clients": 20, // Default is 500
+    "port": 55555,
+    "max_simultaneous_clients": 100,
 
     // Database paths
-    "pmgd_path": "db/graph", // This will be an IP address in the future
-    "png_path": "db/images/pngs/",
-    "jpg_path": "db/images/jpgs/",
-    "tdb_path": "db/images/tiledb/tdb/",
+    "db_root_path": "db",
 
-    "support_info": "[email protected]",
-    "support_phone": "1-800-A-TEAM"
+    "more-info": "github.com/IntelLabs/vdms"
 }