Make number of attempts for RWLock configurable.

Edit multiple add test in python to add slowdown between threads
to avoid connection refusal due to queueing.

config-vdms.json

@@ -6,6 +6,10 @@
     "port": 55555, // Default is 55555
     "max_simultaneous_clients": 20, // Default is 500
 
+    // Tune the number of maximum attempts when acquiring the
+    // reader writer lock for metadata changes.
+    "max_lock_attempts": 10,
+
     // Database paths
     "pmgd_path": "db/graph", // This will be an IP address in the future
     "png_path": "db/images/pngs/",

src/PMGDQueryHandler.cc

@@ -49,12 +49,14 @@ void PMGDQueryHandler::init()
 {
     std::string dbname = VDMSConfig::instance()
                         ->get_string_value("pmgd_path", "default_pmgd");
+    unsigned attempts = VDMSConfig::instance()
+                        ->get_int_value("max_lock_attempts", RWLock::MAX_ATTEMPTS);
 
     // Create a db
     _db = new PMGD::Graph(dbname.c_str(), PMGD::Graph::Create);
 
     // Create the query handler here assuming database is valid now.
-    _dblock = new RWLock();
+    _dblock = new RWLock(attempts);
 }
 
 void PMGDQueryHandler::destroy()
@@ -77,10 +79,20 @@ std::vector<PMGDCmdResponses>
 
     // Assuming one query handler handles one TX at a time.
     _readonly = readonly;
-    if (_readonly)
-        _dblock->read_lock();
-    else
-        _dblock->write_lock();
+    try {
+        if (_readonly)
+            _dblock->read_lock();
+        else
+            _dblock->write_lock();
+    }
+    catch (Exception e) {
+        PMGDCmdResponses &resp_v = responses[0];
+        PMGDCmdResponse *response = new PMGDCmdResponse();
+        set_response(response, PMGDCmdResponse::Exception,
+                        e.name + std::string(": ") +  e.msg);
+        resp_v.push_back(response);
+        return responses;
+    }
 
     for (const auto cmd : cmds) {
         PMGDCmdResponse *response = new PMGDCmdResponse();

src/RWLock.h

@@ -52,15 +52,15 @@ namespace VDMS {
         // Backoff variables.
         // *** Tune experimentally
         static const size_t MIN_BACKOFF_DELAY = 100000;
-        static const size_t MAX_BACKOFF_DELAY = 50000000;
-        static const unsigned MAX_ATTEMPTS      = 10;
+        static const size_t MAX_BACKOFF_DELAY = 500000000;
 
         uint16_t xadd(volatile uint16_t &m, uint16_t v)
             { return ::xadd<uint16_t>(m, v); }
         void atomic_and(volatile uint16_t &m, uint16_t v)
             { ::atomic_and<uint16_t>(m, v); }
 
         volatile uint16_t _rw_lock;
+        unsigned _max_attempts;
 
         // Ideas from here: https://geidav.wordpress.com/tag/exponential-back-off
         void backoff(size_t &cur_max_delay)
@@ -80,7 +80,9 @@ namespace VDMS {
         }
 
     public:
-        RWLock() : _rw_lock(0) {}
+        static const unsigned MAX_ATTEMPTS      = 10;
+
+        RWLock(unsigned max_attempts) : _rw_lock(0), _max_attempts(max_attempts) {}
 
         void read_lock()
         {

tests/python/TestEntities.py

@@ -98,13 +98,13 @@ def findEntity(self, thID):
                                     ["threadid"], thID)
 
     def test_runMultipleAdds(self):
-
         simultaneous = 1000;
         thread_arr = []
         for i in range(1,simultaneous):
             thread_add = Thread(target=self.addEntity,args=(i,) )
             thread_add.start()
             thread_arr.append(thread_add)
+            time.sleep(0.002)
 
         for i in range(1,simultaneous):
             thread_find = Thread(target=self.findEntity,args=(i,) )

tests/python/config-tests.json

@@ -5,6 +5,10 @@
     // Network
     "port": 55557, // Default is 55555
 
+    // Tune the number of maximum attempts when acquiring the
+    // reader writer lock for metadata changes.
+    "max_lock_attempts": 100,
+
     // Database paths
     "pmgd_path": "db/test-graph",
     "png_path": "db/images/pngs/",