Fix status messages checks for neutron provider

W/o this change many of 'bad' status messages returned from Neutron API wouldn't be noticed by neutron provider The suggested fix is to add more status checks and notifications (non-fatal ones) about Neutron API inaccessibility. Closes-bug: #1384097 Change-Id: I227cea60e2b2d2ec14a96ef8aeb43dfad263ff3b Signed-off-by: Bogdan Dobrelya <[email protected]> (cherry picked from commit d64ec6fa45907b06ed4cadce9c4abac32d6723fe)
xbezdick · Jan 20, 2015 · 678d078 · 678d078
1 parent c3dc520
commit 678d078
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 14 deletions.
diff --git a/lib/puppet/provider/neutron.rb b/lib/puppet/provider/neutron.rb
@@ -75,21 +75,40 @@ def self.auth_neutron(*args)
     if q.key?('nova_region_name')
       authenv[:OS_REGION_NAME] = q['nova_region_name']
     end
-    begin
-      withenv authenv do
-        neutron(args)
-      end
-    rescue Exception => e
-      if (e.message =~ /\[Errno 111\] Connection refused/) or
-          (e.message =~ /\(HTTP 400\)/)
-        sleep 10
+    rv = nil
+    timeout = 120
+    end_time = Time.now.to_i + timeout
+    loop do
+      begin
         withenv authenv do
-          neutron(args)
+          rv = neutron(args)
         end
-      else
-       raise(e)
+        break
+      rescue Puppet::ExecutionFailure => e
+        if ! e.message =~ /(\(HTTP\s+400\))|
+              (400-\{\'message\'\:\s+\'\'\})|
+              (\[Errno 111\]\s+Connection\s+refused)|
+              (503\s+Service\s+Unavailable)|
+              (504\s+Gateway\s+Time-out)|
+              (\:\s+Maximum\s+attempts\s+reached)|
+              (Unauthorized\:\s+bad\s+credentials)|
+              (Max\s+retries\s+exceeded)/
+          raise(e)
+        end
+        current_time = Time.now.to_i
+        if current_time > end_time
+          break
+        else
+          wait = end_time - current_time
+          Puppet::debug("Non-fatal error: \"#{e.message}\"")
+          notice("Neutron API not avalaible. Wait up to #{wait} sec.")
+        end
+        sleep(2)
+        # Note(xarses): Don't remove, we know that there is one of the
+        # Recoverable erros above, So we will retry a few more times
       end
     end
+    return rv
   end
 
   def auth_neutron(*args)

diff --git a/spec/unit/provider/neutron_spec.rb b/spec/unit/provider/neutron_spec.rb
@@ -105,12 +105,18 @@ def klass
     end
 
     ['[Errno 111] Connection refused',
-     '(HTTP 400)'].reverse.each do |valid_message|
+     '400-{\'message\': \'\'}',
+     '(HTTP 400)',
+     '503 Service Unavailable',
+     '504 Gateway Time-out',
+     'Maximum attempts reached',
+     'Unauthorized: bad credentials',
+     'Max retries exceeded'].reverse.each do |valid_message|
       it "should retry when neutron cli returns with error #{valid_message}" do
         klass.expects(:get_neutron_credentials).with().returns({})
-        klass.expects(:sleep).with(10).returns(nil)
+        klass.expects(:sleep).with(2).returns(nil)
         klass.expects(:neutron).twice.with(['test_retries']).raises(
-          Exception, valid_message).then.returns('')
+          Puppet::ExecutionFailure, valid_message).then.returns('')
         klass.auth_neutron('test_retries')
       end
     end