diff --git a/lib/puppet/provider/neutron.rb b/lib/puppet/provider/neutron.rb index c640f3cc8..dbcf3e357 100644 --- a/lib/puppet/provider/neutron.rb +++ b/lib/puppet/provider/neutron.rb @@ -75,21 +75,40 @@ def self.auth_neutron(*args) if q.key?('nova_region_name') authenv[:OS_REGION_NAME] = q['nova_region_name'] end - begin - withenv authenv do - neutron(args) - end - rescue Exception => e - if (e.message =~ /\[Errno 111\] Connection refused/) or - (e.message =~ /\(HTTP 400\)/) - sleep 10 + rv = nil + timeout = 120 + end_time = Time.now.to_i + timeout + loop do + begin withenv authenv do - neutron(args) + rv = neutron(args) end - else - raise(e) + break + rescue Puppet::ExecutionFailure => e + if ! e.message =~ /(\(HTTP\s+400\))| + (400-\{\'message\'\:\s+\'\'\})| + (\[Errno 111\]\s+Connection\s+refused)| + (503\s+Service\s+Unavailable)| + (504\s+Gateway\s+Time-out)| + (\:\s+Maximum\s+attempts\s+reached)| + (Unauthorized\:\s+bad\s+credentials)| + (Max\s+retries\s+exceeded)/ + raise(e) + end + current_time = Time.now.to_i + if current_time > end_time + break + else + wait = end_time - current_time + Puppet::debug("Non-fatal error: \"#{e.message}\"") + notice("Neutron API not avalaible. Wait up to #{wait} sec.") + end + sleep(2) + # Note(xarses): Don't remove, we know that there is one of the + # Recoverable erros above, So we will retry a few more times end end + return rv end def auth_neutron(*args) diff --git a/spec/unit/provider/neutron_spec.rb b/spec/unit/provider/neutron_spec.rb index 35a3ff381..ca5d294e4 100644 --- a/spec/unit/provider/neutron_spec.rb +++ b/spec/unit/provider/neutron_spec.rb @@ -105,12 +105,18 @@ def klass end ['[Errno 111] Connection refused', - '(HTTP 400)'].reverse.each do |valid_message| + '400-{\'message\': \'\'}', + '(HTTP 400)', + '503 Service Unavailable', + '504 Gateway Time-out', + 'Maximum attempts reached', + 'Unauthorized: bad credentials', + 'Max retries exceeded'].reverse.each do |valid_message| it "should retry when neutron cli returns with error #{valid_message}" do klass.expects(:get_neutron_credentials).with().returns({}) - klass.expects(:sleep).with(10).returns(nil) + klass.expects(:sleep).with(2).returns(nil) klass.expects(:neutron).twice.with(['test_retries']).raises( - Exception, valid_message).then.returns('') + Puppet::ExecutionFailure, valid_message).then.returns('') klass.auth_neutron('test_retries') end end