Glance provider should retry once when service is not ready

Just b/c the glance service has started correctly does not
mean the service is actually running. Often, the service
is not operational and attempts to use the glance_image provider
results in connection errors.

This commit inserts code that allows the glance provider to
sleep for 10 seconds and perform a single retry.

The template for the code and tests was borrowed from keystone.

Change-Id: I565f1befea6ec3d2347f8e29c9aa05d5a312a827

lib/puppet/provider/glance.rb

@@ -58,6 +58,13 @@ class Puppet::Provider::Glance < Puppet::Provider
     @glance_hash ||= build_glance_hash
   end
 
+  def self.reset
+    @glance_hash        = nil
+    @glance_file        = nil
+    @glance_credentials = nil
+    @auth_endpoint      = nil
+  end
+
   def glance_hash
     self.class.glance_hash
   end
@@ -67,8 +74,12 @@ class Puppet::Provider::Glance < Puppet::Provider
       g = glance_credentials
       glance('-T', g['admin_tenant_name'], '-I', g['admin_user'], '-K', g['admin_password'], '-N', auth_endpoint, args)
     rescue Exception => e
-      # Will probably add conditions later
-      raise(e)
+      if (e.message =~ /\[Errno 111\] Connection refused/) or (e.message =~ /\(HTTP 400\)/) or (e.message =~ /HTTP Unable to establish connection/)
+        sleep 10
+        glance('-T', g['admin_tenant_name'], '-I', g['admin_user'], '-K', g['admin_password'], '-N', auth_endpoint, args)
+      else
+        raise(e)
+      end
     end
   end
 

spec/unit/provider/glance_spec.rb

@@ -0,0 +1,59 @@
+require 'puppet'
+require 'spec_helper'
+require 'puppet/provider/glance'
+require 'tempfile'
+
+
+klass = Puppet::Provider::Glance
+
+describe Puppet::Provider::Glance do
+
+  after :each do
+    klass.reset
+  end
+
+  describe 'when retrieving the auth credentials' do
+
+    it 'should fail if the glance config file does not have the expected contents' do
+      mock = {}
+      Puppet::Util::IniConfig::File.expects(:new).returns(mock)
+      mock.expects(:read).with('/etc/glance/glance-api.conf')
+      expect do
+        klass.glance_credentials
+      end.to raise_error(Puppet::Error, /does not contain all required sections/)
+    end
+
+  describe 'when testing glance connection retries' do
+
+      ['[Errno 111] Connection refused', '(HTTP 400)', 'HTTP Unable to establish connection'].reverse.each do |valid_message|
+        it "should retry when glance is not ready with error #{valid_message}" do
+          mock = {'keystone_authtoken' =>
+            {
+               'auth_host'         => '127.0.0.1',
+               'auth_port'         => '35357',
+               'auth_protocol'     => 'http',
+               'admin_tenant_name' => 'foo',
+               'admin_user'        => 'user',
+               'admin_password'    => 'pass'
+            }
+          }
+          Puppet::Util::IniConfig::File.expects(:new).returns(mock)
+          mock.expects(:read).with('/etc/glance/glance-api.conf')
+          klass.expects(:sleep).with(10).returns(nil)
+          klass.expects(:glance).twice.with(
+            '-T',
+            'foo',
+            '-I',
+            'user',
+            '-K',
+            'pass',
+            '-N',
+            'http://127.0.0.1:35357/v2.0/',
+            ['test_retries']
+          ).raises(Exception, valid_message).then.returns('')
+          klass.auth_glance('test_retries')
+        end
+      end
+    end
+  end
+end