Loop for 600 seconds during user creation

Since the keystone server may not be immediately available when we
run init-keystone, loop for 600 seconds while attempting to create
the admin user.

Change-Id: I24f064cc80df150cdc49978fcea3793f45e181b5

doc/source/usage.rst

@@ -13,6 +13,11 @@ normal authentication by creating the admin and service tenants, the admin
 and Member roles, the admin user, configure certificates and finally
 registers the initial identity endpoint.
 
+ .. note::
+
+    init-keystone will wait up to 10 minutes for a Keystone service to be
+    running on the specified host.
+
 For example::
 
     init-keystone -o 192.0.2.1 -t unset -e admin@example.com -p unset -u root

os_cloud_config/cmd/init_keystone.py

@@ -27,6 +27,9 @@ def parse_args():
     roles, the admin user, configure certificates and finally register the
     initial identity endpoint, after which Keystone may be used with normal
     authentication.
+
+    This command will wait up to 10 minutes for a Keystone service to be
+    running on the specified host.
     """)
     parser = argparse.ArgumentParser(
         formatter_class=argparse.RawDescriptionHelpFormatter,

os_cloud_config/keystone.py

@@ -14,7 +14,9 @@
 
 import logging
 import subprocess
+import time
 
+from keystoneclient.openstack.common.apiclient import exceptions
 import keystoneclient.v2_0.client as ksclient
 
 LOG = logging.getLogger(__name__)
@@ -97,8 +99,14 @@ def _create_roles(keystone):
 
     :param keystone: keystone v2 client
     """
-    LOG.debug('Creating admin role.')
-    keystone.roles.create('admin')
+    for count in range(60):
+        try:
+            LOG.debug('Creating admin role, try %d.' % count)
+            keystone.roles.create('admin')
+            break
+        except (exceptions.ConnectionRefused, exceptions.ServiceUnavailable):
+            LOG.debug('Unable to create, sleeping for 10 seconds.')
+            time.sleep(10)
     LOG.debug('Creating Member role.')
     keystone.roles.create('Member')
 

os_cloud_config/tests/test_keystone.py

@@ -14,6 +14,7 @@
 
 import mock
 
+from keystoneclient.openstack.common.apiclient import exceptions
 from os_cloud_config import keystone
 from os_cloud_config.tests import base
 
@@ -105,6 +106,18 @@ class KeystoneTest(base.TestCase):
         keystone._create_endpoint(self.client, '192.0.0.3', 'regionTwo', None)
         self.assert_endpoint('192.0.0.3', region='regionTwo')
 
+    @mock.patch('time.sleep')
+    def test_create_roles_retry(self, sleep):
+        self._patch_client()
+        side_effect = (exceptions.ConnectionRefused,
+                       exceptions.ServiceUnavailable, mock.DEFAULT,
+                       mock.DEFAULT)
+        self.client.roles.create.side_effect = side_effect
+        keystone._create_roles(self.client)
+        sleep.assert_has_calls([mock.call(10), mock.call(10)])
+        self.client.roles.create.assert_has_calls(
+            [mock.call('admin'), mock.call('Member')])
+
     @mock.patch('os_cloud_config.keystone.ksclient.Client')
     def test_create_admin_client(self, client):
         self.assertEqual(