fuel_upgrade: add retries for getting token

After starting a new Keystone container, the first attempt to get a token
fails. Unfortunately, the logs keep silent and say nothing. As a workaround,
we can just increase retries number.

Change-Id: Ib1bba120614d1964e838c289489f1877074c324d
Closes-Bug: #1403798

fuel_upgrade_system/fuel_upgrade/fuel_upgrade/clients/keystone_client.py

@@ -16,8 +16,14 @@
 
 import json
 import logging
+import socket
+
 import requests
 
+from fuel_upgrade import errors
+from fuel_upgrade import utils
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -44,9 +50,20 @@ class KeystoneClient(object):
         :returns: :class:`requests.Session` object
         """
         session = requests.Session()
-        token = self.get_token()
-        if token:
+
+        # NOTE(ikalnitsky):
+        #   After starting a new Keystone container, the first attempt to
+        #   get a token fails. Unfortunately, the logs keep silent and say
+        #   nothing. As a workaround, we can just increase retries number.
+        #
+        # See https://bugs.launchpad.net/fuel/+bug/1399144 for details.
+        try:
+            token = utils.wait_for_true(self.get_token, timeout=10)
             session.headers.update({'X-Auth-Token': token})
+        except errors.TimeoutError:
+            logger.exception(
+                'Cannot retrieve an auth token - an unauthenticated '
+                'request will be performed.')
 
         return session
 
@@ -75,7 +92,9 @@ class KeystoneClient(object):
 
             return (isinstance(resp, dict) and
                     resp.get('access', {}).get('token', {}).get('id'))
-        except (ValueError, requests.exceptions.RequestException) as exc:
+        except (ValueError,
+                socket.timeout,
+                requests.exceptions.RequestException) as exc:
             logger.debug('Cannot authenticate in keystone: %s', exc)
 
         return None

fuel_upgrade_system/fuel_upgrade/fuel_upgrade/tests/test_keystone_client.py

@@ -46,5 +46,19 @@ class TestKeystoneClient(base.BaseTestCase):
     @mock.patch('fuel_upgrade.clients.keystone_client.requests.post',
                 side_effect=requests.exceptions.HTTPError(''))
     def test_does_not_fail_without_keystone(self, _, __):
-        self.keystone.request
-        self.assertEqual(self.keystone.get_token(), None)
+        with mock.patch('fuel_upgrade.utils.time.time') as time:
+            # Unfortunately, in Python 2.6 the itertools.count() doesn't
+            # support the step argument, so we need to implement our own
+            # bicycle.
+            def timestamp(start, step):
+                x = start
+                while True:
+                    yield x
+                    x += step
+
+            # We need such infinity generator, because time.time() is used
+            # by our loggers, so we can't predict how often it will be called.
+            time.side_effect = timestamp(0, 15)
+
+            self.keystone.request
+            self.assertEqual(self.keystone.get_token(), None)