Merge "Adding ability to retry clients calls"

sahara/config.py

@@ -127,6 +127,7 @@ def list_opts():
     from sahara.service.edp import job_utils
     from sahara.service import periodic
     from sahara.utils import cluster_progress_ops as cpo
+    from sahara.utils.openstack import base
     from sahara.utils.openstack import heat
     from sahara.utils.openstack import neutron
     from sahara.utils.openstack import nova
@@ -151,7 +152,8 @@ def list_opts():
                          periodic.periodic_opts,
                          proxy.opts,
                          cpo.event_log_opts,
-                         wsgi.wsgi_opts)),
+                         wsgi.wsgi_opts,
+                         base.opts)),
         (poll_utils.timeouts.name,
          itertools.chain(poll_utils.timeouts_opts)),
         (api.conductor_group.name,
@@ -167,7 +169,9 @@ def list_opts():
         (swift.swift_group.name,
          itertools.chain(swift.opts)),
         (keystone.keystone_group.name,
-         itertools.chain(keystone.ssl_opts))
+         itertools.chain(keystone.ssl_opts)),
+        (base.retries.name,
+         itertools.chain(base.opts))
     ]
 
 

sahara/exceptions.py

@@ -359,3 +359,14 @@ class UpdateFailedException(SaharaException):
             self.message = message
         self.message = self.message % value
         super(UpdateFailedException, self).__init__()
+
+
+class MaxRetriesExceeded(SaharaException):
+    code = "MAX_RETRIES_EXCEEDED"
+    message = _("Operation %(operation)s wasn't executed correctly after "
+                "%(attempts)d attempts")
+
+    def __init__(self, attempts, operation):
+        self.message = self.message % {'operation': operation,
+                                       'attempts': attempts}
+        super(MaxRetriesExceeded, self).__init__()

sahara/tests/unit/utils/openstack/test_base.py

@@ -13,6 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+from cinderclient import exceptions as cinder_exc
+from heatclient import exc as heat_exc
+from keystoneclient import exceptions as keystone_exc
+import mock
+from neutronclient.common import exceptions as neutron_exc
+from novaclient import exceptions as nova_exc
+
+from sahara import exceptions as sahara_exc
 from sahara.tests.unit import base as testbase
 from sahara.utils.openstack import base
 
@@ -82,3 +91,105 @@ class AuthUrlTest(testbase.SaharaTestCase):
         _assert("https://127.0.0.1:8080/v3/")
         _assert("https://127.0.0.1:8080/v42")
         _assert("https://127.0.0.1:8080/v42/")
+
+
+class ExecuteWithRetryTest(testbase.SaharaTestCase):
+
+    def setUp(self):
+        super(ExecuteWithRetryTest, self).setUp()
+        self.fake_client_call = mock.MagicMock()
+        self.fake_client_call.__name__ = 'fake_client_call'
+        self.override_config('retries_number', 2, 'retries')
+
+    @mock.patch('sahara.context.sleep')
+    def _check_error_without_retry(self, error, code, m_sleep):
+        self.fake_client_call.side_effect = error(code)
+
+        self.assertRaises(error, base.execute_with_retries,
+                          self.fake_client_call)
+        self.assertEqual(1, self.fake_client_call.call_count)
+        self.fake_client_call.reset_mock()
+
+    @mock.patch('sahara.context.sleep')
+    def _check_error_with_retry(self, error, code, m_sleep):
+        self.fake_client_call.side_effect = error(code)
+
+        self.assertRaises(sahara_exc.MaxRetriesExceeded,
+                          base.execute_with_retries, self.fake_client_call)
+        self.assertEqual(3, self.fake_client_call.call_count)
+        self.fake_client_call.reset_mock()
+
+    def test_novaclient_calls_without_retry(self):
+        # check that following errors will not be retried
+        self._check_error_without_retry(nova_exc.BadRequest, 400)
+        self._check_error_without_retry(nova_exc.Unauthorized, 401)
+        self._check_error_without_retry(nova_exc.Forbidden, 403)
+        self._check_error_without_retry(nova_exc.NotFound, 404)
+        self._check_error_without_retry(nova_exc.MethodNotAllowed, 405)
+        self._check_error_without_retry(nova_exc.Conflict, 409)
+        self._check_error_without_retry(nova_exc.HTTPNotImplemented, 501)
+
+    def test_novaclient_calls_with_retry(self):
+        # check that following errors will be retried
+        self._check_error_with_retry(nova_exc.OverLimit, 413)
+        self._check_error_with_retry(nova_exc.RateLimit, 429)
+
+    def test_cinderclient_calls_without_retry(self):
+        # check that following errors will not be retried
+        self._check_error_without_retry(cinder_exc.BadRequest, 400)
+        self._check_error_without_retry(cinder_exc.Unauthorized, 401)
+        self._check_error_without_retry(cinder_exc.Forbidden, 403)
+        self._check_error_without_retry(cinder_exc.NotFound, 404)
+        self._check_error_without_retry(nova_exc.HTTPNotImplemented, 501)
+
+    def test_cinderclient_calls_with_retry(self):
+        # check that following error will be retried
+        self._check_error_with_retry(cinder_exc.OverLimit, 413)
+
+    def test_neutronclient_calls_without_retry(self):
+        # check that following errors will not be retried
+        self._check_error_without_retry(neutron_exc.BadRequest, 400)
+        self._check_error_without_retry(neutron_exc.Forbidden, 403)
+        self._check_error_without_retry(neutron_exc.NotFound, 404)
+        self._check_error_without_retry(neutron_exc.Conflict, 409)
+
+    def test_neutronclient_calls_with_retry(self):
+        # check that following errors will be retried
+        self._check_error_with_retry(neutron_exc.InternalServerError, 500)
+        self._check_error_with_retry(neutron_exc.ServiceUnavailable, 503)
+
+    def test_heatclient_calls_without_retry(self):
+        # check that following errors will not be retried
+        self._check_error_without_retry(heat_exc.HTTPBadRequest, 400)
+        self._check_error_without_retry(heat_exc.HTTPUnauthorized, 401)
+        self._check_error_without_retry(heat_exc.HTTPForbidden, 403)
+        self._check_error_without_retry(heat_exc.HTTPNotFound, 404)
+        self._check_error_without_retry(heat_exc.HTTPMethodNotAllowed, 405)
+        self._check_error_without_retry(heat_exc.HTTPConflict, 409)
+        self._check_error_without_retry(heat_exc.HTTPUnsupported, 415)
+        self._check_error_without_retry(heat_exc.HTTPNotImplemented, 501)
+
+    def test_heatclient_calls_with_retry(self):
+        # check that following errors will be retried
+        self._check_error_with_retry(heat_exc.HTTPInternalServerError, 500)
+        self._check_error_with_retry(heat_exc.HTTPBadGateway, 502)
+        self._check_error_with_retry(heat_exc.HTTPServiceUnavailable, 503)
+
+    def test_keystoneclient_calls_without_retry(self):
+        # check that following errors will not be retried
+        self._check_error_without_retry(keystone_exc.BadRequest, 400)
+        self._check_error_without_retry(keystone_exc.Unauthorized, 401)
+        self._check_error_without_retry(keystone_exc.Forbidden, 403)
+        self._check_error_without_retry(keystone_exc.NotFound, 404)
+        self._check_error_without_retry(keystone_exc.MethodNotAllowed, 405)
+        self._check_error_without_retry(keystone_exc.Conflict, 409)
+        self._check_error_without_retry(keystone_exc.UnsupportedMediaType, 415)
+        self._check_error_without_retry(keystone_exc.HttpNotImplemented, 501)
+
+    def test_keystoneclient_calls_with_retry(self):
+        # check that following errors will be retried
+        self._check_error_with_retry(keystone_exc.RequestTimeout, 408)
+        self._check_error_with_retry(keystone_exc.InternalServerError, 500)
+        self._check_error_with_retry(keystone_exc.BadGateway, 502)
+        self._check_error_with_retry(keystone_exc.ServiceUnavailable, 503)
+        self._check_error_with_retry(keystone_exc.GatewayTimeout, 504)

sahara/utils/openstack/base.py

@@ -14,14 +14,37 @@
 # limitations under the License.
 
 from oslo_config import cfg
+from oslo_log import log as logging
 from oslo_serialization import jsonutils as json
 from six.moves.urllib import parse as urlparse
 
 from sahara import context
 from sahara import exceptions as ex
 from sahara.i18n import _
+from sahara.i18n import _LE
+from sahara.i18n import _LW
+
+LOG = logging.getLogger(__name__)
+
+# List of the errors, that can be retried
+ERRORS_TO_RETRY = [408, 413, 429, 500, 502, 503, 504]
+
+opts = [
+    cfg.IntOpt('retries_number',
+               default=5,
+               help='Number of times to retry the request to client before '
+                    'failing'),
+    cfg.IntOpt('retry_after',
+               default=10,
+               help='Time between the retries to client (in seconds).')
+]
+
+retries = cfg.OptGroup(name='retries',
+                       title='OpenStack clients calls retries')
 
 CONF = cfg.CONF
+CONF.register_group(retries)
+CONF.register_opts(opts, group=retries)
 
 
 def url_for(service_catalog, service_type, admin=False, endpoint_type=None):
@@ -84,3 +107,31 @@ def retrieve_auth_url():
     version = 'v3' if CONF.use_identity_api_v3 else 'v2.0'
 
     return "%s://%s:%s/%s/" % (info.scheme, info.hostname, info.port, version)
+
+
+def execute_with_retries(method, *args, **kwargs):
+    attempts = CONF.retries.retries_number + 1
+    while attempts > 0:
+        try:
+            return method(*args, **kwargs)
+        except Exception as e:
+            error_code = getattr(e, 'http_status', None) or getattr(
+                e, 'status_code', None) or getattr(e, 'code', None)
+            if error_code in ERRORS_TO_RETRY:
+                LOG.warning(_LW('Occasional error occured during "{method}" '
+                                'execution: {error_msg} ({error_code}). '
+                                'Operation will be retried.').format(
+                            method=method.__name__,
+                            error_msg=e,
+                            error_code=error_code))
+                attempts -= 1
+                retry_after = getattr(e, 'retry_after', 0)
+                context.sleep(max(retry_after, CONF.retries.retry_after))
+            else:
+                LOG.error(_LE('Permanent error occured during "{method}" '
+                              'execution: {error_msg}.').format(
+                          method=method.__name__,
+                          error_msg=e))
+                raise e
+    else:
+        raise ex.MaxRetriesExceeded(attempts, method.__name__)