NSX rate limit support

In case of too many requests in a short period of time, the NSX will
return response 429.
In this case (if configured) the nsxlib client will retry sending the request.

This option is controlled by a new parameter in the nsxlib config
rate_limit_retry which is enabled by default.

Change-Id: I20fca36d553e1e74da61292342a87247b53b5d13
This commit is contained in:
Adit Sarfaty 2018-01-01 13:27:35 +02:00
parent 48c8136448
commit b17cd2b6ab
6 changed files with 54 additions and 3 deletions

View File

@ -252,6 +252,19 @@ class TestNsxV3Utils(nsxlib_testcase.NsxClientTestCase):
self.assertRaises(exceptions.NsxLibInvalidInput, func_to_fail, 99)
self.assertEqual(max_retries, total_count['val'])
def test_retry_random(self):
max_retries = 5
total_count = {'val': 0}
@utils.retry_random_upon_exception(exceptions.NsxLibInvalidInput,
max_attempts=max_retries)
def func_to_fail(x):
total_count['val'] = total_count['val'] + 1
raise exceptions.NsxLibInvalidInput(error_message='foo')
self.assertRaises(exceptions.NsxLibInvalidInput, func_to_fail, 99)
self.assertEqual(max_retries, total_count['val'])
@mock.patch.object(utils, '_update_max_tags')
@mock.patch.object(utils, '_update_tag_length')
@mock.patch.object(utils, '_update_resource_length')

View File

@ -53,7 +53,8 @@ class NsxLibBase(object):
self.cluster,
nsx_api_managers=self.nsxlib_config.nsx_api_managers,
max_attempts=self.nsxlib_config.max_attempts,
url_path_base=self.client_url_prefix)
url_path_base=self.client_url_prefix,
rate_limit_retry=self.nsxlib_config.rate_limit_retry)
self.general_apis = utils.NsxLibApiBase(
self.client, self.nsxlib_config)

View File

@ -38,7 +38,8 @@ def http_error_to_exception(status_code, error_code):
requests.codes.INTERNAL_SERVER_ERROR:
{'99': exceptions.ClientCertificateNotTrusted},
requests.codes.FORBIDDEN:
{'98': exceptions.BadXSRFToken}}
{'98': exceptions.BadXSRFToken},
requests.codes.TOO_MANY_REQUESTS: exceptions.TooManyRequests}
if status_code in errors:
if isinstance(errors[status_code], dict):
@ -245,6 +246,7 @@ class NSX3Client(JSONRESTClient):
default_headers=None,
nsx_api_managers=None,
max_attempts=utils.DEFAULT_MAX_ATTEMPTS,
rate_limit_retry=True,
client_obj=None,
url_path_base=NSX_V1_API_PREFIX):
@ -252,9 +254,11 @@ class NSX3Client(JSONRESTClient):
if client_obj:
self.nsx_api_managers = client_obj.nsx_api_managers or []
self.max_attempts = client_obj.max_attempts
self.rate_limit_retry = client_obj.rate_limit_retry
else:
self.nsx_api_managers = nsx_api_managers or []
self.max_attempts = max_attempts
self.rate_limit_retry = rate_limit_retry
url_prefix = url_prefix or url_path_base
if url_prefix and url_path_base not in url_prefix:
@ -278,3 +282,18 @@ class NSX3Client(JSONRESTClient):
operation=operation,
details=result_msg,
error_code=error_code)
def _rest_call(self, url, **kwargs):
if self.rate_limit_retry:
# If too many requests are handled by the nsx at the same time,
# error "429: Too Many Requests" will be returned.
# the client is expected to retry after a random 400-600 milli,
# and later exponentially until 5 seconds wait
@utils.retry_random_upon_exception(
exceptions.TooManyRequests,
max_attempts=self.max_attempts)
def _rest_call_with_retry(self, url, **kwargs):
return super(NSX3Client, self)._rest_call(url, **kwargs)
return _rest_call_with_retry(self, url, **kwargs)
else:
return super(NSX3Client, self)._rest_call(url, **kwargs)

View File

@ -72,6 +72,8 @@ class NsxLibConfig(object):
X-Allow-Overwrite:true will be added to all
the requests, to allow admin user to update/
delete all entries.
:param rate_limit_retry: If True, the client will retry requests failed on
"Too many requests" error
"""
def __init__(self,
@ -94,7 +96,8 @@ class NsxLibConfig(object):
dns_nameservers=None,
dns_domain='openstacklocal',
dhcp_profile_uuid=None,
allow_overwrite_header=False):
allow_overwrite_header=False,
rate_limit_retry=True):
self.nsx_api_managers = nsx_api_managers
self._username = username
@ -115,6 +118,7 @@ class NsxLibConfig(object):
self.dns_nameservers = dns_nameservers or []
self.dns_domain = dns_domain
self.allow_overwrite_header = allow_overwrite_header
self.rate_limit_retry = rate_limit_retry
if dhcp_profile_uuid:
# this is deprecated, and never used.

View File

@ -101,6 +101,10 @@ class StaleRevision(ManagerError):
pass
class TooManyRequests(ManagerError):
pass
class ClientCertificateNotTrusted(ManagerError):
message = _("Certificate not trusted")

View File

@ -165,6 +165,16 @@ def retry_upon_exception(exc, delay=0.5, max_delay=2,
before=_log_before_retry, after=_log_after_retry)
def retry_random_upon_exception(exc, delay=0.5, max_delay=5,
max_attempts=DEFAULT_MAX_ATTEMPTS):
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_exception_type(exc),
wait=tenacity.wait_random_exponential(
multiplier=delay, max=max_delay),
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
def list_match(list1, list2):
# Check if list1 and list2 have identical elements, but relaxed on
# dict elements where list1's dict element can be a subset of list2's