From b17cd2b6ab1c1c5c437af91a3596eee807c96588 Mon Sep 17 00:00:00 2001 From: Adit Sarfaty Date: Mon, 1 Jan 2018 13:27:35 +0200 Subject: [PATCH] NSX rate limit support In case of too many requests in a short period of time, the NSX will return response 429. In this case (if configured) the nsxlib client will retry sending the request. This option is controlled by a new parameter in the nsxlib config rate_limit_retry which is enabled by default. Change-Id: I20fca36d553e1e74da61292342a87247b53b5d13 --- vmware_nsxlib/tests/unit/v3/test_utils.py | 13 +++++++++++++ vmware_nsxlib/v3/__init__.py | 3 ++- vmware_nsxlib/v3/client.py | 21 ++++++++++++++++++++- vmware_nsxlib/v3/config.py | 6 +++++- vmware_nsxlib/v3/exceptions.py | 4 ++++ vmware_nsxlib/v3/utils.py | 10 ++++++++++ 6 files changed, 54 insertions(+), 3 deletions(-) diff --git a/vmware_nsxlib/tests/unit/v3/test_utils.py b/vmware_nsxlib/tests/unit/v3/test_utils.py index 04a1f774..831c33dd 100644 --- a/vmware_nsxlib/tests/unit/v3/test_utils.py +++ b/vmware_nsxlib/tests/unit/v3/test_utils.py @@ -252,6 +252,19 @@ class TestNsxV3Utils(nsxlib_testcase.NsxClientTestCase): self.assertRaises(exceptions.NsxLibInvalidInput, func_to_fail, 99) self.assertEqual(max_retries, total_count['val']) + def test_retry_random(self): + max_retries = 5 + total_count = {'val': 0} + + @utils.retry_random_upon_exception(exceptions.NsxLibInvalidInput, + max_attempts=max_retries) + def func_to_fail(x): + total_count['val'] = total_count['val'] + 1 + raise exceptions.NsxLibInvalidInput(error_message='foo') + + self.assertRaises(exceptions.NsxLibInvalidInput, func_to_fail, 99) + self.assertEqual(max_retries, total_count['val']) + @mock.patch.object(utils, '_update_max_tags') @mock.patch.object(utils, '_update_tag_length') @mock.patch.object(utils, '_update_resource_length') diff --git a/vmware_nsxlib/v3/__init__.py b/vmware_nsxlib/v3/__init__.py index dc8912c6..aa2f982a 100644 --- a/vmware_nsxlib/v3/__init__.py +++ b/vmware_nsxlib/v3/__init__.py @@ -53,7 +53,8 @@ class NsxLibBase(object): self.cluster, nsx_api_managers=self.nsxlib_config.nsx_api_managers, max_attempts=self.nsxlib_config.max_attempts, - url_path_base=self.client_url_prefix) + url_path_base=self.client_url_prefix, + rate_limit_retry=self.nsxlib_config.rate_limit_retry) self.general_apis = utils.NsxLibApiBase( self.client, self.nsxlib_config) diff --git a/vmware_nsxlib/v3/client.py b/vmware_nsxlib/v3/client.py index 0855e2e3..89110866 100644 --- a/vmware_nsxlib/v3/client.py +++ b/vmware_nsxlib/v3/client.py @@ -38,7 +38,8 @@ def http_error_to_exception(status_code, error_code): requests.codes.INTERNAL_SERVER_ERROR: {'99': exceptions.ClientCertificateNotTrusted}, requests.codes.FORBIDDEN: - {'98': exceptions.BadXSRFToken}} + {'98': exceptions.BadXSRFToken}, + requests.codes.TOO_MANY_REQUESTS: exceptions.TooManyRequests} if status_code in errors: if isinstance(errors[status_code], dict): @@ -245,6 +246,7 @@ class NSX3Client(JSONRESTClient): default_headers=None, nsx_api_managers=None, max_attempts=utils.DEFAULT_MAX_ATTEMPTS, + rate_limit_retry=True, client_obj=None, url_path_base=NSX_V1_API_PREFIX): @@ -252,9 +254,11 @@ class NSX3Client(JSONRESTClient): if client_obj: self.nsx_api_managers = client_obj.nsx_api_managers or [] self.max_attempts = client_obj.max_attempts + self.rate_limit_retry = client_obj.rate_limit_retry else: self.nsx_api_managers = nsx_api_managers or [] self.max_attempts = max_attempts + self.rate_limit_retry = rate_limit_retry url_prefix = url_prefix or url_path_base if url_prefix and url_path_base not in url_prefix: @@ -278,3 +282,18 @@ class NSX3Client(JSONRESTClient): operation=operation, details=result_msg, error_code=error_code) + + def _rest_call(self, url, **kwargs): + if self.rate_limit_retry: + # If too many requests are handled by the nsx at the same time, + # error "429: Too Many Requests" will be returned. + # the client is expected to retry after a random 400-600 milli, + # and later exponentially until 5 seconds wait + @utils.retry_random_upon_exception( + exceptions.TooManyRequests, + max_attempts=self.max_attempts) + def _rest_call_with_retry(self, url, **kwargs): + return super(NSX3Client, self)._rest_call(url, **kwargs) + return _rest_call_with_retry(self, url, **kwargs) + else: + return super(NSX3Client, self)._rest_call(url, **kwargs) diff --git a/vmware_nsxlib/v3/config.py b/vmware_nsxlib/v3/config.py index 7d520768..9cb6efb9 100644 --- a/vmware_nsxlib/v3/config.py +++ b/vmware_nsxlib/v3/config.py @@ -72,6 +72,8 @@ class NsxLibConfig(object): X-Allow-Overwrite:true will be added to all the requests, to allow admin user to update/ delete all entries. + :param rate_limit_retry: If True, the client will retry requests failed on + "Too many requests" error """ def __init__(self, @@ -94,7 +96,8 @@ class NsxLibConfig(object): dns_nameservers=None, dns_domain='openstacklocal', dhcp_profile_uuid=None, - allow_overwrite_header=False): + allow_overwrite_header=False, + rate_limit_retry=True): self.nsx_api_managers = nsx_api_managers self._username = username @@ -115,6 +118,7 @@ class NsxLibConfig(object): self.dns_nameservers = dns_nameservers or [] self.dns_domain = dns_domain self.allow_overwrite_header = allow_overwrite_header + self.rate_limit_retry = rate_limit_retry if dhcp_profile_uuid: # this is deprecated, and never used. diff --git a/vmware_nsxlib/v3/exceptions.py b/vmware_nsxlib/v3/exceptions.py index b4013eda..3aec9edf 100644 --- a/vmware_nsxlib/v3/exceptions.py +++ b/vmware_nsxlib/v3/exceptions.py @@ -101,6 +101,10 @@ class StaleRevision(ManagerError): pass +class TooManyRequests(ManagerError): + pass + + class ClientCertificateNotTrusted(ManagerError): message = _("Certificate not trusted") diff --git a/vmware_nsxlib/v3/utils.py b/vmware_nsxlib/v3/utils.py index ae144f77..ab0f41e4 100644 --- a/vmware_nsxlib/v3/utils.py +++ b/vmware_nsxlib/v3/utils.py @@ -165,6 +165,16 @@ def retry_upon_exception(exc, delay=0.5, max_delay=2, before=_log_before_retry, after=_log_after_retry) +def retry_random_upon_exception(exc, delay=0.5, max_delay=5, + max_attempts=DEFAULT_MAX_ATTEMPTS): + return tenacity.retry(reraise=True, + retry=tenacity.retry_if_exception_type(exc), + wait=tenacity.wait_random_exponential( + multiplier=delay, max=max_delay), + stop=tenacity.stop_after_attempt(max_attempts), + before=_log_before_retry, after=_log_after_retry) + + def list_match(list1, list2): # Check if list1 and list2 have identical elements, but relaxed on # dict elements where list1's dict element can be a subset of list2's