Add exponential backoff to ratelimited requests

If we're ratelimited on a request, we should retry the request but back
off significantly.  Previously we're not really handling 429 any
different than other requests which may lead to the requests being
retried too quickly.

Change-Id: I3832332abdfd7daaf373dc0924fec268f159d774
Related-Bug: #1889372
Related-bug: #1889122

tripleo_common/image/exception.py

@@ -18,6 +18,10 @@ class ImageBuilderException(Exception):
     pass
 
 
+class ImageRateLimitedException(Exception):
+    """Rate Limited request"""
+
+
 class ImageSpecificationException(Exception):
     pass
 

tripleo_common/image/image_uploader.py

@@ -37,6 +37,7 @@ from oslo_log import log as logging
 from tripleo_common.actions import ansible
 from tripleo_common.image.base import BaseImageManager
 from tripleo_common.image.exception import ImageNotFoundException
+from tripleo_common.image.exception import ImageRateLimitedException
 from tripleo_common.image.exception import ImageUploaderException
 from tripleo_common.image.exception import ImageUploaderThreadException
 from tripleo_common.image import image_export
@@ -244,6 +245,10 @@ class RegistrySessionHelper(object):
                         )
                         session.reauthenticate(**session.auth_args)
 
+        if status_code == 429:
+            raise ImageRateLimitedException('Rate Limited while requesting '
+                                            '{}'.format(request.url))
+
         request.raise_for_status()
 
     @staticmethod
@@ -296,6 +301,14 @@ class RegistrySessionHelper(object):
         return request_response
 
     @staticmethod
+    @tenacity.retry(  # Retry up to 5 times with longer time for rate limit
+        reraise=True,
+        retry=tenacity.retry_if_exception_type(
+            ImageRateLimitedException
+        ),
+        wait=tenacity.wait_random_exponential(multiplier=1.5, max=60),
+        stop=tenacity.stop_after_attempt(5)
+    )
     def _action(action, request_session, *args, **kwargs):
         """ Perform a session action and retry if auth fails
 
@@ -1781,12 +1794,13 @@ class PythonImageUploader(BaseImageUploader):
         return r.headers['Location']
 
     @classmethod
-    @tenacity.retry(  # Retry up to 5 times with jittered exponential backoff
+    @tenacity.retry(  # Retry up to 5 times with longer time
         reraise=True,
         retry=tenacity.retry_if_exception_type(
-            requests.exceptions.RequestException
+            (requests.exceptions.RequestException,
+             ImageRateLimitedException)
         ),
-        wait=tenacity.wait_random_exponential(multiplier=1, max=10),
+        wait=tenacity.wait_random_exponential(multiplier=1.5, max=60),
         stop=tenacity.stop_after_attempt(5)
     )
     def _layer_stream_registry(cls, digest, source_url, calc_digest,

tripleo_common/tests/image/test_image_uploader.py

@@ -28,6 +28,7 @@ import zlib
 
 from oslo_concurrency import processutils
 from tripleo_common.image.exception import ImageNotFoundException
+from tripleo_common.image.exception import ImageRateLimitedException
 from tripleo_common.image.exception import ImageUploaderException
 from tripleo_common.image import image_uploader
 from tripleo_common.tests import base
@@ -76,6 +77,23 @@ class TestRegistrySessionHelper(base.TestCase):
         session_reauth_mock.assert_called_once_with()
         raise_for_status_mock.assert_called_once()
 
+    def test_check_status_ratelimit(self):
+        session = mock.Mock()
+        session_reauth_mock = mock.Mock()
+        session.headers = {}
+        session.auth_args = {}
+        session.reauthenticate = session_reauth_mock
+        raise_for_status_mock = mock.Mock()
+        request = mock.Mock()
+        request.headers = {'www-authenticate': 'foo'}
+        request.raise_for_status = raise_for_status_mock
+        request.status_code = 429
+
+        self.assertRaises(ImageRateLimitedException,
+                          image_uploader.RegistrySessionHelper.check_status,
+                          session,
+                          request)
+
     def test_check_redirect_trusted_no_redirect(self):
         get_mock = mock.Mock()
         session = mock.Mock()