Added mechanism to allow retries around Requests Lib exceptions

Often exceptions caused by the Request Library are either:
* Impacting, in which case they are consistent
* Non-Impacting, in which case they are sporadic and cause false
  failures

In tests that do API testing using the HTTP plugin, there is a use
case for retrying Request Library exceptions as some of these
failures can be caused by issues unrelated to tests and the system
under test. This PR provides a mechanism to configure how many of
these errors a test runner is willing to ignore.

Change-Id: I2d15095ad435b151df56f0a23f068f0fae662c37

cafe/plugins/http/cafe/engine/http/client.py

@@ -14,11 +14,15 @@
 import requests
 import six
 from time import time
+from warnings import warn
 
 from cafe.common.reporting import cclogging
 from cafe.engine.clients.base import BaseClient
+from cafe.engine.http.config import HTTPPluginConfig
 
 from requests.packages import urllib3
+from requests.exceptions import (
+    ConnectionError, HTTPError, Timeout, TooManyRedirects)
 urllib3.disable_warnings()
 
 
@@ -154,13 +158,37 @@ class BaseHTTPClient(BaseClient):
     _log = cclogging.getLogger(__name__)
 
     def __init__(self):
+        self.__config = HTTPPluginConfig()
         super(BaseHTTPClient, self).__init__()
 
     @_inject_exception(_exception_handlers)
     @_log_transaction(log=_log)
     def request(self, method, url, **kwargs):
         """ Performs <method> HTTP request to <url> using the requests lib"""
-        return requests.request(method, url, **kwargs)
+        retries = self.__config.retries_on_requests_exceptions
+
+        # We always allow one attempt, retries are configured via EngineConfig
+        allowed_attempts = 1 + retries
+
+        # Offsetting xrange range by one to allow proper reporting of which
+        # attempt we are on.
+        for attempt in six.moves.xrange(1, allowed_attempts + 1):
+            try:
+                return requests.request(method, url, **kwargs)
+            except(ConnectionError, HTTPError, Timeout, TooManyRedirects) as e:
+                if retries:
+                    warning_string = (
+                        'Request Lib Error: Attempt {attempt} of '
+                        '{allowed_attempts}\n'.format(
+                            attempt=attempt,
+                            allowed_attempts=allowed_attempts))
+                    warn(warning_string)
+                    warn(e)
+                    warn('\n')
+                    self._log.critical(warning_string)
+                    self._log.exception(e)
+        else:
+            raise e
 
     def put(self, url, **kwargs):
         """ HTTP PUT request """

cafe/plugins/http/cafe/engine/http/config.py

@@ -0,0 +1,46 @@
+# Copyright 2016 Rackspace
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from cafe.engine.models.data_interfaces import (
+    ConfigSectionInterface, _get_path_from_env)
+
+
+class HTTPPluginConfig(ConfigSectionInterface):
+
+    SECTION_NAME = 'PLUGIN.HTTP'
+
+    def __init__(self, config_file_path=None):
+        """Initialization of the HTTP Plugin Engine config section."""
+        config_file_path = config_file_path or _get_path_from_env(
+            'CAFE_ENGINE_CONFIG_FILE_PATH')
+        super(HTTPPluginConfig, self).__init__(
+            config_file_path=config_file_path)
+
+    @property
+    def retries_on_requests_exceptions(self):
+        """
+        Number of retries allowed on Requests library exceptions.
+
+        This is provided to allow retries on exceptions from the Requests
+        library. Specifically this will allow retries on errors resulting
+        from the following exceptions: ConnectionError, HTTPError, Timeout,
+        and TooManyRedirects.
+        """
+        try:
+            return int(self.get('retries_on_requests_exceptions', 0))
+        except ValueError:
+            raw_data = self.get_raw('retries_on_requests_exceptions')
+            raise ValueError(
+                '{} is not a valid input for the '
+                '\"retries_on_requests_exceptions\" congfiguration value. '
+                'Value must be an integer'.format(raw_data))