Add retry to execution commands

There are quite a few instances where the "exec_command" function fails
due unrelated issues. This change adds a retry with a backoff. The exec
function will now retry 3 times in the event of an exception with a cool
down of 2 seconds. The play context for retries will also be set if
undefined making it possible to recover from transient UNREACHABLE
errors.

Change-Id: Ic7c7af49e1c5eba21216b33814b92e276e3fba3e
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

connection/ssh.py

@@ -244,8 +244,11 @@ DOCUMENTATION = '''
         version_added: "2.6"
 '''
 
+import functools
 import imp
 import os
+import time
+
 
 # NOTICE(cloudnull): The connection plugin imported using the full path to the
 #                    file because the ssh connection plugin is not importable.
@@ -264,6 +267,36 @@ if not hasattr(SSH, 'shlex_quote'):
     setattr(SSH, 'shlex_quote', shlex_quote)
 
 
+def retry(ExceptionToCheck, tries=3, delay=1, backoff=2):
+    """Retry calling the decorated function using an exponential backoff.
+
+    :param ExceptionToCheck: the exception to check. may be a tuple of
+                             exceptions to check
+    :type ExceptionToCheck: Exception or tuple
+    :param tries: number of times to try (not retry) before giving up
+    :type tries: int
+    :param delay: initial delay between retries in seconds
+    :type delay: int
+    :param backoff: backoff multiplier e.g. value of 2 will double the delay
+                    each retry
+    :type backoff: int
+    """
+    def deco_retry(f):
+        @functools.wraps(f)
+        def f_retry(*args, **kwargs):
+            mtries, mdelay = tries, delay
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except ExceptionToCheck:
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+        return f_retry
+    return deco_retry
+
+
 class Connection(SSH.Connection):
     """Transport options for containers.
 
@@ -327,6 +360,9 @@ class Connection(SSH.Connection):
             #                  revise this in the future.
             self.container_tech = 'lxc'
 
+        if not hasattr(self._play_context, 'retries'):
+            self._play_context.retries = 3
+
         # Store the container pid for multi-use
         self.container_pid = None
 
@@ -360,6 +396,7 @@ class Connection(SSH.Connection):
                                                      self.physical_host)
         self.host = self._play_context.remote_addr = physical_host_addr
 
+    @retry(ExceptionToCheck=Exception)
     def exec_command(self, cmd, in_data=None, sudoable=True):
         """run a command on the remote host."""