Regenerate ssh port forwardings for docker after controller reboot

Tobiko cloud recovery tests were failing when docker was used. After
a controller reboot, the initially created ssh port forwardings are not
valid anymore and connectivity with controller's docker API failed

When that happens, ssh port forwardings have to be created again

Similar issues have not been detected with podman

Change-Id: Ifad1b2dec6b3af5e805a3c96dbfffea0313be7dd

tobiko/shell/ssh/__init__.py

@@ -31,6 +31,8 @@ ssh_proxy_client = _client.ssh_proxy_client
 SSHConnectFailure = _client.SSHConnectFailure
 gather_ssh_connect_parameters = _client.gather_ssh_connect_parameters
 
+reset_default_ssh_port_forward_manager = \
+    _forward.reset_default_ssh_port_forward_manager
 get_port_forward_url = _forward.get_forward_url
 get_forward_port_address = _forward.get_forward_port_address
 SSHTunnelForwarderFixture = _forward.SSHTunnelForwarderFixture

tobiko/shell/ssh/_forward.py

@@ -49,6 +49,12 @@ def get_forward_url(url, ssh_client=None, manager=None):
     return binding_url(forward_address)
 
 
+def reset_default_ssh_port_forward_manager():
+    # pylint: disable=global-statement
+    global DEFAULT_SSH_PORT_FORWARD_MANAGER
+    DEFAULT_SSH_PORT_FORWARD_MANAGER = SSHPortForwardManager()
+
+
 class SSHPortForwardManager(object):
 
     def __init__(self):

tobiko/tripleo/containers.py

@@ -13,6 +13,7 @@ import tobiko
 from tobiko import podman
 from tobiko import docker
 from tobiko.openstack import topology
+from tobiko.shell import ssh
 from tobiko.tripleo import topology as tripleo_topology
 
 
@@ -55,13 +56,22 @@ def list_node_containers(ssh_client):
 def get_container_client(ssh_client=None):
     """returns a list of containers and their run state"""
 
-    if container_runtime_module == podman:
-        return container_runtime_module.get_podman_client(
-            ssh_client=ssh_client).connect()
-
-    elif container_runtime_module == docker:
-        return container_runtime_module.get_docker_client(
-            ssh_client=ssh_client).connect()
+    for attempt in tobiko.retry(
+            timeout=60.0,
+            interval=5.0):
+        try:
+            if container_runtime_module == podman:
+                return container_runtime_module.get_podman_client(
+                    ssh_client=ssh_client).connect()
+            elif container_runtime_module == docker:
+                return container_runtime_module.get_docker_client(
+                    ssh_client=ssh_client).connect()
+        except dockerlib.errors.DockerException:
+            LOG.debug('Unable to connect to docker API')
+            attempt.check_limits()
+            ssh.reset_default_ssh_port_forward_manager()
+    # no successful connection to docker/podman API has been performed
+    raise RuntimeError('Unable to connect to container mgmt tool')
 
 
 def list_containers_df(group=None):