Increase timeout for assert_is_reachable method up to 10 minutes.

This is intended to increase tollerance to slow VM booting times.

Change-Id: Ib5018f57b184b7155dd2f9a6bf12fac105bc39ab

tobiko/openstack/stacks/_cirros.py

@@ -63,6 +63,9 @@ class CirrosServerStackFixture(_nova.ServerStackFixture):
     #: CirrOS can't get IP addresses from config-drive
     need_dhcp = True
 
+    #: We expect CirrOS based servers to be fast to boot
+    is_reachable_timeout: tobiko.Seconds = 300.
+
 
 class CirrosPeerServerStackFixture(CirrosServerStackFixture,
                                    _nova.PeerServerStackFixture):

tobiko/openstack/stacks/_nova.py

@@ -365,11 +365,23 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
         requirements['port'] += 1
         return requirements
 
-    def assert_is_reachable(self):
+    is_reachable_timeout: tobiko.Seconds = None
-        ping.assert_reachable_hosts([self.ip_address])
 
-    def assert_is_unreachable(self):
+    def assert_is_reachable(self,
-        ping.assert_unreachable_hosts([self.ip_address])
+                            ssh_client: ssh.SSHClientType = None,
+                            timeout: tobiko.Seconds = None):
+        if timeout is None:
+            timeout = self.is_reachable_timeout
+        ping.assert_reachable_hosts([self.ip_address],
+                                    ssh_client=ssh_client,
+                                    timeout=timeout)
+
+    def assert_is_unreachable(self,
+                              ssh_client: ssh.SSHClientType = None,
+                              timeout: tobiko.Seconds = None):
+        ping.assert_unreachable_hosts([self.ip_address],
+                                      ssh_client=ssh_client,
+                                      timeout=timeout)
 
     user_data = None
 
@@ -383,6 +395,9 @@ class CloudInitServerStackFixture(ServerStackFixture, ABC):
     #: nax SWAP file size in bytes
     swap_maxsize: typing.Optional[int] = None
 
+    # I expect cloud-init based servers to be slow to boot
+    is_reachable_timeout: tobiko.Seconds = 600.
+
     @property
     def user_data(self):
         return nova.user_data(self.cloud_config)

tobiko/openstack/stacks/_vlan.py

@@ -91,20 +91,32 @@ class VlanServerStackFixture(_nova.ServerStackFixture, abc.ABC):
         return tobiko.setup_fixture(VlanProxyServerStackFixture).ssh_client
 
     def assert_vlan_is_reachable(self,
-                                 ip_version: int = None):
+                                 ip_version: int = None,
+                                 timeout: tobiko.Seconds = None,
+                                 ssh_client: ssh.SSHClientType = None):
         fixed_ips = self.list_vlan_fixed_ips(ip_version=ip_version)
         if fixed_ips:
-            ping.assert_reachable_hosts(
+            if timeout is None:
-                fixed_ips, ssh_client=self.vlan_ssh_proxy_client)
+                timeout = self.is_reachable_timeout
+            if ssh_client is None:
+                ssh_client = self.vlan_ssh_proxy_client
+            ping.assert_reachable_hosts(fixed_ips,
+                                        ssh_client=ssh_client,
+                                        timeout=timeout)
         else:
             tobiko.fail(f'Server {self.stack_name} has any IP on VLAN port')
 
     def assert_vlan_is_unreachable(self,
-                                   ip_version: int = None):
+                                   ip_version: int = None,
+                                   timeout: tobiko.Seconds = None,
+                                   ssh_client: ssh.SSHClientType = None):
         fixed_ips = self.list_vlan_fixed_ips(ip_version=ip_version)
         if fixed_ips:
-            ping.assert_unreachable_hosts(
+            if ssh_client is None:
-                fixed_ips, ssh_client=self.vlan_ssh_proxy_client)
+                ssh_client = self.vlan_ssh_proxy_client
+            ping.assert_unreachable_hosts(fixed_ips,
+                                          ssh_client=ssh_client,
+                                          timeout=timeout)
         else:
             tobiko.fail(f'Server {self.stack_name} has any IP on VLAN port')
 

tobiko/shell/ping/_exception.py

@@ -58,13 +58,15 @@ class PingFailed(PingError, tobiko.FailureException):
 
 
 class UnreachableHostsException(PingFailed):
-    message = ("timeout of {timeout} seconds expired waiting for "
+    message = ("timeout of {timeout} seconds expired after "
-               "host(s) {hosts} to get pinged successfully")
+               "{elapsed_time} seconds waiting for "
+               "host(s) {hosts} to return ping messages")
 
 
 class ReachableHostsException(PingFailed):
-    message = ("timeout of {timeout} seconds expired waiting for "
+    message = ("timeout of {timeout} seconds expired after "
-               "hosts(s) {hosts} to fail getting pinged")
+               "{elapsed_time} seconds waiting for hosts(s) {hosts} "
+               "to fail returning ping messages")
 
 
 class UnsupportedPingOption(PingError):

tobiko/shell/ping/_ping.py

@@ -65,6 +65,15 @@ def wait_for_ping_hosts(hosts: typing.Iterable[PingHostType],
                         retry_interval: tobiko.Seconds = None,
                         **params) \
         -> None:
+    if retry_timeout is None:
+        retry_timeout = params.get('timeout')
+    LOG.debug("Wait for ping hosts:\n"
+              f"  hosts: {hosts}\n"
+              f"  check_unreachable: {check_unreachable}\n"
+              f"  retry_count: {retry_count}\n"
+              f"  retry_timeout: {retry_timeout}\n"
+              f"  retry_interval: {retry_interval}\n"
+              f"  **params: {params}\n")
     for attempt in tobiko.retry(count=retry_count,
                                 timeout=retry_timeout,
                                 interval=retry_interval,
@@ -76,15 +85,17 @@ def wait_for_ping_hosts(hosts: typing.Iterable[PingHostType],
         else:
             hosts = unreachable
         if hosts:
-            try:
+            if attempt.is_last:
-                attempt.check_limits()
-            except tobiko.RetryLimitError:
                 if check_unreachable:
                     raise _exception.ReachableHostsException(
-                        hosts=hosts, timeout=attempt.timeout) from None
+                        hosts=hosts,
+                        timeout=attempt.timeout,
+                        elapsed_time=attempt.elapsed_time) from None
                 else:
                     raise _exception.UnreachableHostsException(
-                        hosts=hosts, timeout=attempt.timeout) from None
+                        hosts=hosts,
+                        timeout=attempt.timeout,
+                        elapsed_time=attempt.elapsed_time) from None
         else:
             break