Introduce PINGABLE and SSHABLE waiters and wait_until state support

This change introduces two new instance state waiters and uses them to
extend the existing tempest.common.compute.create_test_server wait_until
state support.

They are being introduced in an effort to allow the guest OS time to
start *before* we start attempting to interact with it either directly
by connecting to the instance or indirectly by hot-plugging or
hot-unplugging devices. The latter on some virt backends being an issue
if the guest OS is unable to respond to the underlying ACPI requests
sent to it.

It should be noted that these new states rely on the instance already
being ACTIVE before we begin to wait for the instance to either become
pingable or accessible over SSH. This is taken into account and will
always happen for these states even if validation isn't enabled in the
environment and thus it isn't possible to wait until the instance is
pingable or accessible over ssh.

Change-Id: Ib14fa7dc5c8093eed498049cd0a56c8ac6853660

tempest/common/compute.py

@@ -23,6 +23,7 @@ from urllib import parse as urlparse
 from oslo_log import log as logging
 from oslo_utils import excutils
 
+from tempest.common.utils.linux import remote_client
 from tempest.common import waiters
 from tempest import config
 from tempest import exceptions
@@ -98,7 +99,9 @@ def create_test_server(clients, validatable=False, validation_resources=None,
         server. Include a keypair, a security group and an IP.
     :param tenant_network: Tenant network to be used for creating a server.
     :param wait_until: Server status to wait for the server to reach after
-        its creation.
+        its creation. Additionally PINGABLE and SSHABLE states are also
+        accepted when the server is both validatable and has the required
+        validation_resources provided.
     :param volume_backed: Whether the server is volume backed or not.
         If this is true, a volume will be created and create server will be
         requested with 'block_device_mapping_v2' populated with below values:
@@ -125,8 +128,6 @@ def create_test_server(clients, validatable=False, validation_resources=None,
     :returns: a tuple
     """
 
-    # TODO(jlanoux) add support of wait_until PINGABLE/SSHABLE
-
     if name is None:
         name = data_utils.rand_name(__name__ + "-instance")
     if flavor is None:
@@ -259,18 +260,50 @@ def create_test_server(clients, validatable=False, validation_resources=None,
                 server_id=servers[0]['id'])
 
     if wait_until:
+
+        # NOTE(lyarwood): PINGABLE and SSHABLE both require the instance to
+        # go ACTIVE initially before we can setup the fip(s) etc so stash
+        # this additional wait state for later use.
+        wait_until_extra = None
+        if wait_until in ['PINGABLE', 'SSHABLE']:
+            wait_until_extra = wait_until
+            wait_until = 'ACTIVE'
+
         for server in servers:
             try:
                 waiters.wait_for_server_status(
                     clients.servers_client, server['id'], wait_until,
                     request_id=request_id)
 
-                # Multiple validatable servers are not supported for now. Their
-                # creation will fail with the condition above.
                 if CONF.validation.run_validation and validatable:
+
                     if CONF.validation.connect_method == 'floating':
                         _setup_validation_fip()
 
+                    server_ip = get_server_ip(
+                        server, validation_resources=validation_resources)
+
+                    if wait_until_extra == 'PINGABLE':
+                        waiters.wait_for_ping(
+                            server_ip,
+                            clients.servers_client.build_timeout,
+                            clients.servers_client.build_interval
+                        )
+
+                    if wait_until_extra == 'SSHABLE':
+                        pkey = validation_resources['keypair']['private_key']
+                        ssh_client = remote_client.RemoteClient(
+                            server_ip,
+                            CONF.validation.image_ssh_user,
+                            pkey=pkey,
+                            server=server,
+                            servers_client=clients.servers_client
+                        )
+                        waiters.wait_for_ssh(
+                            ssh_client,
+                            clients.servers_client.build_timeout
+                        )
+
             except Exception:
                 with excutils.save_and_reraise_exception():
                     for server in servers:

tempest/common/waiters.py

@@ -10,6 +10,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
 import re
 import time
 
@@ -570,3 +571,26 @@ def wait_for_server_floating_ip(servers_client, server, floating_ip,
                        'in time.' % (floating_ip, server['id']))
             raise lib_exc.TimeoutException(msg)
         time.sleep(servers_client.build_interval)
+
+
+def wait_for_ping(server_ip, timeout=30, interval=1):
+    """Waits for an address to become pingable"""
+    start_time = int(time.time())
+    while int(time.time()) - start_time < timeout:
+        response = os.system("ping -c 1 " + server_ip)
+        if response == 0:
+            return
+        time.sleep(interval)
+    raise lib_exc.TimeoutException()
+
+
+def wait_for_ssh(ssh_client, timeout=30):
+    """Waits for SSH connection to become usable"""
+    start_time = int(time.time())
+    while int(time.time()) - start_time < timeout:
+        try:
+            ssh_client.validate_authentication()
+            return
+        except lib_exc.SSHTimeout:
+            pass
+    raise lib_exc.TimeoutException()

tempest/tests/common/test_waiters.py

@@ -523,6 +523,70 @@ class TestVolumeWaiters(base.TestCase):
         mock_list_volume_attachments.assert_called_once_with(
             mock.sentinel.server_id)
 
+    @mock.patch('os.system')
+    def test_wait_for_ping_host_alive(self, mock_ping):
+        mock_ping.return_value = 0
+        # Assert that nothing is raised as the host is alive
+        waiters.wait_for_ping('127.0.0.1', 10, 1)
+
+    @mock.patch('os.system')
+    def test_wait_for_ping_host_eventually_alive(self, mock_ping):
+        mock_ping.side_effect = [1, 1, 0]
+        # Assert that nothing is raised when the host is eventually alive
+        waiters.wait_for_ping('127.0.0.1', 10, 1)
+
+    @mock.patch('os.system')
+    def test_wait_for_ping_timeout(self, mock_ping):
+        mock_ping.return_value = 1
+        # Assert that TimeoutException is raised when the host is dead
+        self.assertRaises(
+            lib_exc.TimeoutException,
+            waiters.wait_for_ping,
+            '127.0.0.1',
+            .1,
+            .1
+        )
+
+    def test_wait_for_ssh(self):
+        mock_ssh_client = mock.Mock()
+        mock_ssh_client.validate_authentication.return_value = True
+        # Assert that nothing is raised when validate_authentication returns
+        waiters.wait_for_ssh(mock_ssh_client, .1)
+        mock_ssh_client.validate_authentication.assert_called_once()
+
+    def test_wait_for_ssh_eventually_up(self):
+        mock_ssh_client = mock.Mock()
+        timeout = lib_exc.SSHTimeout(
+            host='foo',
+            username='bar',
+            password='fizz'
+        )
+        mock_ssh_client.validate_authentication.side_effect = [
+            timeout,
+            timeout,
+            True
+        ]
+        # Assert that nothing is raised if validate_authentication passes
+        # before the timeout
+        waiters.wait_for_ssh(mock_ssh_client, 10)
+
+    def test_wait_for_ssh_timeout(self):
+        mock_ssh_client = mock.Mock()
+        timeout = lib_exc.SSHTimeout(
+            host='foo',
+            username='bar',
+            password='fizz'
+        )
+        mock_ssh_client.validate_authentication.side_effect = timeout
+        # Assert that TimeoutException is raised when validate_authentication
+        # doesn't pass in time.
+        self.assertRaises(
+            lib_exc.TimeoutException,
+            waiters.wait_for_ssh,
+            mock_ssh_client,
+            .1
+        )
+
 
 class TestServerFloatingIPWaiters(base.TestCase):