Merge "Check VM's console log before trying to SSH to it."

neutron_tempest_plugin/scenario/base.py

@@ -122,9 +122,11 @@ class BaseTempestTestCase(base_api.BaseNetworkTest):
         if not kwargs.get('security_groups'):
             kwargs['security_groups'] = [{'name': 'default'}]
 
-        client = self.os_primary.servers_client
+        client = kwargs.pop('client', None)
-        if kwargs.get('availability_zone'):
+        if client is None:
-            client = self.os_admin.servers_client
+            client = self.os_primary.servers_client
+            if kwargs.get('availability_zone'):
+                client = self.os_admin.servers_client
 
         server = client.create_server(
             flavorRef=flavor_ref,
@@ -140,6 +142,10 @@ class BaseTempestTestCase(base_api.BaseNetworkTest):
         self.addCleanup(test_utils.call_and_ignore_notfound_exc,
                         client.delete_server,
                         server['server']['id'])
+
+        self.wait_for_server_active(server['server'], client=client)
+        self.wait_for_guest_os_ready(server['server'], client=client)
+
         return server
 
     @classmethod
@@ -253,7 +259,6 @@ class BaseTempestTestCase(base_api.BaseNetworkTest):
             server_kwargs['name'] = server_name
 
         self.server = self.create_server(**server_kwargs)
-        self.wait_for_server_active(self.server['server'])
         self.port = self.client.list_ports(network_id=self.network['id'],
                                            device_id=self.server[
                                                'server']['id'])['ports'][0]
@@ -465,6 +470,28 @@ class BaseTempestTestCase(base_api.BaseNetworkTest):
         self.wait_for_server_status(
             server, constants.SERVER_STATUS_ACTIVE, client)
 
+    def wait_for_guest_os_ready(self, server, client=None):
+        if not CONF.compute_feature_enabled.console_output:
+            LOG.debug('Console output not supported, cannot check if server '
+                      '%s is ready.', server['server']['id'])
+            return
+
+        client = client or self.os_primary.servers_client
+
+        def system_booted():
+            console_output = client.get_console_output(server['id'])['output']
+            for line in console_output.split('\n'):
+                if 'login:' in line.lower():
+                    return True
+            return False
+
+        try:
+            utils.wait_until_true(system_booted, sleep=5)
+        except utils.WaitTimeout:
+            LOG.debug("No correct output in console of server %s found. "
+                      "Guest operating system status can't be checked.",
+                      server['id'])
+
     def check_servers_hostnames(self, servers, timeout=None, log_errors=True,
                                 external_port=None):
         """Compare hostnames of given servers with their names."""

neutron_tempest_plugin/scenario/test_connectivity.py

@@ -65,6 +65,7 @@ class NetworkConnectivityTest(base.BaseTempestTestCase):
 
         for vm in vms:
             self.wait_for_server_active(vm['server'])
+            self.wait_for_guest_os_ready(vm['server'])
 
         return vms
 
@@ -231,6 +232,7 @@ class NetworkConnectivityTest(base.BaseTempestTestCase):
             networks=[{'uuid': network['id']}],
             security_groups=[{'name': self.secgroup['name']}])
         self.wait_for_server_active(vm['server'])
+        self.wait_for_guest_os_ready(vm['server'])
 
         vm_port = self.client.list_ports(
             network_id=network['id'], device_id=vm['server']['id'])['ports'][0]

neutron_tempest_plugin/scenario/test_floatingip.py

@@ -417,6 +417,7 @@ class TestFloatingIPUpdate(FloatingIpTestCasesMixin,
             servers.append(server)
         for server in servers:
             self.wait_for_server_active(server)
+            self.wait_for_guest_os_ready(server)
 
         self.fip = self.create_floatingip(port=ports[0])
         self.check_connectivity(self.fip['floating_ip_address'],

neutron_tempest_plugin/scenario/test_internal_dns.py

@@ -52,6 +52,7 @@ class InternalDNSTest(base.BaseTempestTestCase):
                 {'name': self.security_groups[-1]['name']}],
             name='leia')
         self.wait_for_server_active(leia['server'])
+        self.wait_for_guest_os_ready(leia['server'])
 
         ssh_client = ssh.Client(
             self.fip['floating_ip_address'],

neutron_tempest_plugin/scenario/test_ipv6.py

@@ -145,6 +145,7 @@ class IPv6Test(base.BaseTempestTestCase):
         }
         vm = self.create_server(**server_kwargs)['server']
         self.wait_for_server_active(vm)
+        self.wait_for_guest_os_ready(vm)
         ipv4_port = self.client.list_ports(
             network_id=self.network['id'],
             device_id=vm['id'])['ports'][0]

neutron_tempest_plugin/scenario/test_metadata.py

@@ -126,6 +126,7 @@ class MetadataTest(base.BaseTempestTestCase):
         vm = self._create_server_with_network(
             self.network, use_advanced_image=use_advanced_image)
         self.wait_for_server_active(server=vm.server)
+        self.wait_for_guest_os_ready(vm.server)
         self._assert_has_ssh_connectivity(vm.ssh_client)
         interface = self._get_primary_interface(vm.ssh_client)
 

neutron_tempest_plugin/scenario/test_multicast.py

@@ -201,6 +201,7 @@ class BaseMulticastTest(object):
             security_groups=[{'name': self.secgroup['security_group']['name']}]
         )['server']
         self.wait_for_server_active(server)
+        self.wait_for_guest_os_ready(server)
         port = self.client.list_ports(
             network_id=self.network['id'], device_id=server['id'])['ports'][0]
         server['fip'] = self.create_floatingip(port=port)

neutron_tempest_plugin/scenario/test_port_forwardings.py

@@ -70,6 +70,7 @@ class PortForwardingTestJSON(base.BaseTempestTestCase):
                 networks=[{'port': port['id']}])['server']
             server['name'] = name
             self.wait_for_server_active(server)
+            self.wait_for_guest_os_ready(server)
             server['port_forwarding_tcp'] = self.create_port_forwarding(
                 self.fip['id'],
                 internal_port_id=port['id'],
@@ -255,6 +256,7 @@ class PortForwardingTestJSON(base.BaseTempestTestCase):
             name=name, networks=[{'port': port['id']}])['server']
         server['name'] = name
         self.wait_for_server_active(server)
+        self.wait_for_guest_os_ready(server)
 
         # Add a second fixed_ip address to port (same subnet)
         internal_subnet_id = port['fixed_ips'][0]['subnet_id']

neutron_tempest_plugin/scenario/test_qos.py

@@ -175,6 +175,7 @@ class QoSTestMixin(object):
 
         server = self.create_server(**server_kwargs)
         self.wait_for_server_active(server['server'])
+        self.wait_for_guest_os_ready(server['server'])
         self.check_connectivity(self.fip['floating_ip_address'],
                                 CONF.validation.image_ssh_user,
                                 keypair['private_key'])

neutron_tempest_plugin/scenario/test_trunk.py

@@ -161,6 +161,7 @@ class TrunkTest(base.BaseTempestTestCase):
 
     def _configure_vlan_subport(self, vm, vlan_tag, vlan_subnet):
         self.wait_for_server_active(server=vm.server)
+        self.wait_for_guest_os_ready(vm.server)
         self._wait_for_trunk(trunk=vm.trunk)
         self._wait_for_port(port=vm.port)
         self._wait_for_port(port=vm.subport)
@@ -199,6 +200,7 @@ class TrunkTest(base.BaseTempestTestCase):
         vm2 = self._create_server_with_trunk_port()
         for vm in (vm1, vm2):
             self.wait_for_server_active(server=vm.server)
+            self.wait_for_guest_os_ready(vm.server)
             self._wait_for_trunk(vm.trunk)
             self._assert_has_ssh_connectivity(vm.ssh_client)
 
@@ -325,6 +327,7 @@ class TrunkTest(base.BaseTempestTestCase):
                                 use_advanced_image=use_advanced_image)
         for role in ['migrate', 'connection_test']:
             self.wait_for_server_active(servers[role].server)
+            self.wait_for_guest_os_ready(servers[role].server)
             self._configure_vlan_subport(vm=servers[role],
                                          vlan_tag=vlan_tag,
                                          vlan_subnet=vlan_subnet)
@@ -377,6 +380,7 @@ class TrunkTest(base.BaseTempestTestCase):
                                      vlan_subnet=vlan_subnet)
         for vm in vms:
             self.wait_for_server_active(vm.server)
+            self.wait_for_guest_os_ready(vm.server)
 
         # allow ICMP traffic
         sg_rule = self.create_pingable_secgroup_rule(self.security_group['id'])