Lock VM creation until guest OS is ready and cloud-init done

Avoid concurrency issues by locking VM creation until it is
completed.
Add step that verifies that the VM instance boot is completed before
releasing the lock and before the tests try to ping, ssh, etc that VM.
Add another step that checks the VM FIP is pingable.
Add a third step that checks the VM has completed the cloud-init
procedure.

Some tests (SameHostNetworkTest and DifferentHostNetworkTest) try to
migrate VMs immediately after creating them in order to move them to the
expected compute node. Sometimes, those VMs fail to contact the metadata
service because their migration happens exactly when the guest OS was
trying to obtain the metadata.
With this patch, those VMs are not migrated until the guest OS is ready.

Change-Id: Ia32b8c774e1c94277fa7243b7943a8b19763a501

tobiko/openstack/nova/__init__.py

@@ -38,6 +38,7 @@ list_services = _client.list_services
 nova_client = _client.nova_client
 NovaClientFixture = _client.NovaClientFixture
 wait_for_server_status = _client.wait_for_server_status
+wait_for_guest_os_ready = _client.wait_for_guest_os_ready
 WaitForServerStatusError = _client.WaitForServerStatusError
 WaitForServerStatusTimeout = _client.WaitForServerStatusTimeout
 shutoff_server = _client.shutoff_server

tobiko/openstack/nova/_client.py

@@ -314,6 +314,28 @@ def get_console_output(server: typing.Optional[ServerType] = None,
     return None
 
 
+def wait_for_guest_os_ready(server: typing.Optional[ServerType] = None,
+                            server_id: typing.Optional[str] = None,
+                            timeout: tobiko.Seconds = 120.,
+                            interval: tobiko.Seconds = 5.,
+                            client: NovaClientType = None) -> None:
+
+    def system_booted():
+        console_output = get_console_output(
+            server=server, server_id=server_id, client=client) or ''
+        for line in console_output.splitlines():
+            if 'login:' in line.lower():
+                return True
+        return False
+
+    for _ in tobiko.retry(timeout=timeout, interval=interval):
+        if system_booted():
+            LOG.debug('VM boot completed successfully')
+            return
+        else:
+            LOG.debug('VM boot not completed yet')
+
+
 class HasNovaClientMixin(object):
 
     nova_client: NovaClientType = None

tobiko/openstack/stacks/_nova.py

@@ -20,6 +20,7 @@ import typing
 from abc import ABC
 
 import netaddr
+from oslo_concurrency import lockutils
 from oslo_log import log
 
 import tobiko
@@ -278,20 +279,24 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
             hypervisors = nova.list_servers_hypervisors(self.same_host)
             if hypervisor not in hypervisors:
                 self.migrate_server(live=True,
-                                    host=hypervisors.unique)
+                                    host=hypervisors.unique,
+                                    wait_for_guest_os=True)
 
     def validate_different_host_scheduler_hints(self, hypervisor):
         if self.different_host:
             hypervisors = nova.list_servers_hypervisors(self.different_host)
             if hypervisor in hypervisors:
-                self.migrate_server(live=True)
+                self.migrate_server(live=True, wait_for_guest_os=True)
 
     def migrate_server(self,
                        live=False,
                        host: str = None,
-                       block_migration: bool = None) \
+                       block_migration: bool = None,
+                       wait_for_guest_os: bool = False) \
             -> nova.NovaServer:
         server = nova.activate_server(server=self.server_id)
+        if wait_for_guest_os:
+            nova.wait_for_guest_os_ready(server)
         if live:
             nova.live_migrate_server(server,
                                      host=host,
@@ -424,6 +429,17 @@ class CloudInitServerStackFixture(ServerStackFixture, ABC):
     #: nax SWAP file size in bytes
     swap_maxsize: typing.Optional[int] = None
 
+    @lockutils.synchronized(
+        'cloudinit_server_setup_fixture',
+        external=True, lock_path=tobiko.LOCK_DIR)
+    def setup_fixture(self):
+        super(CloudInitServerStackFixture, self).setup_fixture()
+        nova.wait_for_guest_os_ready(server_id=self.server_id,
+                                     timeout=900.)
+        if self.has_floating_ip:
+            self.assert_is_reachable()
+            self.wait_for_cloud_init_done()
+
     @property
     def is_reachable_timeout(self) -> tobiko.Seconds:
         # I expect cloud-init based servers to be slow to boot

tobiko/openstack/stacks/_ubuntu.py

@@ -41,7 +41,7 @@ class UbuntuMinimalImageFixture(glance.FileGlanceImageFixture):
     is_reachable_timeout = CONF.tobiko.nova.ubuntu_is_reachable_timeout
 
     @lockutils.synchronized(
-        'ubuntu_minimal_setup_fixture',
+        'ubuntu_minimal_image_setup_fixture',
         external=True, lock_path=tobiko.LOCK_DIR)
     def setup_fixture(self):
         super(UbuntuMinimalImageFixture, self).setup_fixture()

tobiko/tests/scenario/nova/test_server.py

@@ -14,7 +14,6 @@
 #    under the License.
 from __future__ import absolute_import
 
-import abc
 import contextlib
 import random
 
@@ -188,18 +187,7 @@ class CirrosServerTest(BaseServerTest):
     stack = tobiko.required_fixture(CirrosServerStackFixture)
 
 
-class CloudInitServerStackFixture(stacks.CloudInitServerStackFixture,
-                                  abc.ABC):
-
-    def validate_created_stack(self):
-        stack = super().validate_created_stack()
-        self.wait_for_cloud_init_done()
-        self.assert_is_reachable()
-        return stack
-
-
-class UbuntuMinimalServerStackFixture(CloudInitServerStackFixture,
-                                      stacks.UbuntuMinimalServerStackFixture):
+class UbuntuMinimalServerStackFixture(stacks.UbuntuMinimalServerStackFixture):
     pass
 
 
@@ -209,8 +197,7 @@ class UbuntuMinimalServerTest(BaseServerTest):
     stack = tobiko.required_fixture(UbuntuMinimalServerStackFixture)
 
 
-class UbuntuServerStackFixture(CloudInitServerStackFixture,
-                               stacks.UbuntuServerStackFixture):
+class UbuntuServerStackFixture(stacks.UbuntuServerStackFixture):
     pass