Implement Nova Server migration tools

Change-Id: I9d794da07cebffa769f9ac4b4d7df3ebb7765334

tobiko/openstack/nova/__init__.py

@@ -34,9 +34,12 @@ list_services = _client.list_services
 nova_client = _client.nova_client
 NovaClientFixture = _client.NovaClientFixture
 wait_for_server_status = _client.wait_for_server_status
-ServerStatusTimeout = _client.ServerStatusTimeout
+WaitForServerStatusError = _client.WaitForServerStatusError
+WaitForServerStatusTimeout = _client.WaitForServerStatusTimeout
 shutoff_server = _client.shutoff_server
 activate_server = _client.activate_server
+migrate_server = _client.migrate_server
+confirm_resize = _client.confirm_resize
 
 WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError
 cloud_config = _cloud_init.cloud_config
@@ -48,6 +51,8 @@ wait_for_cloud_init_status = _cloud_init.wait_for_cloud_init_status
 skip_if_missing_hypervisors = _hypervisor.skip_if_missing_hypervisors
 get_same_host_hypervisors = _hypervisor.get_same_host_hypervisors
 get_different_host_hypervisors = _hypervisor.get_different_host_hypervisors
+get_server_hypervisor = _hypervisor.get_server_hypervisor
+get_servers_hypervisors = _hypervisor.get_servers_hypervisors
 
 find_server_ip_address = _server.find_server_ip_address
 HasServerMixin = _server.HasServerMixin

tobiko/openstack/nova/_client.py

@@ -108,8 +108,32 @@ def find_service(client=None, unique=False, **params):
         return services.first
 
 
+def get_server_id(server):
+    if isinstance(server, str):
+        return server
+    else:
+        return server.id
+
+
 def get_server(server, client=None, **params):
-    return nova_client(client).servers.get(server, **params)
+    server_id = get_server_id(server)
+    return nova_client(client).servers.get(server_id, **params)
+
+
+def migrate_server(server, client=None, **params):
+    # pylint: disable=protected-access
+    server_id = get_server_id(server)
+    LOG.debug(f"Start server migration (server_id='{server_id}', "
+              f"info={params})")
+    return nova_client(client).servers._action('migrate', server_id,
+                                               info=params)
+
+
+def confirm_resize(server, client=None, **params):
+    server_id = get_server_id(server)
+    LOG.debug(f"Confirm server resize (server_id='{server_id}', "
+              f"info={params})")
+    return nova_client(client).servers.confirm_resize(server_id, **params)
 
 
 MAX_SERVER_CONSOLE_OUTPUT_LENGTH = 1024 * 256
@@ -159,30 +183,52 @@ class HasNovaClientMixin(object):
                                   **params)
 
 
-class ServerStatusTimeout(tobiko.TobikoException):
+class WaitForServerStatusError(tobiko.TobikoException):
-    message = ("Server {server_id} didn't change its status to {status} "
+    message = ("Server {server_id} not changing status from {server_status} "
-               "status after {timeout} seconds")
+               "to {status}")
+
+
+class WaitForServerStatusTimeout(WaitForServerStatusError):
+    message = ("Server {server_id} didn't change its status from "
+               "{server_status} to {status} status after {timeout} seconds")
+
+
+NOVA_SERVER_TRANSIENT_STATUS = {
+    'ACTIVE': ('BUILD', 'SHUTOFF'),
+    'SHUTOFF': ('ACTIVE'),
+    'VERIFY_RESIZE': ('RESIZE'),
+}
 
 
 def wait_for_server_status(server, status, client=None, timeout=None,
-                           sleep_time=None):
+                           sleep_time=None, transient_status=None):
     if timeout is None:
         timeout = 300.
     if sleep_time is None:
         sleep_time = 5.
     start_time = time.time()
+    if transient_status is None:
+        transient_status = NOVA_SERVER_TRANSIENT_STATUS.get(status) or tuple()
     while True:
         server = get_server(server=server, client=client)
         if server.status == status:
             break
 
+        if server.status not in transient_status:
+            raise WaitForServerStatusError(server_id=server.id,
+                                           server_status=server.status,
+                                           status=status)
+
         if time.time() - start_time >= timeout:
-            raise ServerStatusTimeout(server_id=server.id,
+            raise WaitForServerStatusTimeout(server_id=server.id,
+                                             server_status=server.status,
                                              status=status,
                                              timeout=timeout)
 
-        LOG.debug('Waiting for server %r status to get from %r to %r',
+        progress = getattr(server, 'progress', None)
-                  server.id, server.status, status)
+        LOG.debug(f"Waiting for server {server.id} status to get from "
+                  f"{server.status} to {status} "
+                  f"(progress={progress}%)")
         time.sleep(sleep_time)
     return server
 
@@ -207,6 +253,13 @@ def activate_server(server, client=None, timeout=None, sleep_time=None):
 
     if server.status == 'SHUTOFF':
         client.servers.start(server.id)
+    elif server.status == 'RESIZE':
+        wait_for_server_status(server=server.id, status='VERIFY_RESIZE',
+                               client=client, timeout=timeout,
+                               sleep_time=sleep_time)
+        client.servers.confirm_resize(server)
+    elif server.status == 'VERIFY_RESIZE':
+        client.servers.confirm_resize(server)
     else:
         client.servers.reboot(server.id, reboot_type='HARD')
 

tobiko/openstack/nova/_hypervisor.py

@@ -54,8 +54,8 @@ def skip_if_missing_hypervisors(count=1, **params):
                           **params)
 
 
-def get_same_host_hypervisors(server_ids, hypervisor):
+def get_same_host_hypervisors(servers, hypervisor):
-    host_hypervisors = get_servers_hypervisors(server_ids)
+    host_hypervisors = get_servers_hypervisors(servers)
     same_host_server_ids = host_hypervisors.pop(hypervisor, None)
     if same_host_server_ids:
         return {hypervisor: same_host_server_ids}
@@ -63,17 +63,27 @@ def get_same_host_hypervisors(server_ids, hypervisor):
         return {}
 
 
-def get_different_host_hypervisors(server_ids, hypervisor):
+def get_different_host_hypervisors(servers, hypervisor):
-    host_hypervisors = get_servers_hypervisors(server_ids)
+    host_hypervisors = get_servers_hypervisors(servers)
     host_hypervisors.pop(hypervisor, None)
     return host_hypervisors
 
 
-def get_servers_hypervisors(server_ids):
+def get_servers_hypervisors(servers, client=None):
     hypervisors = collections.defaultdict(list)
-    if server_ids:
+    for server in (servers or list()):
-        for server_id in (server_ids or list()):
+        client = _client.nova_client(client)
-            server = _client.get_server(server_id)
+        if isinstance(server, str):
-            hypervisor = getattr(server, 'OS-EXT-SRV-ATTR:host')
+            server_id = server
+            server = _client.get_server(server_id, client=client)
+        else:
+            server_id = server.id
+        hypervisor = get_server_hypervisor(server)
         hypervisors[hypervisor].append(server_id)
     return hypervisors
+
+
+def get_server_hypervisor(server, client=None):
+    if isinstance(server, str):
+        server = _client.get_server(server, client=client)
+    return getattr(server, 'OS-EXT-SRV-ATTR:host')

tobiko/openstack/stacks/_nova.py

@@ -19,6 +19,7 @@ import os
 import typing  # noqa
 
 import six
+from oslo_log import log
 
 import tobiko
 from tobiko import config
@@ -32,6 +33,7 @@ from tobiko.shell import sh
 
 
 CONF = config.CONF
+LOG = log.getLogger(__name__)
 
 
 class KeyPairStackFixture(heat.HeatStackFixture):
@@ -266,8 +268,10 @@ class ServerStackFixture(heat.HeatStackFixture):
         tobiko.setup_fixture(self)
         try:
             server = nova.wait_for_server_status(self.server_id, status)
-        except nova.ServerStatusTimeout:
+        except nova.WaitForServerStatusError:
             server = nova.get_server(self.server_id)
+            LOG.debug(f"Server {server.id} status is {server.status} instead "
+                      f"of {status}", exc_info=1)
         if server.status == status:
             return server
         elif status == "ACTIVE":

tobiko/tests/functional/openstack/test_nova.py

@@ -21,8 +21,10 @@ import netaddr
 import testtools
 
 import tobiko
+from tobiko.openstack import keystone
 from tobiko.openstack import nova
 from tobiko.openstack import stacks
+from tobiko.shell import ping
 
 
 class KeyPairTest(testtools.TestCase):
@@ -156,3 +158,63 @@ class ServiceTest(testtools.TestCase):
 
     def test_wait_for_services_up(self):
         nova.wait_for_services_up()
+
+
+class MigrateServerStack(stacks.CirrosServerStackFixture):
+    pass
+
+
+@keystone.skip_unless_has_keystone_credentials()
+@nova.skip_if_missing_hypervisors(count=2)
+class MigrateServerTest(testtools.TestCase):
+
+    stack = tobiko.required_setup_fixture(MigrateServerStack)
+
+    def test_migrate_server(self):
+        """Tests cold migration actually changes hypervisor
+        """
+        server = self.setup_server()
+        initial_hypervisor = nova.get_server_hypervisor(server)
+
+        server = self.migrate_server(server)
+
+        final_hypervisor = nova.get_server_hypervisor(server)
+        self.assertNotEqual(initial_hypervisor, final_hypervisor)
+
+    def test_migrate_server_with_host(self):
+        """Tests cold migration actually ends on target hypervisor
+        """
+        server = self.setup_server()
+        initial_hypervisor = nova.get_server_hypervisor(server)
+        for hypervisor in nova.list_hypervisors(status='enabled', state='up'):
+            if initial_hypervisor != hypervisor.hypervisor_hostname:
+                target_hypervisor = hypervisor.hypervisor_hostname
+                break
+        else:
+            self.skip("Cannot find a valid hypervisor host to migrate server "
+                      "to")
+
+        server = self.migrate_server(server=server, host=target_hypervisor)
+
+        final_hypervisor = nova.get_server_hypervisor(server)
+        self.assertEqual(target_hypervisor, final_hypervisor)
+
+    def setup_server(self):
+        server = self.stack.ensure_server_status('ACTIVE')
+        self.assertEqual('ACTIVE', server.status)
+        return server
+
+    def migrate_server(self, server, **params):
+        self.assertEqual('ACTIVE', server.status)
+        nova.migrate_server(server, **params)
+
+        server = nova.wait_for_server_status(server, 'VERIFY_RESIZE')
+        self.assertEqual('VERIFY_RESIZE', server.status)
+        nova.confirm_resize(server)
+
+        server = nova.wait_for_server_status(
+            server, 'ACTIVE', transient_status={'VERIFY_RESIZE'})
+        self.assertEqual('ACTIVE', server.status)
+
+        ping.assert_reachable_hosts([self.stack.ip_address])
+        return server