From fa7fc4b3428faed13fcc759f73d466a9e7a07a42 Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Wed, 23 Sep 2020 14:45:26 +0200 Subject: [PATCH] Implement Nova Server migration tools Change-Id: I9d794da07cebffa769f9ac4b4d7df3ebb7765334 --- tobiko/openstack/nova/__init__.py | 7 +- tobiko/openstack/nova/_client.py | 75 ++++++++++++++++--- tobiko/openstack/nova/_hypervisor.py | 30 +++++--- tobiko/openstack/stacks/_nova.py | 6 +- .../tests/functional/openstack/test_nova.py | 62 +++++++++++++++ 5 files changed, 157 insertions(+), 23 deletions(-) diff --git a/tobiko/openstack/nova/__init__.py b/tobiko/openstack/nova/__init__.py index c18414914..c46bf8a36 100644 --- a/tobiko/openstack/nova/__init__.py +++ b/tobiko/openstack/nova/__init__.py @@ -34,9 +34,12 @@ list_services = _client.list_services nova_client = _client.nova_client NovaClientFixture = _client.NovaClientFixture wait_for_server_status = _client.wait_for_server_status -ServerStatusTimeout = _client.ServerStatusTimeout +WaitForServerStatusError = _client.WaitForServerStatusError +WaitForServerStatusTimeout = _client.WaitForServerStatusTimeout shutoff_server = _client.shutoff_server activate_server = _client.activate_server +migrate_server = _client.migrate_server +confirm_resize = _client.confirm_resize WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError cloud_config = _cloud_init.cloud_config @@ -48,6 +51,8 @@ wait_for_cloud_init_status = _cloud_init.wait_for_cloud_init_status skip_if_missing_hypervisors = _hypervisor.skip_if_missing_hypervisors get_same_host_hypervisors = _hypervisor.get_same_host_hypervisors get_different_host_hypervisors = _hypervisor.get_different_host_hypervisors +get_server_hypervisor = _hypervisor.get_server_hypervisor +get_servers_hypervisors = _hypervisor.get_servers_hypervisors find_server_ip_address = _server.find_server_ip_address HasServerMixin = _server.HasServerMixin diff --git a/tobiko/openstack/nova/_client.py b/tobiko/openstack/nova/_client.py index 9a3bec9d8..0783dc833 100644 --- a/tobiko/openstack/nova/_client.py +++ b/tobiko/openstack/nova/_client.py @@ -108,8 +108,32 @@ def find_service(client=None, unique=False, **params): return services.first +def get_server_id(server): + if isinstance(server, str): + return server + else: + return server.id + + def get_server(server, client=None, **params): - return nova_client(client).servers.get(server, **params) + server_id = get_server_id(server) + return nova_client(client).servers.get(server_id, **params) + + +def migrate_server(server, client=None, **params): + # pylint: disable=protected-access + server_id = get_server_id(server) + LOG.debug(f"Start server migration (server_id='{server_id}', " + f"info={params})") + return nova_client(client).servers._action('migrate', server_id, + info=params) + + +def confirm_resize(server, client=None, **params): + server_id = get_server_id(server) + LOG.debug(f"Confirm server resize (server_id='{server_id}', " + f"info={params})") + return nova_client(client).servers.confirm_resize(server_id, **params) MAX_SERVER_CONSOLE_OUTPUT_LENGTH = 1024 * 256 @@ -159,30 +183,52 @@ class HasNovaClientMixin(object): **params) -class ServerStatusTimeout(tobiko.TobikoException): - message = ("Server {server_id} didn't change its status to {status} " - "status after {timeout} seconds") +class WaitForServerStatusError(tobiko.TobikoException): + message = ("Server {server_id} not changing status from {server_status} " + "to {status}") + + +class WaitForServerStatusTimeout(WaitForServerStatusError): + message = ("Server {server_id} didn't change its status from " + "{server_status} to {status} status after {timeout} seconds") + + +NOVA_SERVER_TRANSIENT_STATUS = { + 'ACTIVE': ('BUILD', 'SHUTOFF'), + 'SHUTOFF': ('ACTIVE'), + 'VERIFY_RESIZE': ('RESIZE'), +} def wait_for_server_status(server, status, client=None, timeout=None, - sleep_time=None): + sleep_time=None, transient_status=None): if timeout is None: timeout = 300. if sleep_time is None: sleep_time = 5. start_time = time.time() + if transient_status is None: + transient_status = NOVA_SERVER_TRANSIENT_STATUS.get(status) or tuple() while True: server = get_server(server=server, client=client) if server.status == status: break - if time.time() - start_time >= timeout: - raise ServerStatusTimeout(server_id=server.id, - status=status, - timeout=timeout) + if server.status not in transient_status: + raise WaitForServerStatusError(server_id=server.id, + server_status=server.status, + status=status) - LOG.debug('Waiting for server %r status to get from %r to %r', - server.id, server.status, status) + if time.time() - start_time >= timeout: + raise WaitForServerStatusTimeout(server_id=server.id, + server_status=server.status, + status=status, + timeout=timeout) + + progress = getattr(server, 'progress', None) + LOG.debug(f"Waiting for server {server.id} status to get from " + f"{server.status} to {status} " + f"(progress={progress}%)") time.sleep(sleep_time) return server @@ -207,6 +253,13 @@ def activate_server(server, client=None, timeout=None, sleep_time=None): if server.status == 'SHUTOFF': client.servers.start(server.id) + elif server.status == 'RESIZE': + wait_for_server_status(server=server.id, status='VERIFY_RESIZE', + client=client, timeout=timeout, + sleep_time=sleep_time) + client.servers.confirm_resize(server) + elif server.status == 'VERIFY_RESIZE': + client.servers.confirm_resize(server) else: client.servers.reboot(server.id, reboot_type='HARD') diff --git a/tobiko/openstack/nova/_hypervisor.py b/tobiko/openstack/nova/_hypervisor.py index ab4163501..e65854096 100644 --- a/tobiko/openstack/nova/_hypervisor.py +++ b/tobiko/openstack/nova/_hypervisor.py @@ -54,8 +54,8 @@ def skip_if_missing_hypervisors(count=1, **params): **params) -def get_same_host_hypervisors(server_ids, hypervisor): - host_hypervisors = get_servers_hypervisors(server_ids) +def get_same_host_hypervisors(servers, hypervisor): + host_hypervisors = get_servers_hypervisors(servers) same_host_server_ids = host_hypervisors.pop(hypervisor, None) if same_host_server_ids: return {hypervisor: same_host_server_ids} @@ -63,17 +63,27 @@ def get_same_host_hypervisors(server_ids, hypervisor): return {} -def get_different_host_hypervisors(server_ids, hypervisor): - host_hypervisors = get_servers_hypervisors(server_ids) +def get_different_host_hypervisors(servers, hypervisor): + host_hypervisors = get_servers_hypervisors(servers) host_hypervisors.pop(hypervisor, None) return host_hypervisors -def get_servers_hypervisors(server_ids): +def get_servers_hypervisors(servers, client=None): hypervisors = collections.defaultdict(list) - if server_ids: - for server_id in (server_ids or list()): - server = _client.get_server(server_id) - hypervisor = getattr(server, 'OS-EXT-SRV-ATTR:host') - hypervisors[hypervisor].append(server_id) + for server in (servers or list()): + client = _client.nova_client(client) + if isinstance(server, str): + server_id = server + server = _client.get_server(server_id, client=client) + else: + server_id = server.id + hypervisor = get_server_hypervisor(server) + hypervisors[hypervisor].append(server_id) return hypervisors + + +def get_server_hypervisor(server, client=None): + if isinstance(server, str): + server = _client.get_server(server, client=client) + return getattr(server, 'OS-EXT-SRV-ATTR:host') diff --git a/tobiko/openstack/stacks/_nova.py b/tobiko/openstack/stacks/_nova.py index ae3551899..639b87366 100644 --- a/tobiko/openstack/stacks/_nova.py +++ b/tobiko/openstack/stacks/_nova.py @@ -19,6 +19,7 @@ import os import typing # noqa import six +from oslo_log import log import tobiko from tobiko import config @@ -32,6 +33,7 @@ from tobiko.shell import sh CONF = config.CONF +LOG = log.getLogger(__name__) class KeyPairStackFixture(heat.HeatStackFixture): @@ -266,8 +268,10 @@ class ServerStackFixture(heat.HeatStackFixture): tobiko.setup_fixture(self) try: server = nova.wait_for_server_status(self.server_id, status) - except nova.ServerStatusTimeout: + except nova.WaitForServerStatusError: server = nova.get_server(self.server_id) + LOG.debug(f"Server {server.id} status is {server.status} instead " + f"of {status}", exc_info=1) if server.status == status: return server elif status == "ACTIVE": diff --git a/tobiko/tests/functional/openstack/test_nova.py b/tobiko/tests/functional/openstack/test_nova.py index f71109cde..f975e3a7b 100644 --- a/tobiko/tests/functional/openstack/test_nova.py +++ b/tobiko/tests/functional/openstack/test_nova.py @@ -21,8 +21,10 @@ import netaddr import testtools import tobiko +from tobiko.openstack import keystone from tobiko.openstack import nova from tobiko.openstack import stacks +from tobiko.shell import ping class KeyPairTest(testtools.TestCase): @@ -156,3 +158,63 @@ class ServiceTest(testtools.TestCase): def test_wait_for_services_up(self): nova.wait_for_services_up() + + +class MigrateServerStack(stacks.CirrosServerStackFixture): + pass + + +@keystone.skip_unless_has_keystone_credentials() +@nova.skip_if_missing_hypervisors(count=2) +class MigrateServerTest(testtools.TestCase): + + stack = tobiko.required_setup_fixture(MigrateServerStack) + + def test_migrate_server(self): + """Tests cold migration actually changes hypervisor + """ + server = self.setup_server() + initial_hypervisor = nova.get_server_hypervisor(server) + + server = self.migrate_server(server) + + final_hypervisor = nova.get_server_hypervisor(server) + self.assertNotEqual(initial_hypervisor, final_hypervisor) + + def test_migrate_server_with_host(self): + """Tests cold migration actually ends on target hypervisor + """ + server = self.setup_server() + initial_hypervisor = nova.get_server_hypervisor(server) + for hypervisor in nova.list_hypervisors(status='enabled', state='up'): + if initial_hypervisor != hypervisor.hypervisor_hostname: + target_hypervisor = hypervisor.hypervisor_hostname + break + else: + self.skip("Cannot find a valid hypervisor host to migrate server " + "to") + + server = self.migrate_server(server=server, host=target_hypervisor) + + final_hypervisor = nova.get_server_hypervisor(server) + self.assertEqual(target_hypervisor, final_hypervisor) + + def setup_server(self): + server = self.stack.ensure_server_status('ACTIVE') + self.assertEqual('ACTIVE', server.status) + return server + + def migrate_server(self, server, **params): + self.assertEqual('ACTIVE', server.status) + nova.migrate_server(server, **params) + + server = nova.wait_for_server_status(server, 'VERIFY_RESIZE') + self.assertEqual('VERIFY_RESIZE', server.status) + nova.confirm_resize(server) + + server = nova.wait_for_server_status( + server, 'ACTIVE', transient_status={'VERIFY_RESIZE'}) + self.assertEqual('ACTIVE', server.status) + + ping.assert_reachable_hosts([self.stack.ip_address]) + return server