Implement Nova Server migration tools

Change-Id: I9d794da07cebffa769f9ac4b4d7df3ebb7765334
This commit is contained in:
Federico Ressi 2020-09-23 14:45:26 +02:00
parent efe9febf9a
commit fa7fc4b342
5 changed files with 157 additions and 23 deletions

View File

@ -34,9 +34,12 @@ list_services = _client.list_services
nova_client = _client.nova_client nova_client = _client.nova_client
NovaClientFixture = _client.NovaClientFixture NovaClientFixture = _client.NovaClientFixture
wait_for_server_status = _client.wait_for_server_status wait_for_server_status = _client.wait_for_server_status
ServerStatusTimeout = _client.ServerStatusTimeout WaitForServerStatusError = _client.WaitForServerStatusError
WaitForServerStatusTimeout = _client.WaitForServerStatusTimeout
shutoff_server = _client.shutoff_server shutoff_server = _client.shutoff_server
activate_server = _client.activate_server activate_server = _client.activate_server
migrate_server = _client.migrate_server
confirm_resize = _client.confirm_resize
WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError
cloud_config = _cloud_init.cloud_config cloud_config = _cloud_init.cloud_config
@ -48,6 +51,8 @@ wait_for_cloud_init_status = _cloud_init.wait_for_cloud_init_status
skip_if_missing_hypervisors = _hypervisor.skip_if_missing_hypervisors skip_if_missing_hypervisors = _hypervisor.skip_if_missing_hypervisors
get_same_host_hypervisors = _hypervisor.get_same_host_hypervisors get_same_host_hypervisors = _hypervisor.get_same_host_hypervisors
get_different_host_hypervisors = _hypervisor.get_different_host_hypervisors get_different_host_hypervisors = _hypervisor.get_different_host_hypervisors
get_server_hypervisor = _hypervisor.get_server_hypervisor
get_servers_hypervisors = _hypervisor.get_servers_hypervisors
find_server_ip_address = _server.find_server_ip_address find_server_ip_address = _server.find_server_ip_address
HasServerMixin = _server.HasServerMixin HasServerMixin = _server.HasServerMixin

View File

@ -108,8 +108,32 @@ def find_service(client=None, unique=False, **params):
return services.first return services.first
def get_server_id(server):
if isinstance(server, str):
return server
else:
return server.id
def get_server(server, client=None, **params): def get_server(server, client=None, **params):
return nova_client(client).servers.get(server, **params) server_id = get_server_id(server)
return nova_client(client).servers.get(server_id, **params)
def migrate_server(server, client=None, **params):
# pylint: disable=protected-access
server_id = get_server_id(server)
LOG.debug(f"Start server migration (server_id='{server_id}', "
f"info={params})")
return nova_client(client).servers._action('migrate', server_id,
info=params)
def confirm_resize(server, client=None, **params):
server_id = get_server_id(server)
LOG.debug(f"Confirm server resize (server_id='{server_id}', "
f"info={params})")
return nova_client(client).servers.confirm_resize(server_id, **params)
MAX_SERVER_CONSOLE_OUTPUT_LENGTH = 1024 * 256 MAX_SERVER_CONSOLE_OUTPUT_LENGTH = 1024 * 256
@ -159,30 +183,52 @@ class HasNovaClientMixin(object):
**params) **params)
class ServerStatusTimeout(tobiko.TobikoException): class WaitForServerStatusError(tobiko.TobikoException):
message = ("Server {server_id} didn't change its status to {status} " message = ("Server {server_id} not changing status from {server_status} "
"status after {timeout} seconds") "to {status}")
class WaitForServerStatusTimeout(WaitForServerStatusError):
message = ("Server {server_id} didn't change its status from "
"{server_status} to {status} status after {timeout} seconds")
NOVA_SERVER_TRANSIENT_STATUS = {
'ACTIVE': ('BUILD', 'SHUTOFF'),
'SHUTOFF': ('ACTIVE'),
'VERIFY_RESIZE': ('RESIZE'),
}
def wait_for_server_status(server, status, client=None, timeout=None, def wait_for_server_status(server, status, client=None, timeout=None,
sleep_time=None): sleep_time=None, transient_status=None):
if timeout is None: if timeout is None:
timeout = 300. timeout = 300.
if sleep_time is None: if sleep_time is None:
sleep_time = 5. sleep_time = 5.
start_time = time.time() start_time = time.time()
if transient_status is None:
transient_status = NOVA_SERVER_TRANSIENT_STATUS.get(status) or tuple()
while True: while True:
server = get_server(server=server, client=client) server = get_server(server=server, client=client)
if server.status == status: if server.status == status:
break break
if time.time() - start_time >= timeout: if server.status not in transient_status:
raise ServerStatusTimeout(server_id=server.id, raise WaitForServerStatusError(server_id=server.id,
status=status, server_status=server.status,
timeout=timeout) status=status)
LOG.debug('Waiting for server %r status to get from %r to %r', if time.time() - start_time >= timeout:
server.id, server.status, status) raise WaitForServerStatusTimeout(server_id=server.id,
server_status=server.status,
status=status,
timeout=timeout)
progress = getattr(server, 'progress', None)
LOG.debug(f"Waiting for server {server.id} status to get from "
f"{server.status} to {status} "
f"(progress={progress}%)")
time.sleep(sleep_time) time.sleep(sleep_time)
return server return server
@ -207,6 +253,13 @@ def activate_server(server, client=None, timeout=None, sleep_time=None):
if server.status == 'SHUTOFF': if server.status == 'SHUTOFF':
client.servers.start(server.id) client.servers.start(server.id)
elif server.status == 'RESIZE':
wait_for_server_status(server=server.id, status='VERIFY_RESIZE',
client=client, timeout=timeout,
sleep_time=sleep_time)
client.servers.confirm_resize(server)
elif server.status == 'VERIFY_RESIZE':
client.servers.confirm_resize(server)
else: else:
client.servers.reboot(server.id, reboot_type='HARD') client.servers.reboot(server.id, reboot_type='HARD')

View File

@ -54,8 +54,8 @@ def skip_if_missing_hypervisors(count=1, **params):
**params) **params)
def get_same_host_hypervisors(server_ids, hypervisor): def get_same_host_hypervisors(servers, hypervisor):
host_hypervisors = get_servers_hypervisors(server_ids) host_hypervisors = get_servers_hypervisors(servers)
same_host_server_ids = host_hypervisors.pop(hypervisor, None) same_host_server_ids = host_hypervisors.pop(hypervisor, None)
if same_host_server_ids: if same_host_server_ids:
return {hypervisor: same_host_server_ids} return {hypervisor: same_host_server_ids}
@ -63,17 +63,27 @@ def get_same_host_hypervisors(server_ids, hypervisor):
return {} return {}
def get_different_host_hypervisors(server_ids, hypervisor): def get_different_host_hypervisors(servers, hypervisor):
host_hypervisors = get_servers_hypervisors(server_ids) host_hypervisors = get_servers_hypervisors(servers)
host_hypervisors.pop(hypervisor, None) host_hypervisors.pop(hypervisor, None)
return host_hypervisors return host_hypervisors
def get_servers_hypervisors(server_ids): def get_servers_hypervisors(servers, client=None):
hypervisors = collections.defaultdict(list) hypervisors = collections.defaultdict(list)
if server_ids: for server in (servers or list()):
for server_id in (server_ids or list()): client = _client.nova_client(client)
server = _client.get_server(server_id) if isinstance(server, str):
hypervisor = getattr(server, 'OS-EXT-SRV-ATTR:host') server_id = server
hypervisors[hypervisor].append(server_id) server = _client.get_server(server_id, client=client)
else:
server_id = server.id
hypervisor = get_server_hypervisor(server)
hypervisors[hypervisor].append(server_id)
return hypervisors return hypervisors
def get_server_hypervisor(server, client=None):
if isinstance(server, str):
server = _client.get_server(server, client=client)
return getattr(server, 'OS-EXT-SRV-ATTR:host')

View File

@ -19,6 +19,7 @@ import os
import typing # noqa import typing # noqa
import six import six
from oslo_log import log
import tobiko import tobiko
from tobiko import config from tobiko import config
@ -32,6 +33,7 @@ from tobiko.shell import sh
CONF = config.CONF CONF = config.CONF
LOG = log.getLogger(__name__)
class KeyPairStackFixture(heat.HeatStackFixture): class KeyPairStackFixture(heat.HeatStackFixture):
@ -266,8 +268,10 @@ class ServerStackFixture(heat.HeatStackFixture):
tobiko.setup_fixture(self) tobiko.setup_fixture(self)
try: try:
server = nova.wait_for_server_status(self.server_id, status) server = nova.wait_for_server_status(self.server_id, status)
except nova.ServerStatusTimeout: except nova.WaitForServerStatusError:
server = nova.get_server(self.server_id) server = nova.get_server(self.server_id)
LOG.debug(f"Server {server.id} status is {server.status} instead "
f"of {status}", exc_info=1)
if server.status == status: if server.status == status:
return server return server
elif status == "ACTIVE": elif status == "ACTIVE":

View File

@ -21,8 +21,10 @@ import netaddr
import testtools import testtools
import tobiko import tobiko
from tobiko.openstack import keystone
from tobiko.openstack import nova from tobiko.openstack import nova
from tobiko.openstack import stacks from tobiko.openstack import stacks
from tobiko.shell import ping
class KeyPairTest(testtools.TestCase): class KeyPairTest(testtools.TestCase):
@ -156,3 +158,63 @@ class ServiceTest(testtools.TestCase):
def test_wait_for_services_up(self): def test_wait_for_services_up(self):
nova.wait_for_services_up() nova.wait_for_services_up()
class MigrateServerStack(stacks.CirrosServerStackFixture):
pass
@keystone.skip_unless_has_keystone_credentials()
@nova.skip_if_missing_hypervisors(count=2)
class MigrateServerTest(testtools.TestCase):
stack = tobiko.required_setup_fixture(MigrateServerStack)
def test_migrate_server(self):
"""Tests cold migration actually changes hypervisor
"""
server = self.setup_server()
initial_hypervisor = nova.get_server_hypervisor(server)
server = self.migrate_server(server)
final_hypervisor = nova.get_server_hypervisor(server)
self.assertNotEqual(initial_hypervisor, final_hypervisor)
def test_migrate_server_with_host(self):
"""Tests cold migration actually ends on target hypervisor
"""
server = self.setup_server()
initial_hypervisor = nova.get_server_hypervisor(server)
for hypervisor in nova.list_hypervisors(status='enabled', state='up'):
if initial_hypervisor != hypervisor.hypervisor_hostname:
target_hypervisor = hypervisor.hypervisor_hostname
break
else:
self.skip("Cannot find a valid hypervisor host to migrate server "
"to")
server = self.migrate_server(server=server, host=target_hypervisor)
final_hypervisor = nova.get_server_hypervisor(server)
self.assertEqual(target_hypervisor, final_hypervisor)
def setup_server(self):
server = self.stack.ensure_server_status('ACTIVE')
self.assertEqual('ACTIVE', server.status)
return server
def migrate_server(self, server, **params):
self.assertEqual('ACTIVE', server.status)
nova.migrate_server(server, **params)
server = nova.wait_for_server_status(server, 'VERIFY_RESIZE')
self.assertEqual('VERIFY_RESIZE', server.status)
nova.confirm_resize(server)
server = nova.wait_for_server_status(
server, 'ACTIVE', transient_status={'VERIFY_RESIZE'})
self.assertEqual('ACTIVE', server.status)
ping.assert_reachable_hosts([self.stack.ip_address])
return server