x/tobiko
Code Issues Proposed changes

Implement Nova Server migration tools

Browse Source 
Change-Id: I9d794da07cebffa769f9ac4b4d7df3ebb7765334
This commit is contained in:
Federico Ressi 2020-09-23 14:45:26 +02:00
parent efe9febf9a
commit fa7fc4b342
5 changed files with 157 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tobiko/openstack/nova/__init__.py
View File

@ -34,9 +34,12 @@ list_services = _client.list_services
nova_client = _client.nova_client
NovaClientFixture = _client.NovaClientFixture
wait_for_server_status = _client.wait_for_server_status
ServerStatusTimeout = _client.ServerStatusTimeout
WaitForServerStatusError = _client.WaitForServerStatusError
WaitForServerStatusTimeout = _client.WaitForServerStatusTimeout
shutoff_server = _client.shutoff_server
activate_server = _client.activate_server
migrate_server = _client.migrate_server
confirm_resize = _client.confirm_resize
WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError
cloud_config = _cloud_init.cloud_config
@ -48,6 +51,8 @@ wait_for_cloud_init_status = _cloud_init.wait_for_cloud_init_status
skip_if_missing_hypervisors = _hypervisor.skip_if_missing_hypervisors
get_same_host_hypervisors = _hypervisor.get_same_host_hypervisors
get_different_host_hypervisors = _hypervisor.get_different_host_hypervisors
get_server_hypervisor = _hypervisor.get_server_hypervisor
get_servers_hypervisors = _hypervisor.get_servers_hypervisors
find_server_ip_address = _server.find_server_ip_address
HasServerMixin = _server.HasServerMixin

75
tobiko/openstack/nova/_client.py
View File

@ -108,8 +108,32 @@ def find_service(client=None, unique=False, **params):
return services.first
def get_server_id(server):
if isinstance(server, str):
return server
else:
return server.id
def get_server(server, client=None, **params):
return nova_client(client).servers.get(server, **params)
server_id = get_server_id(server)
return nova_client(client).servers.get(server_id, **params)
def migrate_server(server, client=None, **params):
# pylint: disable=protected-access
server_id = get_server_id(server)
LOG.debug(f"Start server migration (server_id='{server_id}', "
f"info={params})")
return nova_client(client).servers._action('migrate', server_id,
info=params)
def confirm_resize(server, client=None, **params):
server_id = get_server_id(server)
LOG.debug(f"Confirm server resize (server_id='{server_id}', "
f"info={params})")
return nova_client(client).servers.confirm_resize(server_id, **params)
MAX_SERVER_CONSOLE_OUTPUT_LENGTH = 1024 * 256
@ -159,30 +183,52 @@ class HasNovaClientMixin(object):
**params)
class ServerStatusTimeout(tobiko.TobikoException):
message = ("Server {server_id} didn't change its status to {status} "
"status after {timeout} seconds")
class WaitForServerStatusError(tobiko.TobikoException):
message = ("Server {server_id} not changing status from {server_status} "
"to {status}")
class WaitForServerStatusTimeout(WaitForServerStatusError):
message = ("Server {server_id} didn't change its status from "
"{server_status} to {status} status after {timeout} seconds")
NOVA_SERVER_TRANSIENT_STATUS = {
'ACTIVE': ('BUILD', 'SHUTOFF'),
'SHUTOFF': ('ACTIVE'),
'VERIFY_RESIZE': ('RESIZE'),
}
def wait_for_server_status(server, status, client=None, timeout=None,
sleep_time=None):
sleep_time=None, transient_status=None):
if timeout is None:
timeout = 300.
if sleep_time is None:
sleep_time = 5.
start_time = time.time()
if transient_status is None:
transient_status = NOVA_SERVER_TRANSIENT_STATUS.get(status) or tuple()
while True:
server = get_server(server=server, client=client)
if server.status == status:
break
if time.time() - start_time >= timeout:
raise ServerStatusTimeout(server_id=server.id,
status=status,
timeout=timeout)
if server.status not in transient_status:
raise WaitForServerStatusError(server_id=server.id,
server_status=server.status,
status=status)
LOG.debug('Waiting for server %r status to get from %r to %r',
server.id, server.status, status)
if time.time() - start_time >= timeout:
raise WaitForServerStatusTimeout(server_id=server.id,
server_status=server.status,
status=status,
timeout=timeout)
progress = getattr(server, 'progress', None)
LOG.debug(f"Waiting for server {server.id} status to get from "
f"{server.status} to {status} "
f"(progress={progress}%)")
time.sleep(sleep_time)
return server
@ -207,6 +253,13 @@ def activate_server(server, client=None, timeout=None, sleep_time=None):
if server.status == 'SHUTOFF':
client.servers.start(server.id)
elif server.status == 'RESIZE':
wait_for_server_status(server=server.id, status='VERIFY_RESIZE',
client=client, timeout=timeout,
sleep_time=sleep_time)
client.servers.confirm_resize(server)
elif server.status == 'VERIFY_RESIZE':
client.servers.confirm_resize(server)
else:
client.servers.reboot(server.id, reboot_type='HARD')

30
tobiko/openstack/nova/_hypervisor.py
View File

@ -54,8 +54,8 @@ def skip_if_missing_hypervisors(count=1, **params):
**params)
def get_same_host_hypervisors(server_ids, hypervisor):
host_hypervisors = get_servers_hypervisors(server_ids)
def get_same_host_hypervisors(servers, hypervisor):
host_hypervisors = get_servers_hypervisors(servers)
same_host_server_ids = host_hypervisors.pop(hypervisor, None)
if same_host_server_ids:
return {hypervisor: same_host_server_ids}
@ -63,17 +63,27 @@ def get_same_host_hypervisors(server_ids, hypervisor):
return {}
def get_different_host_hypervisors(server_ids, hypervisor):
host_hypervisors = get_servers_hypervisors(server_ids)
def get_different_host_hypervisors(servers, hypervisor):
host_hypervisors = get_servers_hypervisors(servers)
host_hypervisors.pop(hypervisor, None)
return host_hypervisors
def get_servers_hypervisors(server_ids):
def get_servers_hypervisors(servers, client=None):
hypervisors = collections.defaultdict(list)
if server_ids:
for server_id in (server_ids or list()):
server = _client.get_server(server_id)
hypervisor = getattr(server, 'OS-EXT-SRV-ATTR:host')
hypervisors[hypervisor].append(server_id)
for server in (servers or list()):
client = _client.nova_client(client)
if isinstance(server, str):
server_id = server
server = _client.get_server(server_id, client=client)
else:
server_id = server.id
hypervisor = get_server_hypervisor(server)
hypervisors[hypervisor].append(server_id)
return hypervisors
def get_server_hypervisor(server, client=None):
if isinstance(server, str):
server = _client.get_server(server, client=client)
return getattr(server, 'OS-EXT-SRV-ATTR:host')

6
tobiko/openstack/stacks/_nova.py
View File

@ -19,6 +19,7 @@ import os
import typing # noqa
import six
from oslo_log import log
import tobiko
from tobiko import config
@ -32,6 +33,7 @@ from tobiko.shell import sh
CONF = config.CONF
LOG = log.getLogger(__name__)
class KeyPairStackFixture(heat.HeatStackFixture):
@ -266,8 +268,10 @@ class ServerStackFixture(heat.HeatStackFixture):
tobiko.setup_fixture(self)
try:
server = nova.wait_for_server_status(self.server_id, status)
except nova.ServerStatusTimeout:
except nova.WaitForServerStatusError:
server = nova.get_server(self.server_id)
LOG.debug(f"Server {server.id} status is {server.status} instead "
f"of {status}", exc_info=1)
if server.status == status:
return server
elif status == "ACTIVE":

62
tobiko/tests/functional/openstack/test_nova.py
View File

@ -21,8 +21,10 @@ import netaddr
import testtools
import tobiko
from tobiko.openstack import keystone
from tobiko.openstack import nova
from tobiko.openstack import stacks
from tobiko.shell import ping
class KeyPairTest(testtools.TestCase):
@ -156,3 +158,63 @@ class ServiceTest(testtools.TestCase):
def test_wait_for_services_up(self):
nova.wait_for_services_up()
class MigrateServerStack(stacks.CirrosServerStackFixture):
pass
@keystone.skip_unless_has_keystone_credentials()
@nova.skip_if_missing_hypervisors(count=2)
class MigrateServerTest(testtools.TestCase):
stack = tobiko.required_setup_fixture(MigrateServerStack)
def test_migrate_server(self):
"""Tests cold migration actually changes hypervisor
"""
server = self.setup_server()
initial_hypervisor = nova.get_server_hypervisor(server)
server = self.migrate_server(server)
final_hypervisor = nova.get_server_hypervisor(server)
self.assertNotEqual(initial_hypervisor, final_hypervisor)
def test_migrate_server_with_host(self):
"""Tests cold migration actually ends on target hypervisor
"""
server = self.setup_server()
initial_hypervisor = nova.get_server_hypervisor(server)
for hypervisor in nova.list_hypervisors(status='enabled', state='up'):
if initial_hypervisor != hypervisor.hypervisor_hostname:
target_hypervisor = hypervisor.hypervisor_hostname
break
else:
self.skip("Cannot find a valid hypervisor host to migrate server "
"to")
server = self.migrate_server(server=server, host=target_hypervisor)
final_hypervisor = nova.get_server_hypervisor(server)
self.assertEqual(target_hypervisor, final_hypervisor)
def setup_server(self):
server = self.stack.ensure_server_status('ACTIVE')
self.assertEqual('ACTIVE', server.status)
return server
def migrate_server(self, server, **params):
self.assertEqual('ACTIVE', server.status)
nova.migrate_server(server, **params)
server = nova.wait_for_server_status(server, 'VERIFY_RESIZE')
self.assertEqual('VERIFY_RESIZE', server.status)
nova.confirm_resize(server)
server = nova.wait_for_server_status(
server, 'ACTIVE', transient_status={'VERIFY_RESIZE'})
self.assertEqual('ACTIVE', server.status)
ping.assert_reachable_hosts([self.stack.ip_address])
return server