openstack
/
trove
Code Issues Proposed changes

Checks guest status during migration. 

During a migration, waits for the guest and service to start up
following the status change to VERIFY_RESIZE. Confirms if everything
is all right, or reverts if not.

Implements blueprint migration-check-guest-status

Change-Id: Ia7c7ed1fd0070429fed93323ca559d1c0742bd8f
This commit is contained in:
Ed Cranford 2013-01-02 16:51:40 -06:00
parent bdb54cc494
commit ce6b98e2d1
18 changed files with 543 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/reddwarf/reddwarf.conf.test
View File

@ -81,6 +81,8 @@ agent_call_high_timeout = 100
server_delete_time_out=10
use_nova_server_volume = False
dns_time_out = 120
resize_time_out = 120
# ============ notifer queue kombu connection options ========================

2
reddwarf/common/cfg.py
View File

@ -88,6 +88,8 @@ common_opts = [
cfg.IntOpt('volume_time_out', default=2),
cfg.IntOpt('reboot_time_out', default=60 * 2),
cfg.StrOpt('service_options', default=['mysql']),
cfg.IntOpt('dns_time_out', default=60 * 2),
cfg.IntOpt('resize_time_out', default=60 * 10),
]

1
reddwarf/common/wsgi.py
View File

@ -17,7 +17,6 @@
"""Wsgi helper utilities for reddwarf"""
import eventlet.wsgi
import os
import paste.urlmap
import re
import traceback

5
reddwarf/guestagent/api.py
View File

@ -188,10 +188,11 @@ class API(proxy.RpcProxy):
self._call("start_mysql_with_conf_changes", AGENT_HIGH_TIMEOUT,
updated_memory_size=updated_memory_size)
def stop_mysql(self):
def stop_mysql(self, do_not_start_on_reboot=False):
"""Stop the MySQL server."""
LOG.debug(_("Sending the call to stop MySQL on the Guest."))
self._call("stop_mysql", AGENT_HIGH_TIMEOUT)
self._call("stop_mysql", AGENT_HIGH_TIMEOUT,
do_not_start_on_reboot=do_not_start_on_reboot)
def upgrade(self):
"""Make an asynchronous call to self upgrade the guest agent"""

28
reddwarf/guestagent/dbaas.py
View File

@ -587,8 +587,32 @@ class MySqlApp(object):
LOG.debug(_("Finished installing mysql server"))
#TODO(rnirmal): Add checks to make sure the package got installed
def stop_mysql(self, update_db=False):
def _enable_mysql_on_boot(self):
'''
# This works in Debian Squeeze, but Ubuntu Precise has other plans.
# Use update-rc.d to enable or disable mysql at boot.
# update-rc.d is idempotent; any substitute method should be, too.
flag = "enable" if enabled else "disable"
LOG.info("Setting mysql to '%s' in rc.d" % flag)
utils.execute_with_timeout("sudo", "update-rc.d", "mysql", flag)
'''
LOG.info("Enabling mysql on boot.")
conf = "/etc/init/mysql.conf"
command = "sudo sed -i '/^manual$/d' %(conf)s"
command = command % locals()
utils.execute_with_timeout(command, with_shell=True)
def _disable_mysql_on_boot(self):
LOG.info("Disabling mysql on boot.")
conf = "/etc/init/mysql.conf"
command = '''sudo sh -c "echo manual >> %(conf)s"'''
command = command % locals()
utils.execute_with_timeout(command, with_shell=True)
def stop_mysql(self, update_db=False, do_not_start_on_reboot=False):
LOG.info(_("Stopping mysql..."))
if do_not_start_on_reboot:
self._disable_mysql_on_boot()
utils.execute_with_timeout("sudo", "/etc/init.d/mysql", "stop")
if not self.status.wait_for_real_status_to_change_to(
rd_models.ServiceStatuses.SHUTDOWN,
@ -712,6 +736,8 @@ class MySqlApp(object):
# Essentially what happens is thaty mysql start fails, but does not
# die. It is then impossible to kill the original, so
self._enable_mysql_on_boot()
try:
utils.execute_with_timeout("sudo", "/etc/init.d/mysql", "start")
except ProcessExecutionError:

4
reddwarf/guestagent/manager.py
View File

@ -80,9 +80,9 @@ class Manager(periodic_task.PeriodicTasks):
app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())
app.start_mysql_with_conf_changes(updated_memory_size)
def stop_mysql(self, context):
def stop_mysql(self, context, do_not_start_on_reboot=False):
app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())
app.stop_mysql()
app.stop_mysql(do_not_start_on_reboot=do_not_start_on_reboot)
def get_filesystem_stats(self, context, fs_path):
""" Gets the filesystem stats for the path given """

3
reddwarf/instance/tasks.py
View File

@ -59,6 +59,9 @@ class InstanceTask(object):
return None
return cls._lookup[code]
def __str__(self):
return "(%d %s %s)" % (self._code, self._action, self._db_text)
class InstanceTasks(object):
NONE = InstanceTask(0x01, 'NONE', 'No tasks for the instance.')

4
reddwarf/openstack/common/processutils.py
View File

@ -86,6 +86,7 @@ def execute(*cmd, **kwargs):
attempts = kwargs.pop('attempts', 1)
run_as_root = kwargs.pop('run_as_root', False)
root_helper = kwargs.pop('root_helper', '')
with_shell = kwargs.pop('with_shell', False)
if len(kwargs):
raise UnknownArgumentError(_('Got unknown keyword args '
'to utils.execute: %r') % kwargs)
@ -102,7 +103,8 @@ def execute(*cmd, **kwargs):
stdin=_PIPE,
stdout=_PIPE,
stderr=_PIPE,
close_fds=True)
close_fds=True,
shell=with_shell)
result = None
if process_input is not None:
result = obj.communicate(process_input)

270
reddwarf/taskmanager/models.py
View File

@ -47,6 +47,10 @@ from reddwarf.openstack.common.gettextutils import _
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
VOLUME_TIME_OUT = CONF.volume_time_out # seconds.
DNS_TIME_OUT = CONF.dns_time_out # seconds.
RESIZE_TIME_OUT = CONF.resize_time_out # seconds.
use_nova_server_volume = CONF.use_nova_server_volume
@ -179,7 +183,7 @@ class FreshInstanceTasks(FreshInstance):
lambda: volume_client.volumes.get(volume_ref.id),
lambda v_ref: v_ref.status in ['available', 'error'],
sleep_time=2,
time_out=2 * 60)
time_out=VOLUME_TIME_OUT)
v_ref = volume_client.volumes.get(volume_ref.id)
if v_ref.status in ['error']:
@ -257,7 +261,7 @@ class FreshInstanceTasks(FreshInstance):
LOG.error(msg % (self.id, server.status))
raise ReddwarfError(status=server.status)
poll_until(get_server, ip_is_available,
sleep_time=1, time_out=60 * 2)
sleep_time=1, time_out=DNS_TIME_OUT)
server = nova_client.servers.get(self.db_info.compute_instance_id)
LOG.info("Creating dns entry...")
dns_client.create_instance_entry(self.id,
@ -328,7 +332,6 @@ class BuiltInstanceTasks(BuiltInstance):
self.update_db(volume_size=volume.size)
self.nova_client.volumes.rescan_server_volume(self.server,
self.volume_id)
self.guest.resize_fs(self.get_volume_mountpoint())
except PollTimeOut as pto:
LOG.error("Timeout trying to rescan or resize the attached volume "
"filesystem for volume: %s" % self.volume_id)
@ -342,104 +345,12 @@ class BuiltInstanceTasks(BuiltInstance):
def resize_flavor(self, new_flavor_id, old_memory_size,
new_memory_size):
self._resize_flavor(new_flavor_id, old_memory_size,
new_memory_size)
action = ResizeAction(self, new_flavor_id, new_memory_size)
action.execute()
def migrate(self):
self._resize_flavor()
def _resize_flavor(self, new_flavor_id=None, old_memory_size=None,
new_memory_size=None):
def resize_status_msg():
return "instance_id=%s, status=%s, flavor_id=%s, "\
"dest. flavor id=%s)" % (self.db_info.id,
self.server.status,
str(self.server.flavor['id']),
str(new_flavor_id))
try:
LOG.debug("Instance %s calling stop_mysql..." % self.db_info.id)
self.guest.stop_mysql()
try:
LOG.debug("Instance %s calling Compute resize..."
% self.db_info.id)
if new_flavor_id:
LOG.debug("Instance with new flavor id")
self.server.resize(new_flavor_id)
else:
LOG.debug("Migrating instance %s without flavor change ..."
% self.db_info.id)
self.server.migrate()
LOG.debug("refreshing the compute status of the instance")
# Do initial check and confirm the status is appropriate.
self._refresh_compute_server_info()
if (self.server.status != "RESIZE" and
self.server.status != "VERIFY_RESIZE"):
msg = "Unexpected status after call to resize! : %s"
raise ReddwarfError(msg % resize_status_msg())
LOG.debug("the compute status of the instance : (%s)"
% self.server.status)
# Wait for the flavor to change.
def update_server_info():
self._refresh_compute_server_info()
LOG.debug("refreshed... compute status (%s)"
% self.server.status)
return self.server.status != 'RESIZE'
LOG.debug("polling the server until its not RESIZE")
utils.poll_until(
update_server_info,
sleep_time=2,
time_out=60 * 10)
LOG.debug("compute status should not be RESIZE now")
LOG.debug("instance_id=%s, status=%s, "
"dest. flavor id=%s)" % (self.db_info.id,
self.server.status,
str(new_flavor_id)))
# Do check to make sure the status and flavor id are correct.
if new_flavor_id:
if str(self.server.flavor['id']) != str(new_flavor_id):
msg = ("Assertion failed! flavor_id=%s and not %s"
% (self.server.flavor['id'], new_flavor_id))
raise ReddwarfError(msg)
if (self.server.status != "VERIFY_RESIZE"):
msg = ("Assertion failed! status=%s and not %s"
% (self.server.status, 'VERIFY_RESIZE'))
raise ReddwarfError(msg)
LOG.debug("wait a sec man!!!")
time.sleep(5)
# Confirm the resize with Nova.
LOG.debug("Instance %s calling Compute confirm resize..."
% self.db_info.id)
self.server.confirm_resize()
LOG.debug("Compute confirm resize DONE ...")
if new_flavor_id:
# Record the new flavor_id in our database.
LOG.debug("Updating instance %s to flavor_id %s."
% (self.id, new_flavor_id))
self.update_db(flavor_id=new_flavor_id)
except Exception as ex:
new_memory_size = old_memory_size
new_flavor_id = None
LOG.exception("Error resizing instance %s." % self.db_info.id)
finally:
# Tell the guest to restart MySQL with the new RAM size.
# This is in the finally because we have to call this, or
# else MySQL could stay turned off on an otherwise usable
# instance.
LOG.debug("Instance %s starting mysql..." % self.db_info.id)
if new_flavor_id:
self.guest.start_mysql_with_conf_changes(new_memory_size)
else:
self.guest.restart()
finally:
self.update_db(task_status=inst_models.InstanceTasks.NONE)
action = MigrateAction(self)
action.execute()
def reboot(self):
try:
@ -461,9 +372,7 @@ class BuiltInstanceTasks(BuiltInstance):
# Set the status to PAUSED. The guest agent will reset the status
# when the reboot completes and MySQL is running.
status = InstanceServiceStatus.find_by(instance_id=self.id)
status.set_status(inst_models.ServiceStatuses.PAUSED)
status.save()
self._set_service_status_to_paused()
LOG.debug("Successfully rebooted instance %s" % self.id)
except Exception, e:
LOG.error("Failed to reboot instance %s: %s" % (self.id, str(e)))
@ -486,3 +395,160 @@ class BuiltInstanceTasks(BuiltInstance):
"""Refreshes the compute server field."""
server = self.nova_client.servers.get(self.server.id)
self.server = server
def _refresh_compute_service_status(self):
"""Refreshes the service status info for an instance."""
service = InstanceServiceStatus.find_by(instance_id=self.id)
self.service_status = service.get_status()
def _set_service_status_to_paused(self):
status = InstanceServiceStatus.find_by(instance_id=self.id)
status.set_status(inst_models.ServiceStatuses.PAUSED)
status.save()
class ResizeActionBase(object):
"""Base class for executing a resize action."""
def __init__(self, instance):
self.instance = instance
def _assert_guest_is_ok(self):
# The guest will never set the status to PAUSED.
self.instance._set_service_status_to_paused()
# Now we wait until it sets it to anything at all,
# so we know it's alive.
utils.poll_until(
self._guest_is_awake,
sleep_time=2,
time_out=RESIZE_TIME_OUT)
def _assert_nova_was_successful(self):
# Make sure Nova thinks things went well.
if self.instance.server.status != "VERIFY_RESIZE":
msg = "Migration failed! status=%s and not %s" \
% (self.instance.server.status, 'VERIFY_RESIZE')
raise ReddwarfError(msg)
def _assert_mysql_is_ok(self):
# Tell the guest to turn on MySQL, and ensure the status becomes
# ACTIVE.
self._start_mysql()
# The guest should do this for us... but sometimes it walks funny.
self.instance._refresh_compute_service_status()
if self.instance.service_status != ServiceStatuses.RUNNING:
raise Exception("Migration failed! Service status was %s."
% self.instance.service_status)
def _assert_processes_are_ok(self):
"""Checks the procs; if anything is wrong, reverts the operation."""
# Tell the guest to turn back on, and make sure it can start.
self._assert_guest_is_ok()
LOG.debug("Nova guest is fine.")
self._assert_mysql_is_ok()
LOG.debug("Mysql is good, too.")
def _confirm_nova_action(self):
LOG.debug("Instance %s calling Compute confirm resize..."
% self.instance.id)
self.instance.server.confirm_resize()
def _revert_nova_action(self):
LOG.debug("Instance %s calling Compute revert resize..."
% self.instance.id)
self.instance.server.revert_resize()
def execute(self):
"""Initiates the action."""
try:
LOG.debug("Instance %s calling stop_mysql..."
% self.instance.id)
self.instance.guest.stop_mysql(do_not_start_on_reboot=True)
self._perform_nova_action()
finally:
self.instance.update_db(task_status=inst_models.InstanceTasks.NONE)
def _guest_is_awake(self):
self.instance._refresh_compute_service_status()
return self.instance.service_status != ServiceStatuses.PAUSED
def _perform_nova_action(self):
"""Calls Nova to resize or migrate an instance, and confirms."""
need_to_revert = False
try:
LOG.debug("Initiating nova action")
self._initiate_nova_action()
LOG.debug("Waiting for nova action")
self._wait_for_nova_action()
LOG.debug("Asserting success")
self._assert_nova_was_successful()
LOG.debug("Asserting processes are OK")
need_to_revert = True
LOG.debug("* * * REVERT BARRIER PASSED * * *")
self._assert_processes_are_ok()
LOG.debug("Confirming nova action")
self._confirm_nova_action()
except Exception as ex:
LOG.exception("Exception during nova action.")
if need_to_revert:
LOG.error("Reverting action for instance %s" %
self.instance.id)
self._revert_nova_action()
self.instance.guest.restart()
LOG.error("Error resizing instance %s." % self.instance.id)
raise ex
LOG.debug("Recording success")
self._record_action_success()
def _wait_for_nova_action(self):
# Wait for the flavor to change.
def update_server_info():
self.instance._refresh_compute_server_info()
return self.instance.server.status != 'RESIZE'
utils.poll_until(
update_server_info,
sleep_time=2,
time_out=RESIZE_TIME_OUT)
class ResizeAction(ResizeActionBase):
def __init__(self, instance, new_flavor_id=None, new_memory_size=None):
self.instance = instance
self.new_flavor_id = new_flavor_id
self.new_memory_size = new_memory_size
def _assert_nova_was_successful(self):
# Do check to make sure the status and flavor id are correct.
if str(self.instance.server.flavor['id']) != str(self.new_flavor_id):
msg = "Assertion failed! flavor_id=%s and not %s" \
% (self.instance.server.flavor['id'], self.new_flavor_id)
raise ReddwarfError(msg)
super(ResizeAction, self)._assert_nova_was_successful()
def _initiate_nova_action(self):
self.instance.server.resize(self.new_flavor_id)
def _record_action_success(self):
LOG.debug("Updating instance %s to flavor_id %s."
% (self.instance.id, self.new_flavor_id))
self.instance.update_db(flavor_id=self.new_flavor_id)
def _start_mysql(self):
self.instance.guest.start_mysql_with_conf_changes(self.new_memory_size)
class MigrateAction(ResizeActionBase):
def _initiate_nova_action(self):
LOG.debug("Migrating instance %s without flavor change ..."
% self.instance.id)
self.instance.server.migrate()
def _record_action_success(self):
LOG.debug("Successfully finished Migration to %s: %s" %
(self.hostname, self.instance.id))
def _start_mysql(self):
self.instance.guest.restart()

26
reddwarf/tests/api/instances.py
View File

@ -241,7 +241,7 @@ class CreateInstance(unittest.TestCase):
def test_instance_size_too_big(self):
vol_ok = CONFIG.get('reddwarf_volume_support', False)
if 'reddwarf_max_accepted_volume_size' in CONFIG.values and vol_ok:
too_big = CONFIG.values['reddwarf_max_accepted_volume_size']
too_big = CONFIG.reddwarf_max_accepted_volume_size
assert_raises(exceptions.OverLimit, dbaas.instances.create,
"way_too_large", instance_info.dbaas_flavor_href,
{'size': too_big + 1}, [])
@ -296,7 +296,7 @@ class CreateInstance(unittest.TestCase):
'name', 'status', 'updated']
if CONFIG.values['reddwarf_volume_support']:
expected_attrs.append('volume')
if CONFIG.values['reddwarf_dns_support']:
if CONFIG.reddwarf_dns_support:
expected_attrs.append('hostname')
with CheckInstance(result._info) as check:
@ -510,7 +510,7 @@ class TestGuestProcess(object):
Test that the guest process is started with all the right parameters
"""
@test(enabled=CONFIG.values['use_local_ovz'])
@test(enabled=CONFIG.use_local_ovz)
@time_out(60 * 10)
def check_process_alive_via_local_ovz(self):
init_re = ("[\w\W\|\-\s\d,]*nova-guest "
@ -559,8 +559,8 @@ class TestGuestProcess(object):
@test(depends_on_classes=[CreateInstance],
groups=[GROUP, GROUP_START,
GROUP_START_SIMPLE, GROUP_TEST, "nova.volumes.instance"],
groups=[GROUP, GROUP_START, GROUP_START_SIMPLE, GROUP_TEST,
"nova.volumes.instance"],
enabled=CONFIG.white_box)
class TestVolume(unittest.TestCase):
"""Make sure the volume is attached to instance correctly."""
@ -638,6 +638,14 @@ class TestInstanceListing(object):
check.links(instance_dict['links'])
check.used_volume()
@test(enabled=CONFIG.reddwarf_dns_support)
def test_instance_hostname(self):
instance = dbaas.instances.get(instance_info.id)
assert_equal(200, dbaas.last_http_code)
hostname_prefix = ("%s" % (hashlib.sha1(instance.id).hexdigest()))
instance_hostname_prefix = instance.hostname.split('.')[0]
assert_equal(hostname_prefix, instance_hostname_prefix)
@test
def test_get_instance_status(self):
result = dbaas.instances.get(instance_info.id)
@ -685,7 +693,7 @@ class TestInstanceListing(object):
assert_raises(exceptions.NotFound,
self.other_client.instances.delete, instance_info.id)
@test(enabled=CONFIG.values['test_mgmt'])
@test(enabled=CONFIG.test_mgmt)
def test_mgmt_get_instance_after_started(self):
result = dbaas_admin.management.show(instance_info.id)
expected_attrs = ['account_id', 'addresses', 'created', 'databases',
@ -756,8 +764,8 @@ class DeleteInstance(object):
except exceptions.NotFound:
pass
except Exception as ex:
fail("A failure occured when trying to GET instance %s for the %d "
"time: %s" % (str(instance_info.id), attempts, str(ex)))
fail("A failure occured when trying to GET instance %s for the %d"
" time: %s" % (str(instance_info.id), attempts, str(ex)))
@time_out(30)
@test(enabled=CONFIG.values["reddwarf_volume_support"],
@ -779,7 +787,7 @@ class DeleteInstance(object):
@test(depends_on_classes=[CreateInstance, VerifyGuestStarted,
WaitForGuestInstallationToFinish],
groups=[GROUP, GROUP_START, GROUP_START_SIMPLE],
enabled=CONFIG.values['test_mgmt'])
enabled=CONFIG.test_mgmt)
class VerifyInstanceMgmtInfo(object):
@before_class

41
reddwarf/tests/api/instances_actions.py
View File

@ -18,12 +18,7 @@ import time
from proboscis import after_class
from proboscis import before_class
from proboscis import test
from proboscis.asserts import assert_equal
from proboscis.asserts import assert_false
from proboscis.asserts import assert_not_equal
from proboscis.asserts import assert_raises
from proboscis.asserts import assert_true
from proboscis.asserts import fail
from proboscis.asserts import *
from proboscis.decorators import time_out
from proboscis import SkipTest
@ -50,11 +45,11 @@ GROUP_RESTART = "dbaas.api.instances.actions.restart"
GROUP_STOP_MYSQL = "dbaas.api.instances.actions.stop"
MYSQL_USERNAME = "test_user"
MYSQL_PASSWORD = "abcde"
FAKE_MODE = CONFIG.values['fake_mode']
FAKE_MODE = CONFIG.fake_mode
# If true, then we will actually log into the database.
USE_IP = not CONFIG.values['fake_mode']
USE_IP = not FAKE_MODE
# If true, then we will actually search for the process
USE_LOCAL_OVZ = CONFIG.values['use_local_ovz']
USE_LOCAL_OVZ = CONFIG.use_local_ovz
class MySqlConnection(object):
@ -99,10 +94,10 @@ class ActionTestBase(object):
def set_up(self):
"""If you're using this as a base class, call this method first."""
self.dbaas = instance_info.dbaas
if USE_IP:
address = instance_info.get_address()
self.connection = MySqlConnection(address)
self.dbaas = instance_info.dbaas
@property
def instance(self):
@ -181,6 +176,8 @@ class RebootTestBase(ActionTestBase):
"""Wait until our connection breaks."""
if not USE_IP:
return
if not hasattr(self, "connection"):
return
poll_until(self.connection.is_connected,
lambda connected: not connected,
time_out=TIME_OUT_TIME)
@ -189,8 +186,6 @@ class RebootTestBase(ActionTestBase):
"""Wait until status becomes running."""
def is_finished_rebooting():
instance = self.instance
instance.get()
print(instance.status)
if instance.status == "REBOOT":
return False
assert_equal("ACTIVE", instance.status)
@ -277,6 +272,8 @@ class RestartTests(RebootTestBase):
@test(depends_on=[test_ensure_mysql_is_running], enabled=not FAKE_MODE)
def test_unsuccessful_restart(self):
"""Restart MySQL via the REST when it should fail, assert it does."""
if FAKE_MODE:
raise SkipTest("Cannot run this in fake mode.")
self.unsuccessful_restart()
@test(depends_on=[test_set_up],
@ -344,18 +341,22 @@ class RebootTests(RebootTestBase):
@before_class
def test_set_up(self):
self.set_up()
assert_true(hasattr(self, 'dbaas'))
assert_true(self.dbaas is not None)
@test
def test_ensure_mysql_is_running(self):
"""Make sure MySQL is accessible before restarting."""
self.ensure_mysql_is_running()
@test(depends_on=[test_ensure_mysql_is_running], enabled=not FAKE_MODE)
@test(depends_on=[test_ensure_mysql_is_running])
def test_unsuccessful_restart(self):
"""Restart MySQL via the REST when it should fail, assert it does."""
if FAKE_MODE:
raise SkipTest("Cannot run this in fake mode.")
self.unsuccessful_restart()
@after_class(always_run=True)
@after_class(depends_on=[test_set_up])
def test_successful_restart(self):
"""Restart MySQL via the REST API successfully."""
self.successful_restart()
@ -414,6 +415,8 @@ class ResizeInstanceTest(ActionTestBase):
assert_equal(len(flavors), 1, "Number of flavors with name '%s' "
"found was '%d'." % (flavor_name, len(flavors)))
flavor = flavors[0]
self.old_dbaas_flavor = instance_info.dbaas_flavor
instance_info.dbaas_flavor = flavor
assert_true(flavor is not None, "Flavor '%s' not found!" % flavor_name)
flavor_href = self.dbaas.find_flavor_self_href(flavor)
assert_true(flavor_href is not None,
@ -424,6 +427,8 @@ class ResizeInstanceTest(ActionTestBase):
def test_status_changed_to_resize(self):
self.log_current_users()
self.obtain_flavor_ids()
if CONFIG.simulate_events:
raise SkipTest("Cannot simulate this test.")
self.dbaas.instances.resize_instance(
self.instance_id,
self.get_flavor_href(flavor_id=self.expected_new_flavor_id))
@ -460,6 +465,8 @@ class ResizeInstanceTest(ActionTestBase):
@test(depends_on=[test_instance_returns_to_active_after_resize],
runs_after=[resize_should_not_delete_users])
def test_make_sure_mysql_is_running_after_resize(self):
if CONFIG.simulate_events:
raise SkipTest("Cannot simulate this test.")
self.ensure_mysql_is_running()
actual = self.get_flavor_href(self.instance.flavor['id'])
expected = self.get_flavor_href(flavor_id=self.expected_new_flavor_id)
@ -468,6 +475,8 @@ class ResizeInstanceTest(ActionTestBase):
@test(depends_on=[test_make_sure_mysql_is_running_after_resize])
@time_out(TIME_OUT_TIME)
def test_resize_down(self):
if CONFIG.simulate_events:
raise SkipTest("Cannot simulate this test.")
expected_dbaas_flavor = self.expected_dbaas_flavor
self.dbaas.instances.resize_instance(
self.instance_id,
@ -489,7 +498,7 @@ def resize_should_not_delete_users():
fail("Somehow, the resize made the test user disappear.")
@test(depends_on_classes=[ResizeInstanceTest], depends_on=[create_user],
@test(runs_after=[ResizeInstanceTest], depends_on=[create_user],
groups=[GROUP, tests.INSTANCES],
enabled=CONFIG.reddwarf_volume_support)
class ResizeInstanceVolume(object):
@ -513,7 +522,7 @@ class ResizeInstanceVolume(object):
instance_info.dbaas.instances.resize_volume(instance_info.id,
self.new_volume_size)
@test
@test(depends_on=[test_volume_resize])
@time_out(300)
def test_volume_resize_success(self):

35
reddwarf/tests/api/instances_mysql_down.py
View File

@ -20,16 +20,8 @@ from proboscis.decorators import time_out
from proboscis import after_class
from proboscis import before_class
from proboscis import test
from proboscis.asserts import assert_equal
from proboscis.asserts import assert_false
from proboscis.asserts import assert_is
from proboscis.asserts import assert_is_not
from proboscis.asserts import assert_is_none
from proboscis.asserts import assert_not_equal
from proboscis.asserts import assert_raises
from proboscis.asserts import assert_true
from proboscis.asserts import Check
from proboscis.asserts import fail
from proboscis import SkipTest
from proboscis.asserts import *
import time
from datetime import datetime
@ -42,6 +34,7 @@ from reddwarf.tests.util import test_config
@test(groups=["dbaas.api.instances.down"])
class TestBase(object):
"""Base class for instance-down tests."""
@before_class
def set_up(self):
@ -63,11 +56,6 @@ class TestBase(object):
lambda instance: instance.status == "ACTIVE",
time_out=(60 * 8))
def _wait_for_new_volume_size(self, new_size):
poll_until(lambda: self.client.instances.get(self.id),
lambda instance: instance.volume['size'] == new_size,
time_out=(60 * 8))
@test
def create_instance(self):
initial = self.client.instances.create(self.name, self.flavor_id,
@ -75,11 +63,14 @@ class TestBase(object):
self.id = initial.id
self._wait_for_active()
@test(depends_on=[create_instance])
def put_into_shutdown_state(self):
def _shutdown_instance(self):
instance = self.client.instances.get(self.id)
self.mgmt_client.management.stop(self.id)
@test(depends_on=[create_instance])
def put_into_shutdown_state(self):
self._shutdown_instance()
@test(depends_on=[put_into_shutdown_state])
@time_out(60 * 5)
def resize_instance_in_shutdown_state(self):
@ -89,20 +80,20 @@ class TestBase(object):
@test(depends_on=[create_instance],
runs_after=[resize_instance_in_shutdown_state])
def put_into_shutdown_state_2(self):
instance = self.client.instances.get(self.id)
self.mgmt_client.management.stop(self.id)
self._shutdown_instance()
@test(depends_on=[put_into_shutdown_state_2])
@time_out(60 * 5)
def resize_volume_in_shutdown_state(self):
self.client.instances.resize_volume(self.id, 2)
self._wait_for_new_volume_size(2)
poll_until(lambda: self.client.instances.get(self.id),
lambda instance: instance.volume['size'] == 2,
time_out=(60 * 8))
@test(depends_on=[create_instance],
runs_after=[resize_volume_in_shutdown_state])
def put_into_shutdown_state_3(self):
instance = self.client.instances.get(self.id)
self.mgmt_client.management.stop(self.id)
self._shutdown_instance()
@test(depends_on=[create_instance],
runs_after=[put_into_shutdown_state_3])

198
reddwarf/tests/api/instances_resize.py Normal file
View File

@ -0,0 +1,198 @@
# Copyright 2013 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mox
from testtools import TestCase
from proboscis import test
from novaclient.exceptions import BadRequest
from novaclient.v1_1.servers import Server
from reddwarf.common.exception import PollTimeOut
from reddwarf.common import utils
from reddwarf.common.context import ReddwarfContext
from reddwarf.guestagent import api as guest
from reddwarf.instance.models import DBInstance
from reddwarf.instance.models import ServiceStatuses
from reddwarf.instance.tasks import InstanceTasks
from reddwarf.openstack.common.rpc.common import RPCException
from reddwarf.taskmanager import models as models
GROUP = 'dbaas.api.instances.resize'
OLD_FLAVOR_ID = 1
NEW_FLAVOR_ID = 2
class ResizeTestBase(TestCase):
def _init(self):
self.mock = mox.Mox()
self.instance_id = 500
context = ReddwarfContext()
self.db_info = DBInstance.create(
name="instance",
flavor_id=OLD_FLAVOR_ID,
tenant_id=999,
volume_size=None,
task_status=InstanceTasks.RESIZING)
self.server = self.mock.CreateMock(Server)
self.instance = models.BuiltInstanceTasks(context,
self.db_info,
self.server,
service_status="ACTIVE")
self.instance.server.flavor = {'id': OLD_FLAVOR_ID}
self.guest = self.mock.CreateMock(guest.API)
self.instance._guest = self.guest
self.instance._refresh_compute_server_info = lambda: None
self.instance._refresh_compute_service_status = lambda: None
self.mock.StubOutWithMock(self.instance, 'update_db')
self.mock.StubOutWithMock(self.instance,
'_set_service_status_to_paused')
self.action = None
def _teardown(self):
try:
self.instance.update_db(task_status=InstanceTasks.NONE)
self.mock.ReplayAll()
self.assertRaises(Exception, self.action.execute)
self.mock.VerifyAll()
finally:
self.mock.UnsetStubs()
self.db_info.delete()
def _stop_mysql(self, reboot=True):
self.guest.stop_mysql(do_not_start_on_reboot=reboot)
def _server_changes_to(self, new_status, new_flavor_id):
def change():
self.server.status = new_status
self.instance.server.flavor['id'] = new_flavor_id
self.mock.StubOutWithMock(utils, "poll_until")
utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120)\
.WithSideEffects(lambda ignore, sleep_time, time_out: change())
def _nova_resizes_successfully(self):
self.server.resize(NEW_FLAVOR_ID)
self._server_changes_to("VERIFY_RESIZE", NEW_FLAVOR_ID)
@test(groups=[GROUP, GROUP + '.resize'])
class ResizeTests(ResizeTestBase):
def setUp(self):
super(ResizeTests, self).setUp()
self._init()
self.action = models.ResizeAction(self.instance,
new_flavor_id=NEW_FLAVOR_ID)
def tearDown(self):
super(ResizeTests, self).tearDown()
self._teardown()
def _start_mysql(self):
self.instance.guest.start_mysql_with_conf_changes(None)
def test_guest_wont_stop_mysql(self):
self.guest.stop_mysql(do_not_start_on_reboot=True)\
.AndRaise(RPCException("Could not stop MySQL!"))
def test_nova_wont_resize(self):
self._stop_mysql()
self.server.resize(NEW_FLAVOR_ID).AndRaise(BadRequest)
self.guest.restart()
def test_nova_resize_timeout(self):
self._stop_mysql()
self.server.resize(NEW_FLAVOR_ID)
self.mock.StubOutWithMock(utils, 'poll_until')
utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120)\
.AndRaise(PollTimeOut)
self.guest.restart()
def test_nova_doesnt_change_flavor(self):
self._stop_mysql()
self.server.resize(NEW_FLAVOR_ID)
self._server_changes_to("VERIFY_RESIZE", OLD_FLAVOR_ID)
self.guest.restart()
def test_nova_resize_fails(self):
self._stop_mysql()
self.server.resize(NEW_FLAVOR_ID)
self._server_changes_to("ACTIVE", OLD_FLAVOR_ID)
self.guest.restart()
def test_nova_resizes_in_weird_state(self):
self._stop_mysql()
self.server.resize(NEW_FLAVOR_ID)
self._server_changes_to("ACTIVE", NEW_FLAVOR_ID)
self.guest.restart()
def test_guest_is_not_okay(self):
self._stop_mysql()
self._nova_resizes_successfully()
self.instance._set_service_status_to_paused()
self.instance.service_status = ServiceStatuses.PAUSED
utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120)\
.AndRaise(PollTimeOut)
self.instance.server.revert_resize()
self.guest.restart()
def test_mysql_is_not_okay(self):
self._stop_mysql()
self._nova_resizes_successfully()
self.instance._set_service_status_to_paused()
self.instance.service_status = ServiceStatuses.SHUTDOWN
utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120)
self._start_mysql()
self.instance.server.revert_resize()
def test_confirm_resize_fails(self):
self._stop_mysql()
self._nova_resizes_successfully()
self.instance._set_service_status_to_paused()
self.instance.service_status = ServiceStatuses.RUNNING
utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120)
self._start_mysql()
self.server.status = "SHUTDOWN"
self.instance.server.confirm_resize()
@test(groups=[GROUP, GROUP + '.migrate'])
class MigrateTests(ResizeTestBase):
def setUp(self):
super(MigrateTests, self).setUp()
self._init()
self.action = models.MigrateAction(self.instance)
def tearDown(self):
super(MigrateTests, self).tearDown()
self._teardown()
def _start_mysql(self):
self.guest.restart()
def test_successful_migrate(self):
self._stop_mysql()
self.server.migrate()
self._server_changes_to("VERIFY_RESIZE", NEW_FLAVOR_ID)
self.instance._set_service_status_to_paused()
self.instance.service_status = ServiceStatuses.RUNNING
utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120)
self._start_mysql()
self.instance.server.confirm_resize()

8
reddwarf/tests/fakes/common.py
View File

@ -18,8 +18,6 @@
"""Common code to help in faking the models."""
import time
import traceback
import sys
from novaclient import exceptions as nova_exceptions
from reddwarf.common import cfg
@ -65,7 +63,6 @@ def event_simulator_sleep(time_to_sleep):
global pending_events
while time_to_sleep > 0:
itr_sleep = 0.5
print pending_events
for i in range(len(pending_events)):
event = pending_events[i]
event["time"] = event["time"] - itr_sleep
@ -77,10 +74,7 @@ def event_simulator_sleep(time_to_sleep):
try:
func()
except Exception as e:
type_, value, tb = sys.exc_info()
LOG.info("Simulated event error.")
LOG.info((traceback.format_exception(type_, value, tb)))
pass # Ignore exceptions, which can potentially occur.
LOG.exception("Simulated event error.")
time_to_sleep -= itr_sleep
sleep_entrance_count -= 1

38
reddwarf/tests/fakes/guestagent.py
View File

@ -110,45 +110,51 @@ class FakeGuest(object):
def prepare(self, memory_mb, databases, users, device_path=None,
mount_point=None):
from reddwarf.instance.models import DBInstance
from reddwarf.instance.models import InstanceServiceStatus
from reddwarf.instance.models import ServiceStatuses
from reddwarf.guestagent.models import AgentHeartBeat
LOG.debug("users... %s" % users)
LOG.debug("databases... %s" % databases)
instance_name = DBInstance.find_by(id=self.id).name
self.create_user(users)
self.create_database(databases)
def update_db():
status = InstanceServiceStatus.find_by(instance_id=self.id)
status.status = ServiceStatuses.RUNNING
if instance_name.endswith('GUEST_ERROR'):
status.status = ServiceStatuses.FAILED
else:
status.status = ServiceStatuses.RUNNING
status.save()
AgentHeartBeat.create(instance_id=self.id)
self.event_spawn(1.0, update_db)
def restart(self):
def _set_status(self, new_status='RUNNING'):
from reddwarf.instance.models import InstanceServiceStatus
from reddwarf.instance.models import ServiceStatuses
print("Setting status to %s" % new_status)
states = {'RUNNING': ServiceStatuses.RUNNING,
'SHUTDOWN': ServiceStatuses.SHUTDOWN,
}
status = InstanceServiceStatus.find_by(instance_id=self.id)
status.status = states[new_status]
status.save()
def restart(self):
# All this does is restart, and shut off the status updates while it
# does so. So there's actually nothing to do to fake this out except
# take a nap.
print("Sleeping for a second.")
time.sleep(1)
status = InstanceServiceStatus.find_by(instance_id=self.id)
status.status = ServiceStatuses.RUNNING
status.save()
self._set_status('RUNNING')
def start_mysql_with_conf_changes(self, updated_memory_size):
from reddwarf.instance.models import InstanceServiceStatus
from reddwarf.instance.models import ServiceStatuses
status = InstanceServiceStatus.find_by(instance_id=self.id)
status.status = ServiceStatuses.RUNNING
status.save()
time.sleep(2)
self._set_status('RUNNING')
def stop_mysql(self):
from reddwarf.instance.models import InstanceServiceStatus
from reddwarf.instance.models import ServiceStatuses
status = InstanceServiceStatus.find_by(instance_id=self.id)
status.status = ServiceStatuses.SHUTDOWN
status.save()
def stop_mysql(self, do_not_start_on_reboot=False):
self._set_status('SHUTDOWN')
def get_volume_info(self):
"""Return used volume information in bytes."""

70
reddwarf/tests/fakes/nova.py
View File

@ -15,17 +15,19 @@
# License for the specific language governing permissions and limitations
# under the License.
import eventlet
from reddwarf.openstack.common import log as logging
from novaclient.v1_1.client import Client
from novaclient import exceptions as nova_exceptions
import uuid
from novaclient.v1_1.client import Client
from reddwarf.common.exception import PollTimeOut
from reddwarf.common.utils import poll_until
from reddwarf.openstack.common import log as logging
from reddwarf.tests.fakes.common import authorize
from reddwarf.tests.fakes.common import get_event_spawer
from reddwarf.common.utils import poll_until
from reddwarf.common.exception import PollTimeOut
import eventlet
import uuid
LOG = logging.getLogger(__name__)
FAKE_HOSTS = ["fake_host_1", "fake_host_2"]
class FakeFlavor(object):
@ -100,6 +102,7 @@ class FakeServer(object):
self.name = name
self.image_id = image_id
self.flavor_ref = flavor_ref
self.old_flavor_ref = None
self.event_spawn = get_event_spawer()
self._current_status = "BUILD"
self.volumes = volumes
@ -111,7 +114,8 @@ class FakeServer(object):
for volume in self.volumes:
info_vols.append({'id': volume.id})
volume.set_attachment(id)
self.host = "fake_host"
self.host = FAKE_HOSTS[0]
self.old_host = None
self._info = {'os:volumes': info_vols}
@ -124,13 +128,23 @@ class FakeServer(object):
raise RuntimeError("Not in resize confirm mode.")
self._current_status = "ACTIVE"
def revert_resize(self):
if self.status != "VERIFY_RESIZE":
raise RuntimeError("Not in resize confirm mode.")
self.host = self.old_host
self.old_host = None
self.flavor_ref = self.old_flavor_ref
self.old_flavor_ref = None
self._current_status = "ACTIVE"
def reboot(self):
LOG.debug("Rebooting server %s" % (self.id))
self._current_status = "REBOOT"
def set_to_active():
self._current_status = "ACTIVE"
self.parent.schedule_simulate_running_server(self.id, 1.5)
self._current_status = "REBOOT"
self.event_spawn(1, set_to_active)
def delete(self):
@ -157,7 +171,10 @@ class FakeServer(object):
return [{"href": url, "rel": link_type}
for link_type in ['self', 'bookmark']]
def resize(self, new_flavor_id):
def migrate(self):
self.resize(None)
def resize(self, new_flavor_id=None):
self._current_status = "RESIZE"
if self.name.endswith("_RESIZE_TIMEOUT"):
raise PollTimeOut()
@ -165,15 +182,32 @@ class FakeServer(object):
def set_to_confirm_mode():
self._current_status = "VERIFY_RESIZE"
def set_to_active():
self.parent.schedule_simulate_running_server(self.id, 1.5)
self.event_spawn(1, set_to_active)
def change_host():
self.old_host = self.host
self.host = [host for host in FAKE_HOSTS if host != self.host][0]
def set_flavor():
if self.name.endswith("_RESIZE_ERROR"):
self._current_status = "ACTIVE"
return
if new_flavor_id is None:
# Migrations are flavorless flavor resizes.
# A resize MIGHT change the host, but a migrate
# deliberately does.
LOG.debug("Migrating fake instance.")
self.event_spawn(0.75, change_host)
else:
LOG.debug("Resizing fake instance.")
self.old_flavor_ref = self.flavor_ref
flavor = self.parent.flavors.get(new_flavor_id)
self.flavor_ref = flavor.links[0]['href']
self.event_spawn(1, set_to_confirm_mode)
self.event_spawn(1, set_to_confirm_mode)
self.event_spawn(1, set_flavor)
self.event_spawn(0.8, set_flavor)
def schedule_status(self, new_status, time_from_now):
"""Makes a new status take effect at the given time."""
@ -291,10 +325,11 @@ class FakeServers(object):
self.event_spawn(time_from_now, delete_server)
def schedule_simulate_running_server(self, id, time_from_now):
from reddwarf.instance.models import DBInstance
from reddwarf.instance.models import InstanceServiceStatus
from reddwarf.instance.models import ServiceStatuses
def set_server_running():
from reddwarf.instance.models import DBInstance
from reddwarf.instance.models import InstanceServiceStatus
from reddwarf.instance.models import ServiceStatuses
instance = DBInstance.find_by(compute_instance_id=id)
LOG.debug("Setting server %s to running" % instance.id)
status = InstanceServiceStatus.find_by(instance_id=instance.id)
@ -530,6 +565,10 @@ class FakeHost(object):
self.totalRAM = 2004 # 16384
self.usedRAM = 0
for server in self.servers.list():
print server
if server.host != self.name:
print "\t...not on this host."
continue
self.instances.append({
'uuid': server.id,
'name': server.name,
@ -550,7 +589,8 @@ class FakeHosts(object):
def __init__(self, servers):
self.hosts = {}
self.add_host(FakeHost('fake_host', servers))
for host in FAKE_HOSTS:
self.add_host(FakeHost(host, servers))
def add_host(self, host):
self.hosts[host.name] = host

1
run_tests.py
View File

@ -116,6 +116,7 @@ if __name__=="__main__":
from reddwarf.tests.api import instances_actions
from reddwarf.tests.api import instances_delete
from reddwarf.tests.api import instances_mysql_down
from reddwarf.tests.api import instances_resize
from reddwarf.tests.api import databases
from reddwarf.tests.api import root
from reddwarf.tests.api import users

1
tools/test-requires
View File

@ -12,6 +12,7 @@ wsgi_intercept
proboscis
python-reddwarfclient
mock
mox
testtools>=0.9.22
pexpect
discover