openstack/rally
Code Issues Proposed changes

Refactoring for generic benchmark cleanup

Browse Source 
Initial refactoring which will be used in the generic benchmark cleanup
implementation. Specifically the utility functions which existed in the
NovaScenarios class are now exposed in benchmark.test_utils. Moreover a
few of these functions have been enahanced to be more generic/flexible
in nature.

Subsequent patch will provide the actual generic cleanup.

Implements partial: blueprint benchmark-cloud-cleanup

Change-Id: I7e25dd006e46bf66fd9419090ea310629ba7d19f
This commit is contained in:
boden
2013-12-09 07:41:21 -05:00
parent bdcf4f8dbb
commit 3ba77da65d
6 changed files with 138 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
rally/benchmark/scenarios/nova/utils.py
View File

@@ -17,28 +17,11 @@ import random
import string
import time
from novaclient import exceptions
from rally.benchmark import base
from rally import exceptions as rally_exceptions
from rally.benchmark import utils as bench_utils
from rally import utils
def _resource_is(status):
return lambda resource: resource.status == status
def _get_from_manager(resource):
resource = resource.manager.get(resource)
if resource.status == "ERROR":
raise rally_exceptions.GetResourceFailure()
return resource
def _false(resource):
return False
class NovaScenario(base.Scenario):
@classmethod
@@ -60,8 +43,9 @@ class NovaScenario(base.Scenario):
# NOTE(msdubov): It is reasonable to wait 5 secs before starting to
# check whether the server is ready => less API calls.
time.sleep(5)
server = utils.wait_for(server, is_ready=_resource_is("ACTIVE"),
update_resource=_get_from_manager,
server = utils.wait_for(server,
is_ready=bench_utils.resource_is("ACTIVE"),
update_resource=bench_utils.get_from_manager(),
timeout=600, check_interval=3)
return server
@@ -78,8 +62,8 @@ class NovaScenario(base.Scenario):
"""
server.reboot(reboot_type=("SOFT" if soft else "HARD"))
time.sleep(5)
utils.wait_for(server, is_ready=_resource_is("ACTIVE"),
update_resource=_get_from_manager,
utils.wait_for(server, is_ready=bench_utils.resource_is("ACTIVE"),
update_resource=bench_utils.get_from_manager(),
timeout=600, check_interval=3)
@classmethod
@@ -92,8 +76,8 @@ class NovaScenario(base.Scenario):
:param server: The server to start and wait to become ACTIVE.
"""
server.start()
utils.wait_for(server, is_ready=_resource_is("ACTIVE"),
update_resource=_get_from_manager,
utils.wait_for(server, is_ready=bench_utils.resource_is("ACTIVE"),
update_resource=bench_utils.get_from_manager(),
timeout=600, check_interval=2)
@classmethod
@@ -106,8 +90,8 @@ class NovaScenario(base.Scenario):
:param server: The server to stop.
"""
server.stop()
utils.wait_for(server, is_ready=_resource_is("SHUTOFF"),
update_resource=_get_from_manager,
utils.wait_for(server, is_ready=bench_utils.resource_is("SHUTOFF"),
update_resource=bench_utils.get_from_manager(),
timeout=600, check_interval=2)
@classmethod
@@ -121,8 +105,8 @@ class NovaScenario(base.Scenario):
"""
server.suspend()
time.sleep(2)
utils.wait_for(server, is_ready=_resource_is("SUSPENDED"),
update_resource=_get_from_manager,
utils.wait_for(server, is_ready=bench_utils.resource_is("SUSPENDED"),
update_resource=bench_utils.get_from_manager(),
timeout=600, check_interval=3)
@classmethod
@@ -134,15 +118,9 @@ class NovaScenario(base.Scenario):
:param server: Server object
"""
server.delete()
# NOTE(msdubov): When the server gets deleted, the
# clients("nova").servers.get() method raises
# a NotFound exception.
try:
utils.wait_for(server, is_ready=_false,
update_resource=_get_from_manager,
timeout=600, check_interval=3)
except exceptions.NotFound:
pass
utils.wait_for(server, is_ready=bench_utils.is_none,
update_resource=bench_utils.get_from_manager(),
timeout=600, check_interval=3)
@classmethod
def _delete_all_servers(cls):
@@ -160,8 +138,8 @@ class NovaScenario(base.Scenario):
:param image: Image object
"""
image.delete()
utils.wait_for(image, is_ready=_resource_is("DELETED"),
update_resource=_get_from_manager,
utils.wait_for(image, is_ready=bench_utils.resource_is("DELETED"),
update_resource=bench_utils.get_from_manager(),
timeout=600, check_interval=3)
@classmethod
@@ -178,8 +156,9 @@ class NovaScenario(base.Scenario):
image_uuid = cls.clients("nova").servers.create_image(server,
server.name)
image = cls.clients("nova").images.get(image_uuid)
image = utils.wait_for(image, is_ready=_resource_is("ACTIVE"),
update_resource=_get_from_manager,
image = utils.wait_for(image,
is_ready=bench_utils.resource_is("ACTIVE"),
update_resource=bench_utils.get_from_manager(),
timeout=600, check_interval=3)
return image
@@ -213,8 +192,10 @@ class NovaScenario(base.Scenario):
servers = filter(lambda server: server.name.startswith(name_prefix),
cls.clients("nova").servers.list())
time.sleep(5)
servers = [utils.wait_for(server, is_ready=_resource_is("ACTIVE"),
update_resource=_get_from_manager,
servers = [utils.wait_for(server,
is_ready=bench_utils.resource_is("ACTIVE"),
update_resource=bench_utils.
get_from_manager(),
timeout=600, check_interval=3)
for server in servers]
return servers
@@ -222,7 +203,3 @@ class NovaScenario(base.Scenario):
@classmethod
def _generate_random_name(cls, length):
return ''.join(random.choice(string.lowercase) for i in range(length))
@classmethod
def cleanup(cls):
cls._delete_all_servers()

61
rally/benchmark/utils.py
View File

@@ -25,6 +25,7 @@ import uuid
import fuel_health.cleanup as fuel_cleanup
from rally.benchmark import base
from rally import exceptions as rally_exceptions
from rally.openstack.common.gettextutils import _ # noqa
from rally.openstack.common import log as logging
from rally import osclients
@@ -39,6 +40,66 @@ __admin_clients__ = {}
__scenario_context__ = {}
def resource_is(status):
return lambda resource: resource.status == status
def is_none(obj):
return obj is None
def get_from_manager(error_statuses=["ERROR"]):
def _get_from_manager(resource):
try:
resource = resource.manager.get(resource)
except Exception as e:
if getattr(e, 'http_status', 400) == 404:
return None
raise e
if resource.status in error_statuses:
raise rally_exceptions.GetResourceFailure(status=resource.status)
return resource
return _get_from_manager
def manager_list_size(sizes):
def _list(mgr):
return len(mgr.list()) in sizes
return _list
def _wait_for_list_statuses(mgr, statuses, list_query={},
timeout=10, check_interval=1):
def _list_statuses(mgr):
for resource in mgr.list(**list_query):
if resource.status not in statuses:
return False
return True
utils.wait_for(mgr, is_ready=_list_statuses, update_resource=None,
timeout=timeout, check_interval=check_interval)
def _wait_for_empty_list(mgr, timeout=10, check_interval=1):
_wait_for_list_size(mgr, sizes=[0], timeout=timeout,
check_interval=check_interval)
def _wait_for_list_size(mgr, sizes=[0], timeout=10, check_interval=1):
utils.wait_for(mgr, is_ready=manager_list_size(sizes),
update_resource=None, timeout=timeout,
check_interval=check_interval)
def false(resource):
return False
def _async_cleanup(cls, indicies):
cls._cleanup_with_clients(indicies)
def _format_exc(exc):
return [str(type(exc)), str(exc), traceback.format_exc()]

3
rally/exceptions.py
View File

@@ -144,7 +144,8 @@ class TimeoutException(RallyException):
class GetResourceFailure(RallyException):
msg_fmt = _("Failed to get the resource.")
msg_fmt = _("Failed to get the resource due to invalid status:"
"`%(status)s`")
class SSHError(RallyException):

6
rally/serverprovider/providers/openstack.py
View File

@@ -18,7 +18,7 @@ import os
import time
import urllib2
from rally.benchmark.scenarios.nova import utils as nova_utils
from rally.benchmark import utils as benchmark_utils
from rally import exceptions
from rally.openstack.common.gettextutils import _ # noqa
from rally.openstack.common import log as logging
@@ -157,8 +157,8 @@ class OpenStackProvider(provider.ProviderFactory):
self.resources.create({'id': server.id}, type=SERVER_TYPE)
kwargs = {
'is_ready': nova_utils._resource_is("ACTIVE"),
'update_resource': nova_utils._get_from_manager,
'is_ready': benchmark_utils.resource_is("ACTIVE"),
'update_resource': benchmark_utils.get_from_manager(),
'timeout': 120,
'check_interval': 5
}

132
tests/benchmark/scenarios/nova/test_utils.py
View File

@@ -210,13 +210,6 @@ class NovaScenarioTestCase(test.TestCase):
def setUp(self):
super(NovaScenarioTestCase, self).setUp()
self.rally_utils = "rally.benchmark.scenarios.nova.utils.utils"
self.utils_resource_is = ("rally.benchmark.scenarios.nova.utils"
"._resource_is")
self.osclients = "rally.benchmark.utils.osclients"
self.nova_scenario = ("rally.benchmark.scenarios.nova.utils."
"NovaScenario")
self.sleep = "rally.benchmark.scenarios.nova.utils.time.sleep"
def test_generate_random_name(self):
for length in [8, 16, 32, 64]:
@@ -227,94 +220,59 @@ class NovaScenarioTestCase(test.TestCase):
def test_failed_server_status(self):
server_manager = FakeFailedServerManager()
self.assertRaises(rally_exceptions.GetResourceFailure,
utils._get_from_manager,
butils.get_from_manager(),
server_manager.create('fails', '1', '2'))
def test_cleanup_failed(self):
with mock.patch(self.osclients) as mock_osclients:
fc = FakeClients()
mock_osclients.Clients.return_value = fc
failed_nova = FakeNovaClient(failed_server_manager=True)
fc.get_nova_client = lambda: failed_nova
@mock.patch("rally.benchmark.scenarios.nova.utils.time.sleep")
@mock.patch("rally.utils")
@mock.patch("rally.benchmark.utils.osclients")
@mock.patch("rally.benchmark.utils.resource_is")
def test_server_helper_methods(self, mock_ris, mock_osclients,
mock_rally_utils, mock_sleep):
temp_keys = ["username", "password", "tenant_name", "uri"]
users_endpoints = [dict(zip(temp_keys, temp_keys))]
utils.NovaScenario._clients = butils._create_openstack_clients(
users_endpoints, temp_keys)[0]
def _is_ready(resource):
return resource.status == "ACTIVE"
with mock.patch(self.sleep):
# NOTE(boden): verify failed server cleanup
self.assertRaises(rally_exceptions.GetResourceFailure,
utils.NovaScenario._boot_server,
"fails", 0, 1)
self.assertEquals(len(failed_nova.servers.list()), 1,
"Server not created")
utils.NovaScenario.cleanup()
self.assertEquals(len(failed_nova.servers.list()), 0,
"Servers not purged")
mock_ris.return_value = _is_ready
get_from_mgr = butils.get_from_manager()
def test_cleanup(self):
with mock.patch(self.osclients) as mock_osclients:
fc = FakeClients()
mock_osclients.Clients.return_value = fc
fake_nova = FakeNovaClient()
fc.get_nova_client = lambda: fake_nova
fc = FakeClients()
mock_osclients.Clients.return_value = fc
fake_nova = FakeNovaClient()
fc.get_nova_client = lambda: fake_nova
fsm = FakeServerManager()
fake_server = fsm.create("s1", "i1", 1)
fsm.create = lambda name, iid, fid: fake_server
fake_nova.servers = fsm
fake_image_id = fsm.create_image(fake_server, 'img')
fake_image = fake_nova.images.get(fake_image_id)
fsm.create_image = lambda svr, name: fake_image
temp_keys = ["username", "password", "tenant_name", "uri"]
users_endpoints = [dict(zip(temp_keys, temp_keys))]
utils.NovaScenario._clients = butils.\
_create_openstack_clients(users_endpoints, temp_keys)[0]
temp_keys = ["username", "password", "tenant_name", "uri"]
users_endpoints = [dict(zip(temp_keys, temp_keys))]
utils.NovaScenario._clients = butils._create_openstack_clients(
users_endpoints, temp_keys)[0]
utils.utils = mock_rally_utils
utils.bench_utils.get_from_manager = lambda: get_from_mgr
# NOTE(boden): verify active server cleanup
with mock.patch(self.sleep):
for i in range(5):
utils.NovaScenario._boot_server("server-%s" % i, 0, 1)
self.assertEquals(len(fake_nova.servers.list()), 5,
"Server not created")
utils.NovaScenario.cleanup()
self.assertEquals(len(fake_nova.servers.list()), 0,
"Servers not purged")
def test_server_helper_methods(self):
with mock.patch(self.rally_utils) as mock_rally_utils:
with mock.patch(self.utils_resource_is) as mock_resource_is:
mock_resource_is.return_value = {}
with mock.patch(self.osclients) as mock_osclients:
fc = FakeClients()
mock_osclients.Clients.return_value = fc
fake_nova = FakeNovaClient()
fc.get_nova_client = lambda: fake_nova
fsm = FakeServerManager()
fake_server = fsm.create("s1", "i1", 1)
fsm.create = lambda name, iid, fid: fake_server
fake_nova.servers = fsm
temp_keys = ["username", "password",
"tenant_name", "uri"]
users_endpoints = [dict(zip(temp_keys, temp_keys))]
utils.NovaScenario._clients = butils.\
_create_openstack_clients(users_endpoints,
temp_keys)[0]
with mock.patch(self.sleep):
utils.NovaScenario._boot_server("s1", "i1", 1)
utils.NovaScenario._create_image(fake_server)
utils.NovaScenario._suspend_server(fake_server)
utils.NovaScenario._delete_server(fake_server)
utils.NovaScenario._boot_server("s1", "i1", 1)
utils.NovaScenario._create_image(fake_server)
utils.NovaScenario._suspend_server(fake_server)
utils.NovaScenario._delete_server(fake_server)
expected = [
mock.call.wait_for(fake_server, is_ready={},
update_resource=utils._get_from_manager,
timeout=600, check_interval=3),
mock.call.wait_for("img_uuid", is_ready={},
update_resource=utils._get_from_manager,
timeout=600, check_interval=3),
mock.call.wait_for(fake_server, is_ready={},
update_resource=utils._get_from_manager,
timeout=600, check_interval=3),
mock.call.wait_for(fake_server, is_ready=utils._false,
update_resource=utils._get_from_manager,
timeout=600, check_interval=3)
mock.call.wait_for(fake_server, is_ready=_is_ready,
update_resource=butils.get_from_manager(),
check_interval=3, timeout=600),
mock.call.wait_for(fake_image, is_ready=_is_ready,
update_resource=butils.get_from_manager(),
check_interval=3, timeout=600),
mock.call.wait_for(fake_server, is_ready=_is_ready,
update_resource=butils.get_from_manager(),
check_interval=3, timeout=600),
mock.call.wait_for(fake_server, is_ready=butils.is_none,
update_resource=butils.get_from_manager(),
check_interval=3, timeout=600)
]
self.assertEqual(mock_rally_utils.mock_calls, expected)
self.assertEqual(expected, mock_rally_utils.mock_calls)

5
tests/serverprovider/test_openstack_provider.py
View File

@@ -128,8 +128,9 @@ class OpenStackProviderTestCase(test.TestCase):
@mock.patch(MOD_NAME + '.osclients')
@mock.patch(MOD_NAME + '.open', create=True)
@mock.patch(MOD_NAME + '.provider')
@mock.patch(MOD_NAME + '.nova_utils._get_from_manager', new=lambda r: r)
def test_openstack_provider_create_vms(self, g, provider, clients):
@mock.patch('rally.benchmark.utils.get_from_manager')
def test_openstack_provider_create_vms(self, get, g, provider, clients):
get.return_value = lambda r: r
self._init_mock_clients()
clients.Clients = mock.MagicMock(return_value=self.clients)
provider.Server = mock.MagicMock()