openstack/rally
Code Issues Proposed changes

Add benchmark-context manager

Browse Source 
This patch actually mostly finish support of benchmark-context.

* Add context manager that allows to use only specified context.
  Context could be specified through 2 ways:
  1) @base.scenario(context={...})
  2) via task config

* Support of two types of context:
  1) __ctx_hidden__ = True => could be setup only using @base.scenario
  2) __ctx_hidden__ = False => could be setup in both ways

* Support of validation of context types

* Support of order of loading context __ctx_order__
  Context manager will use this number to sort context and chose order
  of creating them.

* Unified "cleanup" context to work only via @base.scenario()

* Great Rally perf optimization - use only required context

* Cleanup decorator is merged to @base.scenario

* Add logging for context

* Improve NovaScenario._boot_server to add to secgroup allow_ssh
  only when we are using allow_ssh context

* Add unit test that checks that all scenario have proper predefined
  context config

blueprint benchmark-context

Change-Id: I4fce466a88075a694a87e72ace72cc522605b72a
This commit is contained in:
Boris Pavlovic 2014-04-06 17:50:16 +04:00
parent a680e0ec57
commit 4f7303fe86
30 changed files with 382 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
rally/benchmark/context/base.py
View File

@ -24,18 +24,20 @@ from rally import utils
@six.add_metaclass(abc.ABCMeta)
class Context(object):
"""We will use this class in future as a factory for context classes.
"""This class is a factory for context classes.
It will cover:
1) Auto discovering
2) Validation of input args
3) Common logging
Every context class should be a subclass of this method and implement
2 abstract methods: setup() and cleanup()
Actually the same functionality as
runners.base.ScenarioRunner and scenarios.base.Scenario
It covers:
1) proper setting up of context config
2) Auto discovering & get by name
3) Validation by CONFIG_SCHEMA
4) Order of context creation
"""
__ctx_name__ = "base"
__ctx_order__ = 0
__ctx_hidden__ = True
CONFIG_SCHEMA = {}
@ -44,11 +46,11 @@ class Context(object):
self.context = context
self.task = context["task"]
@staticmethod
def validate(context):
for name, config in context.iteritems():
ctx = Context.get_by_name(name)
jsonschema.validate(config, ctx.CONFIG_SCHEMA)
@classmethod
def validate(cls, config, non_hidden=False):
if non_hidden and cls.__ctx_hidden__:
raise exceptions.NoSuchContext(name=cls.__ctx_name__)
jsonschema.validate(config, cls.CONFIG_SCHEMA)
@staticmethod
def get_by_name(name):
@ -71,3 +73,45 @@ class Context(object):
def __exit__(self, exc_type, exc_value, exc_traceback):
self.cleanup()
class ContextManager(object):
"""Creates context environment and runs method inside it."""
@staticmethod
def run(context, func, *args, **kwargs):
ctxlst = [Context.get_by_name(name) for name in context["config"]]
ctxlst = map(lambda ctx: ctx(context),
sorted(ctxlst, key=lambda x: x.__ctx_order__))
return ContextManager._magic(ctxlst, func, *args, **kwargs)
@staticmethod
def validate(context, non_hidden=False):
for name, config in context.iteritems():
Context.get_by_name(name).validate(config, non_hidden=non_hidden)
@staticmethod
def _magic(ctxlst, func, *args, **kwargs):
"""Some kind of contextlib.nested but with black jack & recursion.
This method uses recursion to build nested "with" from list of context
objects. As it's actually a combination of dark and voodoo magic I
called it "_magic". Please don't repeat at home.
:param ctxlst: list of instances of subclasses of Context
:param func: function that will be called inside this context
:param args: args that will be passed to function `func`
:param kwargs: kwargs that will be passed to function `func`
:returns: result of function call
"""
if not ctxlst:
return func(*args, **kwargs)
with ctxlst[0]:
# TODO(boris-42): call of setup could be moved inside __enter__
# but it should be in try-except, and in except
# we should call by hand __exit__
ctxlst[0].setup()
tmp = ContextManager._magic(ctxlst[1:], func, *args, **kwargs)
return tmp

42
rally/benchmark/context/cleaner.py
View File

@ -17,7 +17,6 @@ import functools
import sys
from rally.benchmark.context import base
from rally.benchmark.scenarios import base as scenario_base
from rally.benchmark import utils
from rally.openstack.common.gettextutils import _
from rally.openstack.common import log as logging
@ -31,19 +30,24 @@ LOG = logging.getLogger(__name__)
class ResourceCleaner(base.Context):
"""Context class for resource cleanup (both admin and non-admin)."""
__ctx_name__ = "cleaner"
__ctx_name__ = "cleanup"
__ctx_order__ = 200
__ctx_hidden__ = True
CONFIG_SCHEMA = {
"type": "object",
"$schema": "http://json-schema.org/draft-03/schema",
"properties": {},
"additionalProperties": False
"type": "array",
"$schema": "http://json-schema.org/draft-04/schema",
"items": {
"type": "string",
"enum": ["nova", "glance", "cinder"]
},
"uniqueItems": True
}
def __init__(self, context):
super(ResourceCleaner, self).__init__(context)
self.admin = None
self.users = None
self.admin = []
self.users = []
if "admin" in context and context["admin"]:
self.admin = context["admin"]["endpoint"]
if "users" in context and context["users"]:
@ -51,19 +55,6 @@ class ResourceCleaner(base.Context):
@rutils.log_task_wrapper(LOG.info, _("Cleanup users resources."))
def _cleanup_users_resources(self):
def _init_services_to_cleanup(cleanup_methods):
scenario_name = self.context.get('scenario_name')
if scenario_name:
cls_name, method_name = scenario_name.split(".", 1)
scenario = scenario_base.Scenario.get_by_name(cls_name)()
scenario_method = getattr(scenario, method_name)
if hasattr(scenario_method, "cleanup_services"):
return getattr(scenario_method, "cleanup_services")
return cleanup_methods.keys()
if not self.users:
return
for user in self.users:
clients = osclients.Clients(user)
cleanup_methods = {
@ -76,7 +67,7 @@ class ResourceCleaner(base.Context):
clients.cinder())
}
for service in _init_services_to_cleanup(cleanup_methods):
for service in self.config:
try:
cleanup_methods[service]()
except Exception as e:
@ -87,9 +78,6 @@ class ResourceCleaner(base.Context):
@rutils.log_task_wrapper(LOG.info, _("Cleanup admin resources."))
def _cleanup_admin_resources(self):
if not self.admin:
return
try:
admin = osclients.Clients(self.admin)
utils.delete_keystone_resources(admin.keystone())
@ -99,11 +87,13 @@ class ResourceCleaner(base.Context):
LOG.warning(_('Unable to fully cleanup keystone service: %s') %
(e.message))
@rutils.log_task_wrapper(LOG.info, _("Enter context: `cleanup`"))
def setup(self):
pass
@rutils.log_task_wrapper(LOG.info, _("Exit context: `cleanup`"))
def cleanup(self):
if self.users:
if self.users and self.config:
self._cleanup_users_resources()
if self.admin:
self._cleanup_admin_resources()

7
rally/benchmark/context/keypair.py
View File

@ -16,8 +16,10 @@
import novaclient.exceptions
from rally.benchmark.context import base
from rally.openstack.common.gettextutils import _
from rally.openstack.common import log as logging
from rally import osclients
from rally import utils
LOG = logging.getLogger(__name__)
@ -25,6 +27,9 @@ LOG = logging.getLogger(__name__)
class Keypair(base.Context):
__ctx_name__ = "keypair"
__ctx_order__ = 300
__ctx_hidden__ = True
KEYPAIR_NAME = "rally_ssh_key"
def _get_nova_client(self, endpoint):
@ -47,11 +52,13 @@ class Keypair(base.Context):
return {"private": keypair.private_key,
"public": keypair.public_key}
@utils.log_task_wrapper(LOG.info, _("Enter context: `keypair`"))
def setup(self):
for user in self.context["users"]:
keypair = self._generate_keypair(user["endpoint"])
user["keypair"] = keypair
@utils.log_task_wrapper(LOG.info, _("Exit context: `keypair`"))
def cleanup(self):
for user in self.context["users"]:
endpoint = user['endpoint']

10
rally/benchmark/context/quotas.py
View File

@ -14,7 +14,13 @@
# under the License.
from rally.benchmark.context import base
from rally.openstack.common.gettextutils import _
from rally.openstack.common import log as logging
from rally import osclients
from rally import utils
LOG = logging.getLogger(__name__)
class NovaQuotas(object):
@ -124,6 +130,8 @@ class Quotas(base.Context):
"""Context class for updating benchmarks' tenants quotas."""
__ctx_name__ = "quotas"
__ctx_order__ = 210
__ctx_hidden__ = False
CONFIG_SCHEMA = {
"type": "object",
@ -141,6 +149,7 @@ class Quotas(base.Context):
self.nova_quotas = NovaQuotas(self.clients.nova())
self.cinder_quotas = CinderQuotas(self.clients.cinder())
@utils.log_task_wrapper(LOG.info, _("Enter context: `quotas`"))
def setup(self):
for tenant in self.context["tenants"]:
if "nova" in self.config and len(self.config["nova"]) > 0:
@ -151,6 +160,7 @@ class Quotas(base.Context):
self.cinder_quotas.update(tenant["id"],
**self.config["cinder"])
@utils.log_task_wrapper(LOG.info, _("Exit context: `quotas`"))
def cleanup(self):
for tenant in self.context["tenants"]:
# Always cleanup quotas before deleting a tenant

7
rally/benchmark/context/secgroup.py
View File

@ -14,8 +14,10 @@
# under the License.
from rally.benchmark.context import base
from rally.openstack.common.gettextutils import _
from rally.openstack.common import log as logging
from rally import osclients
from rally import utils
LOG = logging.getLogger(__name__)
@ -77,11 +79,15 @@ def _prepare_open_secgroup(endpoint):
class AllowSSH(base.Context):
__ctx_name__ = "allow_ssh"
__ctx_order__ = 301
__ctx_hidden__ = True
def __init__(self, context):
super(AllowSSH, self).__init__(context)
self.context["allow_ssh"] = SSH_GROUP_NAME
self.secgroup = []
@utils.log_task_wrapper(LOG.info, _("Exit context: `allow_ssh`"))
def setup(self):
used_tenants = []
for user in self.context['users']:
@ -92,6 +98,7 @@ class AllowSSH(base.Context):
self.secgroup.append(secgroup)
used_tenants.append(tenant)
@utils.log_task_wrapper(LOG.info, _("Exit context: `allow_ssh`"))
def cleanup(self):
for secgroup in self.secgroup:
try:

14
rally/benchmark/context/users.py
View File

@ -20,8 +20,10 @@ from rally.benchmark.context import base
from rally.benchmark import utils
from rally import consts
from rally.objects import endpoint
from rally.openstack.common.gettextutils import _
from rally.openstack.common import log as logging
from rally import osclients
from rally import utils as rutils
LOG = logging.getLogger(__name__)
@ -43,6 +45,8 @@ class UserGenerator(base.Context):
"""Context class for generating temporary users/tenants for benchmarks."""
__ctx_name__ = "users"
__ctx_order__ = 100
__ctx_hidden__ = False
CONFIG_SCHEMA = {
"type": "object",
@ -145,6 +149,7 @@ class UserGenerator(base.Context):
"Exception: %(ex)s" %
{"user_id": user["id"], "ex": ex})
@rutils.log_task_wrapper(LOG.info, _("Enter context: `users`"))
def setup(self):
"""Create tenants and users, using pool of threads."""
@ -164,6 +169,7 @@ class UserGenerator(base.Context):
self.context["tenants"].append(tenant)
self.context["users"] += users
@rutils.log_task_wrapper(LOG.info, _("Exit context: `users`"))
def cleanup(self):
"""Delete tenants and users, using pool of threads."""
@ -182,11 +188,3 @@ class UserGenerator(base.Context):
concurrent,
self._delete_tenants,
[(self.endpoint, tenants) for tenants in tenants_chunks])
# NOTE(amaretskiy): Consider that after cleanup() is complete, this has
# actually deleted (all or some of) users and tenants
# in openstack, but we *STILL HAVE*
# self.context["users"] and self.context["tenants"].
# Should we ignore that, or just reset these lists
# after cleanup() is done, or actually synchronize
# for all successfully deleted objects?

20
rally/benchmark/engine.py
View File

@ -102,7 +102,8 @@ class BenchmarkEngine(object):
for pos, kw in enumerate(values):
try:
base_runner.ScenarioRunner.validate(kw.get("runner", {}))
base_ctx.Context.validate(kw.get("context", {}))
base_ctx.ContextManager.validate(kw.get("context", {}),
non_hidden=True)
except (exceptions.RallyException,
jsonschema.ValidationError) as e:
raise exceptions.InvalidBenchmarkConfig(name=scenario,
@ -150,6 +151,12 @@ class BenchmarkEngine(object):
self.task.set_failed(log=log)
raise exceptions.InvalidTaskException(message=str(e))
def _get_runner(self, config):
runner = config.get("runner", {})
runner.setdefault("type", "continuous")
return base_runner.ScenarioRunner.get_runner(self.task, self.endpoints,
runner)
@rutils.log_task_wrapper(LOG.info, _("Benchmarking."))
def run(self):
"""Runs the benchmarks according to the test configuration
@ -161,12 +168,11 @@ class BenchmarkEngine(object):
self.task.update_status(consts.TaskStatus.RUNNING)
results = {}
for name in self.config:
for n, kwargs in enumerate(self.config[name]):
key = {'name': name, 'pos': n, 'kw': kwargs}
runner = kwargs.get("runner", {}).get("type", "continuous")
scenario_runner = base_runner.ScenarioRunner.get_runner(
self.task, self.endpoints, runner)
result = scenario_runner.run(name, kwargs)
for n, kw in enumerate(self.config[name]):
key = {'name': name, 'pos': n, 'kw': kw}
runner = self._get_runner(kw)
result = runner.run(name, kw.get("context", {}),
kw.get("args", {}))
self.task.append_results(key, {"raw": result})
results[json.dumps(key)] = result
self.task.update_status(consts.TaskStatus.FINISHED)

65
rally/benchmark/runners/base.py
View File

@ -19,11 +19,7 @@ import random
import jsonschema
from oslo.config import cfg
from rally.benchmark.context import cleaner as cleaner_ctx
from rally.benchmark.context import keypair as keypair_ctx
from rally.benchmark.context import quotas as quotas_ctx
from rally.benchmark.context import secgroup as secgroup_ctx
from rally.benchmark.context import users as users_ctx
from rally.benchmark.context import base as base_ctx
from rally.benchmark.scenarios import base
from rally.benchmark import utils
from rally import exceptions
@ -145,13 +141,14 @@ class ScenarioRunner(object):
CONFIG_SCHEMA = {}
def __init__(self, task, endpoints):
def __init__(self, task, endpoints, config):
self.task = task
self.endpoints = endpoints
# NOTE(msdubov): Passing predefined user endpoints hasn't been
# implemented yet, so the scenario runner always gets
# a single admin endpoint here.
self.admin_user = endpoints[0]
self.config = config
@staticmethod
def _get_cls(runner_type):
@ -161,9 +158,9 @@ class ScenarioRunner(object):
raise exceptions.NoSuchRunner(type=runner_type)
@staticmethod
def get_runner(task, endpoint, runner_type):
def get_runner(task, endpoint, config):
"""Returns instance of a scenario runner for execution type."""
return ScenarioRunner._get_cls(runner_type)(task, endpoint)
return ScenarioRunner._get_cls(config["type"])(task, endpoint, config)
@staticmethod
def validate(config):
@ -172,7 +169,7 @@ class ScenarioRunner(object):
jsonschema.validate(config, runner.CONFIG_SCHEMA)
@abc.abstractmethod
def _run_scenario(self, cls, method_name, context, args, config):
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified benchmark scenario with given arguments.
:param cls: The Scenario class where the scenario is implemented
@ -180,57 +177,33 @@ class ScenarioRunner(object):
:param context: Benchmark context that contains users, admin & other
information, that was created before benchmark started.
:param args: Arguments to call the scenario method with
:param config: Configuration dictionary that contains strategy-specific
parameters like the number of times to run the scenario
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
def _prepare_and_run_scenario(self, context, name, kwargs):
def run(self, name, context, args):
cls_name, method_name = name.split(".", 1)
cls = base.Scenario.get_by_name(cls_name)
args = kwargs.get('args', {})
config = kwargs.get('runner', {})
scenario_context = getattr(cls, method_name).context
# TODO(boris-42): We should keep default behavior for `users` context
# as a part of work on pre-created users this should be
# removed.
scenario_context.setdefault("users", {})
# merge scenario context and task context configuration
scenario_context.update(context)
with secgroup_ctx.AllowSSH(context) as allow_ssh:
allow_ssh.setup()
with keypair_ctx.Keypair(context) as keypair:
keypair.setup()
LOG.debug("Context: %s" % context)
return self._run_scenario(cls, method_name, context,
args, config)
def _run_as_admin(self, name, kwargs):
context = {
context_obj = {
"task": self.task,
"admin": {"endpoint": self.admin_user},
"scenario_name": name,
"config": kwargs.get("context", {})
"config": scenario_context
}
with users_ctx.UserGenerator(context) as generator:
generator.setup()
with quotas_ctx.Quotas(context) as quotas:
quotas.setup()
with cleaner_ctx.ResourceCleaner(context) as cleaner:
cleaner.setup()
return self._prepare_and_run_scenario(context,
name, kwargs)
def _run_as_non_admin(self, name, kwargs):
# TODO(boris-42): It makes sense to use UserGenerator here as well
# take a look at comment in UserGenerator.__init__()
context = {"scenario_name": name}
with cleaner_ctx.ResourceCleaner(context):
return self._prepare_and_run_scenario(context, name, kwargs)
def run(self, name, kwargs):
if self.admin_user:
results = self._run_as_admin(name, kwargs)
else:
results = self._run_as_non_admin(name, kwargs)
results = base_ctx.ContextManager.run(context_obj, self._run_scenario,
cls, method_name, context_obj,
args)
if not isinstance(results, ScenarioRunnerResult):
name = self.__execution_type__

18
rally/benchmark/runners/continuous.py
View File

@ -126,25 +126,25 @@ class ContinuousScenarioRunner(base.ScenarioRunner):
return results
def _run_scenario(self, cls, method_name, context, args, config):
def _run_scenario(self, cls, method_name, context, args):
timeout = config.get("timeout", 600)
concurrent = config.get("active_users", 1)
timeout = self.config.get("timeout", 600)
concurrent = self.config.get("active_users", 1)
# NOTE(msdubov): If not specified, perform single scenario run.
if "duration" not in config and "times" not in config:
config["times"] = 1
if "duration" not in self.config and "times" not in self.config:
self.config["times"] = 1
# Continiously run a benchmark scenario the specified
# amount of times.
if "times" in config:
times = config["times"]
if "times" in self.config:
times = self.config["times"]
results = self._run_scenario_continuously_for_times(
cls, method_name, context, args, times, concurrent, timeout)
# Continiously run a scenario as many times as needed
# to fill up the given period of time.
elif "duration" in config:
duration = config["duration"]
elif "duration" in self.config:
duration = self.config["duration"]
results = self._run_scenario_continuously_for_duration(
cls, method_name, context, args, duration, concurrent, timeout)

8
rally/benchmark/runners/periodic.py
View File

@ -61,11 +61,11 @@ class PeriodicScenarioRunner(base.ScenarioRunner):
"additionalProperties": False
}
def _run_scenario(self, cls, method_name, context, args, config):
def _run_scenario(self, cls, method_name, context, args):
times = config["times"]
period = config["period"]
timeout = config.get("timeout", 600)
times = self.config["times"]
period = self.config["period"]
timeout = self.config.get("timeout", 600)
async_results = []

5
rally/benchmark/runners/serial.py
View File

@ -45,9 +45,8 @@ class SerialScenarioRunner(base.ScenarioRunner):
"additionalProperties": True
}
def _run_scenario(self, cls, method_name, context, args, config):
times = config.get('times', 1)
def _run_scenario(self, cls, method_name, context, args):
times = self.config.get('times', 1)
results = []

2
rally/benchmark/scenarios/authenticate/authenticate.py
View File

@ -12,7 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.benchmark.context import cleaner as context_cleaner
from rally.benchmark.scenarios import base
@ -22,6 +21,5 @@ class Authenticate(base.Scenario):
"""
@base.scenario()
@context_cleaner.cleanup([])
def keystone(self, **kwargs):
self.clients("keystone")

3
rally/benchmark/scenarios/base.py
View File

@ -22,7 +22,7 @@ from rally import exceptions
from rally import utils
def scenario(admin_only=False):
def scenario(admin_only=False, context=None):
"""This method is used as decorator for the methods of benchmark scenarios
and it adds following extra fields to the methods.
'is_scenario' is set to True
@ -31,6 +31,7 @@ def scenario(admin_only=False):
def wrapper(func):
func.is_scenario = True
func.admin_only = admin_only
func.context = context or {}
return func
return wrapper

10
rally/benchmark/scenarios/cinder/volumes.py
View File

@ -13,15 +13,13 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.benchmark.context import cleaner as context_cleaner
from rally.benchmark.scenarios import base
from rally.benchmark.scenarios.cinder import utils
class CinderVolumes(utils.CinderScenario):
@base.scenario()
@context_cleaner.cleanup(['cinder'])
@base.scenario(context={"cleanup": ["cinder"]})
def create_and_list_volume(self, size, detailed=True, **kwargs):
"""Tests creating a volume and listing volumes.
@ -38,8 +36,7 @@ class CinderVolumes(utils.CinderScenario):
self._create_volume(size, **kwargs)
self._list_volumes(detailed)
@base.scenario()
@context_cleaner.cleanup(['cinder'])
@base.scenario(context={"cleanup": ["cinder"]})
def create_and_delete_volume(self, size, min_sleep=0, max_sleep=0,
**kwargs):
"""Tests creating and then deleting a volume.
@ -51,8 +48,7 @@ class CinderVolumes(utils.CinderScenario):
self.sleep_between(min_sleep, max_sleep)
self._delete_volume(volume)
@base.scenario()
@context_cleaner.cleanup(['cinder'])
@base.scenario(context={"cleanup": ["cinder"]})
def create_volume(self, size, **kwargs):
"""Test creating volumes perfromance.

10
rally/benchmark/scenarios/glance/images.py
View File

@ -13,7 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.benchmark.context import cleaner as context_cleaner
from rally.benchmark.scenarios import base
from rally.benchmark.scenarios.glance import utils
from rally.benchmark.scenarios.nova import utils as nova_utils
@ -22,8 +21,7 @@ from rally.benchmark import validation
class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
@base.scenario()
@context_cleaner.cleanup(['glance'])
@base.scenario(context={"cleanup": ["glance"]})
def create_and_list_image(self, container_format,
image_location, disk_format, **kwargs):
"""Test adding an image and then listing all images.
@ -46,8 +44,7 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
**kwargs)
self._list_images()
@base.scenario()
@context_cleaner.cleanup(['glance'])
@base.scenario(context={"cleanup": ["glance"]})
def create_and_delete_image(self, container_format,
image_location, disk_format, **kwargs):
"""Test adds and then deletes image."""
@ -59,9 +56,8 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario):
**kwargs)
self._delete_image(image)
@context_cleaner.cleanup(['glance', 'nova'])
@validation.add_validator(validation.flavor_exists("flavor_id"))
@base.scenario()
@base.scenario(context={"cleanup": ["glance", "nova"]})
def create_image_and_boot_instances(self, container_format,
image_location, disk_format,
flavor_id, number_instances,

12
rally/benchmark/scenarios/keystone/basic.py
View File

@ -13,7 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.benchmark.context import cleaner as context_cleaner
from rally.benchmark.scenarios import base
from rally.benchmark.scenarios.keystone import utils as kutils
from rally.benchmark import validation as valid
@ -21,24 +20,21 @@ from rally.benchmark import validation as valid
class KeystoneBasic(kutils.KeystoneScenario):
@base.scenario(admin_only=True)
@base.scenario(admin_only=True, context={"cleanup": []})
def create_user(self, name_length=10, **kwargs):
self._user_create(name_length=name_length, **kwargs)
@base.scenario(admin_only=True)
@context_cleaner.cleanup([])
@base.scenario(admin_only=True, context={"cleanup": []})
def create_delete_user(self, name_length=10, **kwargs):
user = self._user_create(name_length=name_length, **kwargs)
self._resource_delete(user)
@base.scenario(admin_only=True)
@context_cleaner.cleanup([])
@base.scenario(admin_only=True, context={"cleanup": []})
def create_tenant(self, name_length=10, **kwargs):
self._tenant_create(name_length=name_length, **kwargs)
@base.scenario(admin_only=True)
@base.scenario(admin_only=True, context={"cleanup": []})
@valid.add_validator(valid.required_parameters(['users_per_tenant']))
@context_cleaner.cleanup([])
def create_tenant_with_users(self, users_per_tenant, name_length=10,
**kwargs):
tenant = self._tenant_create(name_length=name_length, **kwargs)

26
rally/benchmark/scenarios/nova/servers.py
View File

@ -17,7 +17,6 @@ import json
import jsonschema
import random
from rally.benchmark.context import cleaner as context_cleaner
from rally.benchmark.scenarios import base
from rally.benchmark.scenarios.cinder import utils as cinder_utils
from rally.benchmark.scenarios.nova import utils
@ -38,9 +37,8 @@ class NovaServers(utils.NovaScenario,
def __init__(self, *args, **kwargs):
super(NovaServers, self).__init__(*args, **kwargs)
@context_cleaner.cleanup(['nova'])
@valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id"))
@base.scenario()
@base.scenario(context={"cleanup": ["nova"]})
def boot_and_list_server(self, image_id, flavor_id,
detailed=True, **kwargs):
"""Tests booting an image and then listing servers.
@ -60,9 +58,8 @@ class NovaServers(utils.NovaScenario,
self._boot_server(server_name, image_id, flavor_id, **kwargs)
self._list_servers(detailed)
@context_cleaner.cleanup(['nova'])
@valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id"))
@base.scenario()
@base.scenario(context={"cleanup": ["nova"]})
def boot_and_delete_server(self, image_id, flavor_id,
min_sleep=0, max_sleep=0, **kwargs):
"""Tests booting and then deleting an image."""
@ -72,9 +69,8 @@ class NovaServers(utils.NovaScenario,
self.sleep_between(min_sleep, max_sleep)
self._delete_server(server)
@context_cleaner.cleanup(['nova', 'cinder'])
@valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id"))
@base.scenario()
@base.scenario(context={"cleanup": ["nova", "cinder"]})
def boot_server_from_volume_and_delete(self, image_id, flavor_id,
volume_size,
min_sleep=0, max_sleep=0, **kwargs):
@ -89,9 +85,9 @@ class NovaServers(utils.NovaScenario,
self.sleep_between(min_sleep, max_sleep)
self._delete_server(server)
@context_cleaner.cleanup(['nova'])
@valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id"))
@base.scenario()
@base.scenario(context={"cleanup": ["nova"],
"keypair": {}, "allow_ssh": {}})
def boot_runcommand_delete_server(self, image_id, flavor_id,
script, interpreter, network='private',
username='ubuntu', ip_version=4,
@ -147,9 +143,8 @@ class NovaServers(utils.NovaScenario,
stdout=out, stderr=err))
return {"data": out, "errors": err}
@context_cleaner.cleanup(['nova'])
@valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id"))
@base.scenario()
@base.scenario(context={"cleanup": ["nova"]})
def boot_and_bounce_server(self, image_id, flavor_id, **kwargs):
"""Tests booting a server then performing stop/start or hard/soft
reboot a number of times.
@ -168,9 +163,8 @@ class NovaServers(utils.NovaScenario,
action()
self._delete_server(server)
@context_cleaner.cleanup(['nova', 'glance'])
@valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id"))
@base.scenario()
@base.scenario(context={"cleanup": ["nova", "glance"]})
def snapshot_server(self, image_id, flavor_id, **kwargs):
"""Tests Nova instance snapshotting."""
server_name = self._generate_random_name(16)
@ -183,9 +177,8 @@ class NovaServers(utils.NovaScenario,
self._delete_server(server)
self._delete_image(image)
@context_cleaner.cleanup(['nova'])
@valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id"))
@base.scenario()
@base.scenario(context={"cleanup": ["nova"]})
def boot_server(self, image_id, flavor_id, **kwargs):
"""Test VM boot - assumed clean-up is done elsewhere."""
server_name = self._generate_random_name(16)
@ -196,9 +189,8 @@ class NovaServers(utils.NovaScenario,
kwargs['nics'] = [{'net-id': random_nic.id}]
self._boot_server(server_name, image_id, flavor_id, **kwargs)
@context_cleaner.cleanup(['nova', 'cinder'])
@valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id"))
@base.scenario()
@base.scenario(context={"cleanup": ["nova", "cinder"]})
def boot_server_from_volume(self, image_id, flavor_id,
volume_size, **kwargs):
"""Test VM boot from volume - assumed clean-up is done elsewhere."""

13
rally/benchmark/scenarios/nova/utils.py
View File

@ -18,7 +18,6 @@ import random
import string
import time
from rally.benchmark.context import secgroup
from rally.benchmark.scenarios import base
from rally.benchmark.scenarios import utils as scenario_utils
from rally.benchmark import utils as bench_utils
@ -86,12 +85,12 @@ class NovaScenario(base.Scenario):
:returns: Created server object
"""
if 'security_groups' not in kwargs:
kwargs['security_groups'] = [secgroup.SSH_GROUP_NAME]
else:
if secgroup.SSH_GROUP_NAME not in kwargs['security_groups']:
kwargs['security_groups'].append(secgroup.SSH_GROUP_NAME)
allow_ssh_secgroup = self.context().get("allow_ssh")
if allow_ssh_secgroup:
if 'security_groups' not in kwargs:
kwargs['security_groups'] = [allow_ssh_secgroup]
elif allow_ssh_secgroup not in kwargs['security_groups']:
kwargs['security_groups'].append(allow_ssh_secgroup)
server = self.clients("nova").servers.create(server_name, image_id,
flavor_id, **kwargs)

105
tests/benchmark/context/test_base.py
View File

@ -49,24 +49,11 @@ class BaseContextTestCase(test.TestCase):
self.assertEqual(ctx.context, context)
def test_validate__context(self):
context = {
"fake": {"test": 2}
}
base.Context.validate(context)
fakes.FakeContext.validate({"test": 2})
def test_validate__wrong_context(self):
context = {
"fake": {"nonexisting": 2}
}
self.assertRaises(jsonschema.ValidationError,
base.Context.validate, context)
def test_validate__non_existing_context(self):
config = {
"nonexisting": {"nonexisting": 2}
}
self.assertRaises(exceptions.NoSuchContext,
base.Context.validate, config)
fakes.FakeContext.validate, {"nonexisting": 2})
@mock.patch("rally.benchmark.context.base.utils.itersubclasses")
def test_get_by_name(self, mock_itersubclasses):
@ -85,6 +72,10 @@ class BaseContextTestCase(test.TestCase):
self.assertRaises(exceptions.NoSuchContext,
base.Context.get_by_name, "nonexisting")
def test_get_by_name_hidder(self):
self.assertRaises(exceptions.NoSuchContext,
base.Context.validate, {}, non_hidden=True)
def test_setup_is_abstract(self):
class A(base.Context):
@ -114,3 +105,87 @@ class BaseContextTestCase(test.TestCase):
self.assertEqual(ctx, entered_ctx)
ctx.cleanup.assert_called_once_with()
class ContextManagerTestCase(test.TestCase):
@mock.patch("rally.benchmark.context.base.ContextManager._magic")
@mock.patch("rally.benchmark.context.base.Context.get_by_name")
def test_run(self, mock_get, mock_magic):
context = {
"config": {
"a": mock.MagicMock(),
"b": mock.MagicMock()
}
}
cc = mock.MagicMock()
cc.__ctx_order__ = 10
mock_get.return_value = cc
mock_magic.return_value = 5
result = base.ContextManager.run(context, lambda x, y: x + y, 1, 2)
self.assertEqual(result, 5)
mock_get.assert_has_calls([
mock.call("a"),
mock.call("b"),
mock.call()(context),
mock.call()(context)
])
@mock.patch("rally.benchmark.context.base.Context.get_by_name")
def test_validate(self, mock_get):
config = {
"ctx1": mock.MagicMock(),
"ctx2": mock.MagicMock()
}
base.ContextManager.validate(config)
mock_get.assert_has_calls([
mock.call("ctx1"),
mock.call().validate(config["ctx1"], non_hidden=False),
mock.call("ctx2"),
mock.call().validate(config["ctx2"], non_hidden=False)
])
@mock.patch("rally.benchmark.context.base.Context.get_by_name")
def test_validate_non_hidden(self, mock_get):
config = {
"ctx1": mock.MagicMock(),
"ctx2": mock.MagicMock()
}
base.ContextManager.validate(config, non_hidden=True)
mock_get.assert_has_calls([
mock.call("ctx1"),
mock.call().validate(config["ctx1"], non_hidden=True),
mock.call("ctx2"),
mock.call().validate(config["ctx2"], non_hidden=True)
])
def test_validate__non_existing_context(self):
config = {
"nonexisting": {"nonexisting": 2}
}
self.assertRaises(exceptions.NoSuchContext,
base.ContextManager.validate, config)
def test__magic(self):
func = lambda x, y: x + y
result = base.ContextManager._magic([], func, 2, 3)
self.assertEqual(result, 5)
def test__magic_with_ctx(self):
ctx = [mock.MagicMock(), mock.MagicMock()]
func = lambda x, y: x + y
result = base.ContextManager._magic(ctx, func, 2, 3)
self.assertEqual(result, 5)
expected = [mock.call.__enter__(), mock.call.setup(),
mock.call.__exit__(None, None, None)]
for c in ctx:
ctx[0].assert_has_calls(expected)

28
tests/benchmark/context/test_cleaner.py
View File

@ -32,7 +32,6 @@ class ResourceCleanerTestCase(test.TestCase):
"admin": None,
"users": [],
"tenants": [],
"scenario_name": "NovaServers.boot_server_from_volume_and_delete"
}
resource_cleaner = cleaner_ctx.ResourceCleaner(context)
with resource_cleaner:
@ -40,6 +39,7 @@ class ResourceCleanerTestCase(test.TestCase):
def test_with_statement(self):
fake_user_ctx = fakes.FakeUserContext({}).context
fake_user_ctx["config"] = {"cleanup": ["nova"]}
res_cleaner = cleaner_ctx.ResourceCleaner(fake_user_ctx)
res_cleaner._cleanup_users_resources = mock.MagicMock()
@ -56,7 +56,7 @@ class ResourceCleanerTestCase(test.TestCase):
def test_cleaner_admin(self, mock_del_keystone, mock_clients):
context = {
"task": mock.MagicMock(),
"scenario_name": 'NovaServers.boot_server_from_volume_and_delete',
"config": {"cleanup": ["cinder", "nova"]},
"admin": {"endpoint": mock.MagicMock()},
}
res_cleaner = cleaner_ctx.ResourceCleaner(context)
@ -70,21 +70,18 @@ class ResourceCleanerTestCase(test.TestCase):
mock_clients.return_value.keystone.assert_called_once_with()
mock_del_keystone.assert_called_once_with('keystone')
@mock.patch("rally.benchmark.scenarios.nova.servers.NovaServers."
"boot_server_from_volume_and_delete")
@mock.patch("%s.osclients.Clients" % BASE)
@mock.patch("%s.utils.delete_nova_resources" % BASE)
@mock.patch("%s.utils.delete_glance_resources" % BASE)
@mock.patch("%s.utils.delete_cinder_resources" % BASE)
def test_cleaner_users_all_services(self, mock_del_cinder,
mock_del_glance, mock_del_nova,
mock_clients, mock_scenario_method):
del mock_scenario_method.cleanup_services
mock_clients):
context = {
"task": mock.MagicMock(),
"users": [{"endpoint": mock.MagicMock()},
{"endpoint": mock.MagicMock()}],
"scenario_name": "NovaServers.boot_server_from_volume_and_delete",
"config": {"cleanup": ["cinder", "nova", "glance"]},
"tenants": [mock.MagicMock()]
}
res_cleaner = cleaner_ctx.ResourceCleaner(context)
@ -100,18 +97,31 @@ class ResourceCleanerTestCase(test.TestCase):
self.assertEqual(mock_del_glance.call_count, 2)
self.assertEqual(mock_del_cinder.call_count, 2)
@mock.patch("%s.ResourceCleaner._cleanup_users_resources" % BASE)
def test_cleaner_users_default_behavior(self, mock_cleanup):
context = {
"task": mock.MagicMock(),
"users": [{"endpoint": mock.MagicMock()},
{"endpoint": mock.MagicMock()}],
}
res_cleaner = cleaner_ctx.ResourceCleaner(context)
with res_cleaner:
res_cleaner.setup()
self.assertEqual(mock_cleanup.call_count, 0)
@mock.patch("%s.osclients.Clients" % BASE)
@mock.patch("%s.utils.delete_nova_resources" % BASE)
@mock.patch("%s.utils.delete_glance_resources" % BASE)
@mock.patch("%s.utils.delete_cinder_resources" % BASE)
def test_cleaner_users_by_service(self, mock_del_cinder, mock_del_glance,
mock_del_nova, mock_clients):
context = {
"task": mock.MagicMock(),
"users": [{"endpoint": mock.MagicMock()},
{"endpoint": mock.MagicMock()}],
"scenario_name": 'NovaServers.boot_server_from_volume_and_delete',
"config": {"cleanup": ["cinder", "nova"]},
"tenants": [mock.MagicMock()]
}
res_cleaner = cleaner_ctx.ResourceCleaner(context)

7
tests/benchmark/context/test_keypair.py
View File

@ -26,15 +26,16 @@ class KeyPairContextTestCase(test.TestCase):
def setUp(self):
super(KeyPairContextTestCase, self).setUp()
self.users = 2
task = mock.MagicMock()
self.ctx_with_keys = {
"users": [
{"keypair": "key", "endpoint": "endpoint"},
] * self.users,
"task": {}
"task": task
}
self.ctx_without_keys = {
"users": [{'endpoint': 'endpoint'}] * self.users,
"task": {}
"users": [{'endpoint': 'endpoint'}] * self.users,
"task": task
}
@mock.patch("%s.keypair.Keypair._generate_keypair" % CTX)

16
tests/benchmark/context/test_quotas.py
View File

@ -97,7 +97,7 @@ class QuotasTestCase(test.TestCase):
{"endpoint": mock.MagicMock(), "id": mock.MagicMock()}
],
"admin": {"endpoint": mock.MagicMock()},
"task": {}
"task": mock.MagicMock()
}
def test_quotas_schemas(self):
@ -128,7 +128,7 @@ class QuotasTestCase(test.TestCase):
# Test invalid values
ctx["config"]["quotas"][service][key] = self.unlimited - 1
try:
quotas.Quotas.validate(ctx["config"])
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
pass
else:
@ -137,7 +137,7 @@ class QuotasTestCase(test.TestCase):
ctx["config"]["quotas"][service][key] = 2.5
try:
quotas.Quotas.validate(ctx["config"])
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
pass
else:
@ -146,7 +146,7 @@ class QuotasTestCase(test.TestCase):
ctx["config"]["quotas"][service][key] = "-1"
try:
quotas.Quotas.validate(ctx["config"])
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
pass
else:
@ -157,20 +157,20 @@ class QuotasTestCase(test.TestCase):
ctx["config"]["quotas"][service][key] = \
random.randint(0, 1000000)
try:
quotas.Quotas.validate(ctx["config"])
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
self.fail("Positive integers are valid quota values")
ctx["config"]["quotas"][service][key] = self.unlimited
try:
quotas.Quotas.validate(ctx["config"])
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
self.fail("%d is a valid quota value" % self.unlimited)
# Test additional keys are refused
ctx["config"]["quotas"][service]["additional"] = self.unlimited
try:
quotas.Quotas.validate(ctx["config"])
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
pass
else:
@ -180,7 +180,7 @@ class QuotasTestCase(test.TestCase):
# Test valid keys are optional
ctx["config"]["quotas"][service] = {}
try:
quotas.Quotas.validate(ctx["config"])
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
self.fail("Valid quota keys are optional")

46
tests/benchmark/runners/test_base.py
View File

@ -18,6 +18,7 @@ import mock
from rally.benchmark.runners import base
from rally.benchmark.runners import continuous
from rally.benchmark.scenarios import base as base_scenario
from rally import consts
from rally import exceptions
from tests import fakes
@ -180,17 +181,19 @@ class ScenarioRunnerTestCase(test.TestCase):
task = mock.MagicMock()
endpoints = [mock.MagicMock(), mock.MagicMock()]
runner = base.ScenarioRunner.get_runner(task, endpoints, "new_runner")
config = {"type": "new_runner", "a": 123}
runner = base.ScenarioRunner.get_runner(task, endpoints, config)
self.assertEqual(runner.task, task)
self.assertEqual(runner.endpoints, endpoints)
self.assertEqual(runner.admin_user, endpoints[0])
self.assertEqual(runner.config, config)
self.assertIsInstance(runner, NewRunner)
def test_get_runner_no_such(self):
self.assertRaises(exceptions.NoSuchRunner,
base.ScenarioRunner.get_runner,
None, None, "NoSuchRunner")
None, None, {"type": "NoSuchRunner"})
@mock.patch("rally.benchmark.runners.base.jsonschema.validate")
def test_validate_default_runner(self, mock_validate):
@ -200,10 +203,39 @@ class ScenarioRunnerTestCase(test.TestCase):
config,
continuous.ContinuousScenarioRunner.CONFIG_SCHEMA)
@mock.patch("rally.benchmark.runners.base.ScenarioRunner._run_as_admin")
def test_run_scenario_runner_results_exception(self, mock_run_method):
@mock.patch("rally.benchmark.runners.base.base_ctx.ContextManager")
def test_run(self, mock_ctx_manager):
runner = continuous.ContinuousScenarioRunner(mock.MagicMock(),
self.fake_endpoints)
self.fake_endpoints,
mock.MagicMock())
mock_ctx_manager.run.return_value = base.ScenarioRunnerResult([])
scenario_name = "NovaServers.boot_server_from_volume_and_delete"
result = runner.run(scenario_name, {"some_ctx": 2}, [1, 2, 3])
self.assertEqual(result, mock_ctx_manager.run.return_value)
cls_name, method_name = scenario_name.split(".", 1)
cls = base_scenario.Scenario.get_by_name(cls_name)
context_obj = {
"task": runner.task,
"admin": {"endpoint": runner.admin_user},
"scenario_name": scenario_name,
"config": {
"cleanup": ["nova", "cinder"], "some_ctx": 2, "users": {}
}
}
expected = [context_obj, runner._run_scenario, cls, method_name,
context_obj, [1, 2, 3]]
mock_ctx_manager.run.assert_called_once_with(*expected)
@mock.patch("rally.benchmark.runners.base.base_ctx.ContextManager")
def test_run__scenario_runner_results_exception(self, mock_ctx_manager):
runner = continuous.ContinuousScenarioRunner(mock.MagicMock(),
self.fake_endpoints,
mock.MagicMock())
self.assertRaises(exceptions.InvalidRunnerResult,
runner.run, "NovaServers.boot_server_from_volume_"
"and_delete", mock.MagicMock())
runner.run,
"NovaServers.boot_server_from_volume_and_delete",
mock.MagicMock(), {})

7
tests/benchmark/runners/test_continuous.py
View File

@ -41,8 +41,9 @@ class ContinuousScenarioRunnerTestCase(test.TestCase):
def test_run_scenario_continuously_for_times(self):
context = fakes.FakeUserContext({"task": None}).context
runner = continuous.ContinuousScenarioRunner(
None, [context["admin"]["endpoint"]])
None, [context["admin"]["endpoint"]], {})
times = 4
concurrent = 2
timeout = 2
@ -56,7 +57,7 @@ class ContinuousScenarioRunnerTestCase(test.TestCase):
def test_run_scenario_continuously_for_times_exception(self):
context = fakes.FakeUserContext({"task": None}).context
runner = continuous.ContinuousScenarioRunner(
None, [context["admin"]["endpoint"]])
None, [context["admin"]["endpoint"]], {})
times = 4
concurrent = 2
timeout = 2
@ -72,7 +73,7 @@ class ContinuousScenarioRunnerTestCase(test.TestCase):
self.skipTest("This test produce a lot of races so we should fix it "
"before running inside in gates")
runner = continuous.ContinuousScenarioRunner(mock.MagicMock(),
[mock.MagicMock()])
[mock.MagicMock()], {})
duration = 0
active_users = 4
timeout = 5

37
tests/benchmark/runners/test_periodic.py
View File

@ -48,29 +48,24 @@ class PeriodicScenarioRunnerTestCase(test.TestCase):
def test_run_scenario(self):
context = fakes.FakeUserContext({}).context
config = {"times": 3, "period": 0, "timeout": 5}
runner = periodic.PeriodicScenarioRunner(
None, [context["admin"]["endpoint"]])
times = 3
period = 0
None, [context["admin"]["endpoint"]], config)
result = runner._run_scenario(fakes.FakeScenario, "do_it", context, {},
{"times": times, "period": period,
"timeout": 5})
self.assertEqual(len(result), times)
result = runner._run_scenario(fakes.FakeScenario, "do_it", context, {})
self.assertEqual(len(result), config["times"])
self.assertIsNotNone(base.ScenarioRunnerResult(result))
def test_run_scenario_exception(self):
context = fakes.FakeUserContext({}).context
config = {"times": 4, "period": 0}
runner = periodic.PeriodicScenarioRunner(
None, [context["admin"]["endpoint"]])
times = 4
period = 0
None, [context["admin"]["endpoint"]], config)
result = runner._run_scenario(fakes.FakeScenario,
"something_went_wrong", context, {},
{"times": times, "period": period,
"timeout": 5})
self.assertEqual(len(result), times)
"something_went_wrong", context, {})
self.assertEqual(len(result), config["times"])
self.assertIsNotNone(base.ScenarioRunnerResult(result))
@mock.patch("rally.benchmark.runners.periodic.base.ScenarioRunnerResult")
@ -79,19 +74,17 @@ class PeriodicScenarioRunnerTestCase(test.TestCase):
def test_run_scenario_internal_logic(self, mock_time, mock_pool,
mock_result):
context = fakes.FakeUserContext({}).context
config = {"times": 4, "period": 0, "timeout": 5}
runner = periodic.PeriodicScenarioRunner(
None, [context["admin"]["endpoint"]])
times = 4
period = 0
None, [context["admin"]["endpoint"]], config)
mock_pool_inst = mock.MagicMock()
mock_pool.ThreadPool.return_value = mock_pool_inst
runner._run_scenario(fakes.FakeScenario, "do_it", context, {},
{"times": times, "period": period, "timeout": 5})
runner._run_scenario(fakes.FakeScenario, "do_it", context, {})
exptected_pool_inst_call = []
for i in range(times):
for i in range(config["times"]):
args = (
base._run_scenario_once,
((i, fakes.FakeScenario, "do_it",
@ -99,7 +92,7 @@ class PeriodicScenarioRunnerTestCase(test.TestCase):
)
exptected_pool_inst_call.append(mock.call.apply_async(*args))
for i in range(times):
for i in range(config["times"]):
call = mock.call.apply_async().get(timeout=5)
exptected_pool_inst_call.append(call)
@ -117,5 +110,5 @@ class PeriodicScenarioRunnerTestCase(test.TestCase):
runner = base.ScenarioRunner.get_runner(mock.MagicMock(),
self.fake_endpoints,
"periodic")
{"type": "periodic"})
self.assertTrue(runner is not None)

6
tests/benchmark/runners/test_serial.py
View File

@ -40,9 +40,9 @@ class SerialScenarioRunnerTestCase(test.TestCase):
expected_results = [result for i in range(times)]
runner = serial.SerialScenarioRunner(mock.MagicMock(),
self.fake_endpoints)
self.fake_endpoints,
{"times": times})
results = runner._run_scenario(fakes.FakeScenario, "do_it",
fakes.FakeUserContext({}).context,
{}, {"type": "serial", "times": times})
fakes.FakeUserContext({}).context, {})
self.assertEqual(mock_run_once.call_count, times)
self.assertEqual(results, expected_results)

2
tests/benchmark/scenarios/nova/test_utils.py
View File

@ -80,7 +80,7 @@ class NovaScenarioTestCase(test.TestCase):
@mock.patch(NOVA_UTILS + '.NovaScenario.clients')
def test__boot_server(self, mock_clients):
mock_clients("nova").servers.create.return_value = self.server
nova_scenario = utils.NovaScenario()
nova_scenario = utils.NovaScenario(context={})
return_server = nova_scenario._boot_server('server_name', 'image_id',
'flavor_id')
self.wait_for.mock.assert_called_once_with(

16
tests/benchmark/scenarios/test_base.py
View File

@ -14,7 +14,9 @@
# under the License.
import mock
import traceback
from rally.benchmark.context import base as base_ctx
from rally.benchmark.scenarios import base
from rally.benchmark import validation
from rally import consts
@ -173,3 +175,17 @@ class ScenarioTestCase(test.TestCase):
scenario = base.Scenario(admin_clients=clients)
self.assertEqual(clients.nova(), scenario.admin_clients("nova"))
self.assertEqual(clients.glance(), scenario.admin_clients("glance"))
def test_scenario_context_are_valid(self):
scenarios = base.Scenario.list_benchmark_scenarios()
for scenario in scenarios:
cls_name, method_name = scenario.split(".", 1)
cls = base.Scenario.get_by_name(cls_name)
context = getattr(cls, method_name).context
try:
base_ctx.ContextManager.validate(context)
except Exception:
print(traceback.format_exc())
self.assertTrue(False,
"Scenario `%s` has wrong context" % scenario)

9
tests/benchmark/test_engine.py
View File

@ -121,16 +121,17 @@ class BenchmarkEngineTestCase(test.TestCase):
eng._validate_config_scenarios_name, config)
@mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner.validate")
@mock.patch("rally.benchmark.engine.base_ctx.Context.validate")
@mock.patch("rally.benchmark.engine.base_ctx.ContextManager.validate")
def test__validate_config_syntax(self, mock_context, mock_runner):
config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]}
eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock())
eng._validate_config_syntax(config)
mock_runner.assert_has_calls([mock.call({}), mock.call("b")])
mock_context.assert_has_calls([mock.call("a"), mock.call({})])
mock_context.assert_has_calls([mock.call("a", non_hidden=True),
mock.call({}, non_hidden=True)])
@mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner")
@mock.patch("rally.benchmark.engine.base_ctx.Context.validate")
@mock.patch("rally.benchmark.engine.base_ctx.ContextManager.validate")
def test__validate_config_syntax__wrong_runner(self, mock_context,
mock_runner):
config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]}
@ -142,7 +143,7 @@ class BenchmarkEngineTestCase(test.TestCase):
eng._validate_config_syntax, config)
@mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner.validate")
@mock.patch("rally.benchmark.engine.base_ctx.Context")
@mock.patch("rally.benchmark.engine.base_ctx.ContextManager")
def test__validate_config_syntax__wrong_context(self, mock_context,
mock_runner):
config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]}

2
tests/doc/test_task_samples.py
View File

@ -43,7 +43,7 @@ class TaskSampleTestCase(test.TestCase):
eng = engine.BenchmarkEngine(task_config,
mock.MagicMock())
eng.validate()
except Exception :
except Exception:
print(traceback.format_exc())
self.assertTrue(False,
"Wrong task config %s" % full_path)