From 4f7303fe864fd64b16d9feed2dabcb778f0cf6d2 Mon Sep 17 00:00:00 2001 From: Boris Pavlovic Date: Sun, 6 Apr 2014 17:50:16 +0400 Subject: [PATCH] Add benchmark-context manager This patch actually mostly finish support of benchmark-context. * Add context manager that allows to use only specified context. Context could be specified through 2 ways: 1) @base.scenario(context={...}) 2) via task config * Support of two types of context: 1) __ctx_hidden__ = True => could be setup only using @base.scenario 2) __ctx_hidden__ = False => could be setup in both ways * Support of validation of context types * Support of order of loading context __ctx_order__ Context manager will use this number to sort context and chose order of creating them. * Unified "cleanup" context to work only via @base.scenario() * Great Rally perf optimization - use only required context * Cleanup decorator is merged to @base.scenario * Add logging for context * Improve NovaScenario._boot_server to add to secgroup allow_ssh only when we are using allow_ssh context * Add unit test that checks that all scenario have proper predefined context config blueprint benchmark-context Change-Id: I4fce466a88075a694a87e72ace72cc522605b72a --- rally/benchmark/context/base.py | 70 +++++++++--- rally/benchmark/context/cleaner.py | 42 +++---- rally/benchmark/context/keypair.py | 7 ++ rally/benchmark/context/quotas.py | 10 ++ rally/benchmark/context/secgroup.py | 7 ++ rally/benchmark/context/users.py | 14 +-- rally/benchmark/engine.py | 20 ++-- rally/benchmark/runners/base.py | 65 ++++------- rally/benchmark/runners/continuous.py | 18 +-- rally/benchmark/runners/periodic.py | 8 +- rally/benchmark/runners/serial.py | 5 +- .../scenarios/authenticate/authenticate.py | 2 - rally/benchmark/scenarios/base.py | 3 +- rally/benchmark/scenarios/cinder/volumes.py | 10 +- rally/benchmark/scenarios/glance/images.py | 10 +- rally/benchmark/scenarios/keystone/basic.py | 12 +- rally/benchmark/scenarios/nova/servers.py | 26 ++--- rally/benchmark/scenarios/nova/utils.py | 13 +-- tests/benchmark/context/test_base.py | 105 +++++++++++++++--- tests/benchmark/context/test_cleaner.py | 28 +++-- tests/benchmark/context/test_keypair.py | 7 +- tests/benchmark/context/test_quotas.py | 16 +-- tests/benchmark/runners/test_base.py | 46 ++++++-- tests/benchmark/runners/test_continuous.py | 7 +- tests/benchmark/runners/test_periodic.py | 37 +++--- tests/benchmark/runners/test_serial.py | 6 +- tests/benchmark/scenarios/nova/test_utils.py | 2 +- tests/benchmark/scenarios/test_base.py | 16 +++ tests/benchmark/test_engine.py | 9 +- tests/doc/test_task_samples.py | 2 +- 30 files changed, 382 insertions(+), 241 deletions(-) diff --git a/rally/benchmark/context/base.py b/rally/benchmark/context/base.py index 380eda2754..209f56ea75 100644 --- a/rally/benchmark/context/base.py +++ b/rally/benchmark/context/base.py @@ -24,18 +24,20 @@ from rally import utils @six.add_metaclass(abc.ABCMeta) class Context(object): - """We will use this class in future as a factory for context classes. + """This class is a factory for context classes. - It will cover: - 1) Auto discovering - 2) Validation of input args - 3) Common logging + Every context class should be a subclass of this method and implement + 2 abstract methods: setup() and cleanup() - Actually the same functionality as - runners.base.ScenarioRunner and scenarios.base.Scenario + It covers: + 1) proper setting up of context config + 2) Auto discovering & get by name + 3) Validation by CONFIG_SCHEMA + 4) Order of context creation """ - __ctx_name__ = "base" + __ctx_order__ = 0 + __ctx_hidden__ = True CONFIG_SCHEMA = {} @@ -44,11 +46,11 @@ class Context(object): self.context = context self.task = context["task"] - @staticmethod - def validate(context): - for name, config in context.iteritems(): - ctx = Context.get_by_name(name) - jsonschema.validate(config, ctx.CONFIG_SCHEMA) + @classmethod + def validate(cls, config, non_hidden=False): + if non_hidden and cls.__ctx_hidden__: + raise exceptions.NoSuchContext(name=cls.__ctx_name__) + jsonschema.validate(config, cls.CONFIG_SCHEMA) @staticmethod def get_by_name(name): @@ -71,3 +73,45 @@ class Context(object): def __exit__(self, exc_type, exc_value, exc_traceback): self.cleanup() + + +class ContextManager(object): + """Creates context environment and runs method inside it.""" + + @staticmethod + def run(context, func, *args, **kwargs): + ctxlst = [Context.get_by_name(name) for name in context["config"]] + ctxlst = map(lambda ctx: ctx(context), + sorted(ctxlst, key=lambda x: x.__ctx_order__)) + + return ContextManager._magic(ctxlst, func, *args, **kwargs) + + @staticmethod + def validate(context, non_hidden=False): + for name, config in context.iteritems(): + Context.get_by_name(name).validate(config, non_hidden=non_hidden) + + @staticmethod + def _magic(ctxlst, func, *args, **kwargs): + """Some kind of contextlib.nested but with black jack & recursion. + + This method uses recursion to build nested "with" from list of context + objects. As it's actually a combination of dark and voodoo magic I + called it "_magic". Please don't repeat at home. + + :param ctxlst: list of instances of subclasses of Context + :param func: function that will be called inside this context + :param args: args that will be passed to function `func` + :param kwargs: kwargs that will be passed to function `func` + :returns: result of function call + """ + if not ctxlst: + return func(*args, **kwargs) + + with ctxlst[0]: + # TODO(boris-42): call of setup could be moved inside __enter__ + # but it should be in try-except, and in except + # we should call by hand __exit__ + ctxlst[0].setup() + tmp = ContextManager._magic(ctxlst[1:], func, *args, **kwargs) + return tmp diff --git a/rally/benchmark/context/cleaner.py b/rally/benchmark/context/cleaner.py index 2f697fdaa0..04aa1a34ce 100644 --- a/rally/benchmark/context/cleaner.py +++ b/rally/benchmark/context/cleaner.py @@ -17,7 +17,6 @@ import functools import sys from rally.benchmark.context import base -from rally.benchmark.scenarios import base as scenario_base from rally.benchmark import utils from rally.openstack.common.gettextutils import _ from rally.openstack.common import log as logging @@ -31,19 +30,24 @@ LOG = logging.getLogger(__name__) class ResourceCleaner(base.Context): """Context class for resource cleanup (both admin and non-admin).""" - __ctx_name__ = "cleaner" + __ctx_name__ = "cleanup" + __ctx_order__ = 200 + __ctx_hidden__ = True CONFIG_SCHEMA = { - "type": "object", - "$schema": "http://json-schema.org/draft-03/schema", - "properties": {}, - "additionalProperties": False + "type": "array", + "$schema": "http://json-schema.org/draft-04/schema", + "items": { + "type": "string", + "enum": ["nova", "glance", "cinder"] + }, + "uniqueItems": True } def __init__(self, context): super(ResourceCleaner, self).__init__(context) - self.admin = None - self.users = None + self.admin = [] + self.users = [] if "admin" in context and context["admin"]: self.admin = context["admin"]["endpoint"] if "users" in context and context["users"]: @@ -51,19 +55,6 @@ class ResourceCleaner(base.Context): @rutils.log_task_wrapper(LOG.info, _("Cleanup users resources.")) def _cleanup_users_resources(self): - def _init_services_to_cleanup(cleanup_methods): - scenario_name = self.context.get('scenario_name') - if scenario_name: - cls_name, method_name = scenario_name.split(".", 1) - scenario = scenario_base.Scenario.get_by_name(cls_name)() - scenario_method = getattr(scenario, method_name) - if hasattr(scenario_method, "cleanup_services"): - return getattr(scenario_method, "cleanup_services") - return cleanup_methods.keys() - - if not self.users: - return - for user in self.users: clients = osclients.Clients(user) cleanup_methods = { @@ -76,7 +67,7 @@ class ResourceCleaner(base.Context): clients.cinder()) } - for service in _init_services_to_cleanup(cleanup_methods): + for service in self.config: try: cleanup_methods[service]() except Exception as e: @@ -87,9 +78,6 @@ class ResourceCleaner(base.Context): @rutils.log_task_wrapper(LOG.info, _("Cleanup admin resources.")) def _cleanup_admin_resources(self): - if not self.admin: - return - try: admin = osclients.Clients(self.admin) utils.delete_keystone_resources(admin.keystone()) @@ -99,11 +87,13 @@ class ResourceCleaner(base.Context): LOG.warning(_('Unable to fully cleanup keystone service: %s') % (e.message)) + @rutils.log_task_wrapper(LOG.info, _("Enter context: `cleanup`")) def setup(self): pass + @rutils.log_task_wrapper(LOG.info, _("Exit context: `cleanup`")) def cleanup(self): - if self.users: + if self.users and self.config: self._cleanup_users_resources() if self.admin: self._cleanup_admin_resources() diff --git a/rally/benchmark/context/keypair.py b/rally/benchmark/context/keypair.py index 3442aeb405..ce2c65a3f5 100644 --- a/rally/benchmark/context/keypair.py +++ b/rally/benchmark/context/keypair.py @@ -16,8 +16,10 @@ import novaclient.exceptions from rally.benchmark.context import base +from rally.openstack.common.gettextutils import _ from rally.openstack.common import log as logging from rally import osclients +from rally import utils LOG = logging.getLogger(__name__) @@ -25,6 +27,9 @@ LOG = logging.getLogger(__name__) class Keypair(base.Context): __ctx_name__ = "keypair" + __ctx_order__ = 300 + __ctx_hidden__ = True + KEYPAIR_NAME = "rally_ssh_key" def _get_nova_client(self, endpoint): @@ -47,11 +52,13 @@ class Keypair(base.Context): return {"private": keypair.private_key, "public": keypair.public_key} + @utils.log_task_wrapper(LOG.info, _("Enter context: `keypair`")) def setup(self): for user in self.context["users"]: keypair = self._generate_keypair(user["endpoint"]) user["keypair"] = keypair + @utils.log_task_wrapper(LOG.info, _("Exit context: `keypair`")) def cleanup(self): for user in self.context["users"]: endpoint = user['endpoint'] diff --git a/rally/benchmark/context/quotas.py b/rally/benchmark/context/quotas.py index 7f0459e41d..cad95f1874 100644 --- a/rally/benchmark/context/quotas.py +++ b/rally/benchmark/context/quotas.py @@ -14,7 +14,13 @@ # under the License. from rally.benchmark.context import base +from rally.openstack.common.gettextutils import _ +from rally.openstack.common import log as logging from rally import osclients +from rally import utils + + +LOG = logging.getLogger(__name__) class NovaQuotas(object): @@ -124,6 +130,8 @@ class Quotas(base.Context): """Context class for updating benchmarks' tenants quotas.""" __ctx_name__ = "quotas" + __ctx_order__ = 210 + __ctx_hidden__ = False CONFIG_SCHEMA = { "type": "object", @@ -141,6 +149,7 @@ class Quotas(base.Context): self.nova_quotas = NovaQuotas(self.clients.nova()) self.cinder_quotas = CinderQuotas(self.clients.cinder()) + @utils.log_task_wrapper(LOG.info, _("Enter context: `quotas`")) def setup(self): for tenant in self.context["tenants"]: if "nova" in self.config and len(self.config["nova"]) > 0: @@ -151,6 +160,7 @@ class Quotas(base.Context): self.cinder_quotas.update(tenant["id"], **self.config["cinder"]) + @utils.log_task_wrapper(LOG.info, _("Exit context: `quotas`")) def cleanup(self): for tenant in self.context["tenants"]: # Always cleanup quotas before deleting a tenant diff --git a/rally/benchmark/context/secgroup.py b/rally/benchmark/context/secgroup.py index 3bb6bf2baf..cce1af3c14 100644 --- a/rally/benchmark/context/secgroup.py +++ b/rally/benchmark/context/secgroup.py @@ -14,8 +14,10 @@ # under the License. from rally.benchmark.context import base +from rally.openstack.common.gettextutils import _ from rally.openstack.common import log as logging from rally import osclients +from rally import utils LOG = logging.getLogger(__name__) @@ -77,11 +79,15 @@ def _prepare_open_secgroup(endpoint): class AllowSSH(base.Context): __ctx_name__ = "allow_ssh" + __ctx_order__ = 301 + __ctx_hidden__ = True def __init__(self, context): super(AllowSSH, self).__init__(context) + self.context["allow_ssh"] = SSH_GROUP_NAME self.secgroup = [] + @utils.log_task_wrapper(LOG.info, _("Exit context: `allow_ssh`")) def setup(self): used_tenants = [] for user in self.context['users']: @@ -92,6 +98,7 @@ class AllowSSH(base.Context): self.secgroup.append(secgroup) used_tenants.append(tenant) + @utils.log_task_wrapper(LOG.info, _("Exit context: `allow_ssh`")) def cleanup(self): for secgroup in self.secgroup: try: diff --git a/rally/benchmark/context/users.py b/rally/benchmark/context/users.py index 49597108fb..15a4eada9c 100644 --- a/rally/benchmark/context/users.py +++ b/rally/benchmark/context/users.py @@ -20,8 +20,10 @@ from rally.benchmark.context import base from rally.benchmark import utils from rally import consts from rally.objects import endpoint +from rally.openstack.common.gettextutils import _ from rally.openstack.common import log as logging from rally import osclients +from rally import utils as rutils LOG = logging.getLogger(__name__) @@ -43,6 +45,8 @@ class UserGenerator(base.Context): """Context class for generating temporary users/tenants for benchmarks.""" __ctx_name__ = "users" + __ctx_order__ = 100 + __ctx_hidden__ = False CONFIG_SCHEMA = { "type": "object", @@ -145,6 +149,7 @@ class UserGenerator(base.Context): "Exception: %(ex)s" % {"user_id": user["id"], "ex": ex}) + @rutils.log_task_wrapper(LOG.info, _("Enter context: `users`")) def setup(self): """Create tenants and users, using pool of threads.""" @@ -164,6 +169,7 @@ class UserGenerator(base.Context): self.context["tenants"].append(tenant) self.context["users"] += users + @rutils.log_task_wrapper(LOG.info, _("Exit context: `users`")) def cleanup(self): """Delete tenants and users, using pool of threads.""" @@ -182,11 +188,3 @@ class UserGenerator(base.Context): concurrent, self._delete_tenants, [(self.endpoint, tenants) for tenants in tenants_chunks]) - - # NOTE(amaretskiy): Consider that after cleanup() is complete, this has - # actually deleted (all or some of) users and tenants - # in openstack, but we *STILL HAVE* - # self.context["users"] and self.context["tenants"]. - # Should we ignore that, or just reset these lists - # after cleanup() is done, or actually synchronize - # for all successfully deleted objects? diff --git a/rally/benchmark/engine.py b/rally/benchmark/engine.py index 877a5e86da..9c217a8156 100644 --- a/rally/benchmark/engine.py +++ b/rally/benchmark/engine.py @@ -102,7 +102,8 @@ class BenchmarkEngine(object): for pos, kw in enumerate(values): try: base_runner.ScenarioRunner.validate(kw.get("runner", {})) - base_ctx.Context.validate(kw.get("context", {})) + base_ctx.ContextManager.validate(kw.get("context", {}), + non_hidden=True) except (exceptions.RallyException, jsonschema.ValidationError) as e: raise exceptions.InvalidBenchmarkConfig(name=scenario, @@ -150,6 +151,12 @@ class BenchmarkEngine(object): self.task.set_failed(log=log) raise exceptions.InvalidTaskException(message=str(e)) + def _get_runner(self, config): + runner = config.get("runner", {}) + runner.setdefault("type", "continuous") + return base_runner.ScenarioRunner.get_runner(self.task, self.endpoints, + runner) + @rutils.log_task_wrapper(LOG.info, _("Benchmarking.")) def run(self): """Runs the benchmarks according to the test configuration @@ -161,12 +168,11 @@ class BenchmarkEngine(object): self.task.update_status(consts.TaskStatus.RUNNING) results = {} for name in self.config: - for n, kwargs in enumerate(self.config[name]): - key = {'name': name, 'pos': n, 'kw': kwargs} - runner = kwargs.get("runner", {}).get("type", "continuous") - scenario_runner = base_runner.ScenarioRunner.get_runner( - self.task, self.endpoints, runner) - result = scenario_runner.run(name, kwargs) + for n, kw in enumerate(self.config[name]): + key = {'name': name, 'pos': n, 'kw': kw} + runner = self._get_runner(kw) + result = runner.run(name, kw.get("context", {}), + kw.get("args", {})) self.task.append_results(key, {"raw": result}) results[json.dumps(key)] = result self.task.update_status(consts.TaskStatus.FINISHED) diff --git a/rally/benchmark/runners/base.py b/rally/benchmark/runners/base.py index 25cb022f1b..90318bae87 100644 --- a/rally/benchmark/runners/base.py +++ b/rally/benchmark/runners/base.py @@ -19,11 +19,7 @@ import random import jsonschema from oslo.config import cfg -from rally.benchmark.context import cleaner as cleaner_ctx -from rally.benchmark.context import keypair as keypair_ctx -from rally.benchmark.context import quotas as quotas_ctx -from rally.benchmark.context import secgroup as secgroup_ctx -from rally.benchmark.context import users as users_ctx +from rally.benchmark.context import base as base_ctx from rally.benchmark.scenarios import base from rally.benchmark import utils from rally import exceptions @@ -145,13 +141,14 @@ class ScenarioRunner(object): CONFIG_SCHEMA = {} - def __init__(self, task, endpoints): + def __init__(self, task, endpoints, config): self.task = task self.endpoints = endpoints # NOTE(msdubov): Passing predefined user endpoints hasn't been # implemented yet, so the scenario runner always gets # a single admin endpoint here. self.admin_user = endpoints[0] + self.config = config @staticmethod def _get_cls(runner_type): @@ -161,9 +158,9 @@ class ScenarioRunner(object): raise exceptions.NoSuchRunner(type=runner_type) @staticmethod - def get_runner(task, endpoint, runner_type): + def get_runner(task, endpoint, config): """Returns instance of a scenario runner for execution type.""" - return ScenarioRunner._get_cls(runner_type)(task, endpoint) + return ScenarioRunner._get_cls(config["type"])(task, endpoint, config) @staticmethod def validate(config): @@ -172,7 +169,7 @@ class ScenarioRunner(object): jsonschema.validate(config, runner.CONFIG_SCHEMA) @abc.abstractmethod - def _run_scenario(self, cls, method_name, context, args, config): + def _run_scenario(self, cls, method_name, context, args): """Runs the specified benchmark scenario with given arguments. :param cls: The Scenario class where the scenario is implemented @@ -180,57 +177,33 @@ class ScenarioRunner(object): :param context: Benchmark context that contains users, admin & other information, that was created before benchmark started. :param args: Arguments to call the scenario method with - :param config: Configuration dictionary that contains strategy-specific - parameters like the number of times to run the scenario :returns: List of results fore each single scenario iteration, where each result is a dictionary """ - def _prepare_and_run_scenario(self, context, name, kwargs): + def run(self, name, context, args): cls_name, method_name = name.split(".", 1) cls = base.Scenario.get_by_name(cls_name) - args = kwargs.get('args', {}) - config = kwargs.get('runner', {}) + scenario_context = getattr(cls, method_name).context + # TODO(boris-42): We should keep default behavior for `users` context + # as a part of work on pre-created users this should be + # removed. + scenario_context.setdefault("users", {}) + # merge scenario context and task context configuration + scenario_context.update(context) - with secgroup_ctx.AllowSSH(context) as allow_ssh: - allow_ssh.setup() - with keypair_ctx.Keypair(context) as keypair: - keypair.setup() - LOG.debug("Context: %s" % context) - return self._run_scenario(cls, method_name, context, - args, config) - - def _run_as_admin(self, name, kwargs): - context = { + context_obj = { "task": self.task, "admin": {"endpoint": self.admin_user}, "scenario_name": name, - "config": kwargs.get("context", {}) + "config": scenario_context } - with users_ctx.UserGenerator(context) as generator: - generator.setup() - with quotas_ctx.Quotas(context) as quotas: - quotas.setup() - with cleaner_ctx.ResourceCleaner(context) as cleaner: - cleaner.setup() - return self._prepare_and_run_scenario(context, - name, kwargs) - - def _run_as_non_admin(self, name, kwargs): - # TODO(boris-42): It makes sense to use UserGenerator here as well - # take a look at comment in UserGenerator.__init__() - context = {"scenario_name": name} - with cleaner_ctx.ResourceCleaner(context): - return self._prepare_and_run_scenario(context, name, kwargs) - - def run(self, name, kwargs): - if self.admin_user: - results = self._run_as_admin(name, kwargs) - else: - results = self._run_as_non_admin(name, kwargs) + results = base_ctx.ContextManager.run(context_obj, self._run_scenario, + cls, method_name, context_obj, + args) if not isinstance(results, ScenarioRunnerResult): name = self.__execution_type__ diff --git a/rally/benchmark/runners/continuous.py b/rally/benchmark/runners/continuous.py index 37b1bea605..f8c1c94dfe 100644 --- a/rally/benchmark/runners/continuous.py +++ b/rally/benchmark/runners/continuous.py @@ -126,25 +126,25 @@ class ContinuousScenarioRunner(base.ScenarioRunner): return results - def _run_scenario(self, cls, method_name, context, args, config): + def _run_scenario(self, cls, method_name, context, args): - timeout = config.get("timeout", 600) - concurrent = config.get("active_users", 1) + timeout = self.config.get("timeout", 600) + concurrent = self.config.get("active_users", 1) # NOTE(msdubov): If not specified, perform single scenario run. - if "duration" not in config and "times" not in config: - config["times"] = 1 + if "duration" not in self.config and "times" not in self.config: + self.config["times"] = 1 # Continiously run a benchmark scenario the specified # amount of times. - if "times" in config: - times = config["times"] + if "times" in self.config: + times = self.config["times"] results = self._run_scenario_continuously_for_times( cls, method_name, context, args, times, concurrent, timeout) # Continiously run a scenario as many times as needed # to fill up the given period of time. - elif "duration" in config: - duration = config["duration"] + elif "duration" in self.config: + duration = self.config["duration"] results = self._run_scenario_continuously_for_duration( cls, method_name, context, args, duration, concurrent, timeout) diff --git a/rally/benchmark/runners/periodic.py b/rally/benchmark/runners/periodic.py index 6609df82f5..1c03d045d2 100644 --- a/rally/benchmark/runners/periodic.py +++ b/rally/benchmark/runners/periodic.py @@ -61,11 +61,11 @@ class PeriodicScenarioRunner(base.ScenarioRunner): "additionalProperties": False } - def _run_scenario(self, cls, method_name, context, args, config): + def _run_scenario(self, cls, method_name, context, args): - times = config["times"] - period = config["period"] - timeout = config.get("timeout", 600) + times = self.config["times"] + period = self.config["period"] + timeout = self.config.get("timeout", 600) async_results = [] diff --git a/rally/benchmark/runners/serial.py b/rally/benchmark/runners/serial.py index d4c0cc35e8..b11249919f 100644 --- a/rally/benchmark/runners/serial.py +++ b/rally/benchmark/runners/serial.py @@ -45,9 +45,8 @@ class SerialScenarioRunner(base.ScenarioRunner): "additionalProperties": True } - def _run_scenario(self, cls, method_name, context, args, config): - - times = config.get('times', 1) + def _run_scenario(self, cls, method_name, context, args): + times = self.config.get('times', 1) results = [] diff --git a/rally/benchmark/scenarios/authenticate/authenticate.py b/rally/benchmark/scenarios/authenticate/authenticate.py index 2d3d134738..c322cea25d 100644 --- a/rally/benchmark/scenarios/authenticate/authenticate.py +++ b/rally/benchmark/scenarios/authenticate/authenticate.py @@ -12,7 +12,6 @@ # License for the specific language governing permissions and limitations # under the License. -from rally.benchmark.context import cleaner as context_cleaner from rally.benchmark.scenarios import base @@ -22,6 +21,5 @@ class Authenticate(base.Scenario): """ @base.scenario() - @context_cleaner.cleanup([]) def keystone(self, **kwargs): self.clients("keystone") diff --git a/rally/benchmark/scenarios/base.py b/rally/benchmark/scenarios/base.py index 68cc75fc72..c0e905462d 100644 --- a/rally/benchmark/scenarios/base.py +++ b/rally/benchmark/scenarios/base.py @@ -22,7 +22,7 @@ from rally import exceptions from rally import utils -def scenario(admin_only=False): +def scenario(admin_only=False, context=None): """This method is used as decorator for the methods of benchmark scenarios and it adds following extra fields to the methods. 'is_scenario' is set to True @@ -31,6 +31,7 @@ def scenario(admin_only=False): def wrapper(func): func.is_scenario = True func.admin_only = admin_only + func.context = context or {} return func return wrapper diff --git a/rally/benchmark/scenarios/cinder/volumes.py b/rally/benchmark/scenarios/cinder/volumes.py index 3d4cc0f5c3..c45f2b7a5d 100644 --- a/rally/benchmark/scenarios/cinder/volumes.py +++ b/rally/benchmark/scenarios/cinder/volumes.py @@ -13,15 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. -from rally.benchmark.context import cleaner as context_cleaner from rally.benchmark.scenarios import base from rally.benchmark.scenarios.cinder import utils class CinderVolumes(utils.CinderScenario): - @base.scenario() - @context_cleaner.cleanup(['cinder']) + @base.scenario(context={"cleanup": ["cinder"]}) def create_and_list_volume(self, size, detailed=True, **kwargs): """Tests creating a volume and listing volumes. @@ -38,8 +36,7 @@ class CinderVolumes(utils.CinderScenario): self._create_volume(size, **kwargs) self._list_volumes(detailed) - @base.scenario() - @context_cleaner.cleanup(['cinder']) + @base.scenario(context={"cleanup": ["cinder"]}) def create_and_delete_volume(self, size, min_sleep=0, max_sleep=0, **kwargs): """Tests creating and then deleting a volume. @@ -51,8 +48,7 @@ class CinderVolumes(utils.CinderScenario): self.sleep_between(min_sleep, max_sleep) self._delete_volume(volume) - @base.scenario() - @context_cleaner.cleanup(['cinder']) + @base.scenario(context={"cleanup": ["cinder"]}) def create_volume(self, size, **kwargs): """Test creating volumes perfromance. diff --git a/rally/benchmark/scenarios/glance/images.py b/rally/benchmark/scenarios/glance/images.py index 0906b65db9..9b2e158341 100644 --- a/rally/benchmark/scenarios/glance/images.py +++ b/rally/benchmark/scenarios/glance/images.py @@ -13,7 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -from rally.benchmark.context import cleaner as context_cleaner from rally.benchmark.scenarios import base from rally.benchmark.scenarios.glance import utils from rally.benchmark.scenarios.nova import utils as nova_utils @@ -22,8 +21,7 @@ from rally.benchmark import validation class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario): - @base.scenario() - @context_cleaner.cleanup(['glance']) + @base.scenario(context={"cleanup": ["glance"]}) def create_and_list_image(self, container_format, image_location, disk_format, **kwargs): """Test adding an image and then listing all images. @@ -46,8 +44,7 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario): **kwargs) self._list_images() - @base.scenario() - @context_cleaner.cleanup(['glance']) + @base.scenario(context={"cleanup": ["glance"]}) def create_and_delete_image(self, container_format, image_location, disk_format, **kwargs): """Test adds and then deletes image.""" @@ -59,9 +56,8 @@ class GlanceImages(utils.GlanceScenario, nova_utils.NovaScenario): **kwargs) self._delete_image(image) - @context_cleaner.cleanup(['glance', 'nova']) @validation.add_validator(validation.flavor_exists("flavor_id")) - @base.scenario() + @base.scenario(context={"cleanup": ["glance", "nova"]}) def create_image_and_boot_instances(self, container_format, image_location, disk_format, flavor_id, number_instances, diff --git a/rally/benchmark/scenarios/keystone/basic.py b/rally/benchmark/scenarios/keystone/basic.py index a5e49b9e4a..d3599c0ab1 100644 --- a/rally/benchmark/scenarios/keystone/basic.py +++ b/rally/benchmark/scenarios/keystone/basic.py @@ -13,7 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -from rally.benchmark.context import cleaner as context_cleaner from rally.benchmark.scenarios import base from rally.benchmark.scenarios.keystone import utils as kutils from rally.benchmark import validation as valid @@ -21,24 +20,21 @@ from rally.benchmark import validation as valid class KeystoneBasic(kutils.KeystoneScenario): - @base.scenario(admin_only=True) + @base.scenario(admin_only=True, context={"cleanup": []}) def create_user(self, name_length=10, **kwargs): self._user_create(name_length=name_length, **kwargs) - @base.scenario(admin_only=True) - @context_cleaner.cleanup([]) + @base.scenario(admin_only=True, context={"cleanup": []}) def create_delete_user(self, name_length=10, **kwargs): user = self._user_create(name_length=name_length, **kwargs) self._resource_delete(user) - @base.scenario(admin_only=True) - @context_cleaner.cleanup([]) + @base.scenario(admin_only=True, context={"cleanup": []}) def create_tenant(self, name_length=10, **kwargs): self._tenant_create(name_length=name_length, **kwargs) - @base.scenario(admin_only=True) + @base.scenario(admin_only=True, context={"cleanup": []}) @valid.add_validator(valid.required_parameters(['users_per_tenant'])) - @context_cleaner.cleanup([]) def create_tenant_with_users(self, users_per_tenant, name_length=10, **kwargs): tenant = self._tenant_create(name_length=name_length, **kwargs) diff --git a/rally/benchmark/scenarios/nova/servers.py b/rally/benchmark/scenarios/nova/servers.py index 1f1ef4372f..d98a7d0a48 100644 --- a/rally/benchmark/scenarios/nova/servers.py +++ b/rally/benchmark/scenarios/nova/servers.py @@ -17,7 +17,6 @@ import json import jsonschema import random -from rally.benchmark.context import cleaner as context_cleaner from rally.benchmark.scenarios import base from rally.benchmark.scenarios.cinder import utils as cinder_utils from rally.benchmark.scenarios.nova import utils @@ -38,9 +37,8 @@ class NovaServers(utils.NovaScenario, def __init__(self, *args, **kwargs): super(NovaServers, self).__init__(*args, **kwargs) - @context_cleaner.cleanup(['nova']) @valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id")) - @base.scenario() + @base.scenario(context={"cleanup": ["nova"]}) def boot_and_list_server(self, image_id, flavor_id, detailed=True, **kwargs): """Tests booting an image and then listing servers. @@ -60,9 +58,8 @@ class NovaServers(utils.NovaScenario, self._boot_server(server_name, image_id, flavor_id, **kwargs) self._list_servers(detailed) - @context_cleaner.cleanup(['nova']) @valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id")) - @base.scenario() + @base.scenario(context={"cleanup": ["nova"]}) def boot_and_delete_server(self, image_id, flavor_id, min_sleep=0, max_sleep=0, **kwargs): """Tests booting and then deleting an image.""" @@ -72,9 +69,8 @@ class NovaServers(utils.NovaScenario, self.sleep_between(min_sleep, max_sleep) self._delete_server(server) - @context_cleaner.cleanup(['nova', 'cinder']) @valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id")) - @base.scenario() + @base.scenario(context={"cleanup": ["nova", "cinder"]}) def boot_server_from_volume_and_delete(self, image_id, flavor_id, volume_size, min_sleep=0, max_sleep=0, **kwargs): @@ -89,9 +85,9 @@ class NovaServers(utils.NovaScenario, self.sleep_between(min_sleep, max_sleep) self._delete_server(server) - @context_cleaner.cleanup(['nova']) @valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id")) - @base.scenario() + @base.scenario(context={"cleanup": ["nova"], + "keypair": {}, "allow_ssh": {}}) def boot_runcommand_delete_server(self, image_id, flavor_id, script, interpreter, network='private', username='ubuntu', ip_version=4, @@ -147,9 +143,8 @@ class NovaServers(utils.NovaScenario, stdout=out, stderr=err)) return {"data": out, "errors": err} - @context_cleaner.cleanup(['nova']) @valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id")) - @base.scenario() + @base.scenario(context={"cleanup": ["nova"]}) def boot_and_bounce_server(self, image_id, flavor_id, **kwargs): """Tests booting a server then performing stop/start or hard/soft reboot a number of times. @@ -168,9 +163,8 @@ class NovaServers(utils.NovaScenario, action() self._delete_server(server) - @context_cleaner.cleanup(['nova', 'glance']) @valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id")) - @base.scenario() + @base.scenario(context={"cleanup": ["nova", "glance"]}) def snapshot_server(self, image_id, flavor_id, **kwargs): """Tests Nova instance snapshotting.""" server_name = self._generate_random_name(16) @@ -183,9 +177,8 @@ class NovaServers(utils.NovaScenario, self._delete_server(server) self._delete_image(image) - @context_cleaner.cleanup(['nova']) @valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id")) - @base.scenario() + @base.scenario(context={"cleanup": ["nova"]}) def boot_server(self, image_id, flavor_id, **kwargs): """Test VM boot - assumed clean-up is done elsewhere.""" server_name = self._generate_random_name(16) @@ -196,9 +189,8 @@ class NovaServers(utils.NovaScenario, kwargs['nics'] = [{'net-id': random_nic.id}] self._boot_server(server_name, image_id, flavor_id, **kwargs) - @context_cleaner.cleanup(['nova', 'cinder']) @valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id")) - @base.scenario() + @base.scenario(context={"cleanup": ["nova", "cinder"]}) def boot_server_from_volume(self, image_id, flavor_id, volume_size, **kwargs): """Test VM boot from volume - assumed clean-up is done elsewhere.""" diff --git a/rally/benchmark/scenarios/nova/utils.py b/rally/benchmark/scenarios/nova/utils.py index 641f95a065..be29d119c6 100644 --- a/rally/benchmark/scenarios/nova/utils.py +++ b/rally/benchmark/scenarios/nova/utils.py @@ -18,7 +18,6 @@ import random import string import time -from rally.benchmark.context import secgroup from rally.benchmark.scenarios import base from rally.benchmark.scenarios import utils as scenario_utils from rally.benchmark import utils as bench_utils @@ -86,12 +85,12 @@ class NovaScenario(base.Scenario): :returns: Created server object """ - - if 'security_groups' not in kwargs: - kwargs['security_groups'] = [secgroup.SSH_GROUP_NAME] - else: - if secgroup.SSH_GROUP_NAME not in kwargs['security_groups']: - kwargs['security_groups'].append(secgroup.SSH_GROUP_NAME) + allow_ssh_secgroup = self.context().get("allow_ssh") + if allow_ssh_secgroup: + if 'security_groups' not in kwargs: + kwargs['security_groups'] = [allow_ssh_secgroup] + elif allow_ssh_secgroup not in kwargs['security_groups']: + kwargs['security_groups'].append(allow_ssh_secgroup) server = self.clients("nova").servers.create(server_name, image_id, flavor_id, **kwargs) diff --git a/tests/benchmark/context/test_base.py b/tests/benchmark/context/test_base.py index 17315dbc75..d21031eb4e 100644 --- a/tests/benchmark/context/test_base.py +++ b/tests/benchmark/context/test_base.py @@ -49,24 +49,11 @@ class BaseContextTestCase(test.TestCase): self.assertEqual(ctx.context, context) def test_validate__context(self): - context = { - "fake": {"test": 2} - } - base.Context.validate(context) + fakes.FakeContext.validate({"test": 2}) def test_validate__wrong_context(self): - context = { - "fake": {"nonexisting": 2} - } self.assertRaises(jsonschema.ValidationError, - base.Context.validate, context) - - def test_validate__non_existing_context(self): - config = { - "nonexisting": {"nonexisting": 2} - } - self.assertRaises(exceptions.NoSuchContext, - base.Context.validate, config) + fakes.FakeContext.validate, {"nonexisting": 2}) @mock.patch("rally.benchmark.context.base.utils.itersubclasses") def test_get_by_name(self, mock_itersubclasses): @@ -85,6 +72,10 @@ class BaseContextTestCase(test.TestCase): self.assertRaises(exceptions.NoSuchContext, base.Context.get_by_name, "nonexisting") + def test_get_by_name_hidder(self): + self.assertRaises(exceptions.NoSuchContext, + base.Context.validate, {}, non_hidden=True) + def test_setup_is_abstract(self): class A(base.Context): @@ -114,3 +105,87 @@ class BaseContextTestCase(test.TestCase): self.assertEqual(ctx, entered_ctx) ctx.cleanup.assert_called_once_with() + + +class ContextManagerTestCase(test.TestCase): + + @mock.patch("rally.benchmark.context.base.ContextManager._magic") + @mock.patch("rally.benchmark.context.base.Context.get_by_name") + def test_run(self, mock_get, mock_magic): + context = { + "config": { + "a": mock.MagicMock(), + "b": mock.MagicMock() + } + } + + cc = mock.MagicMock() + cc.__ctx_order__ = 10 + mock_get.return_value = cc + + mock_magic.return_value = 5 + + result = base.ContextManager.run(context, lambda x, y: x + y, 1, 2) + self.assertEqual(result, 5) + + mock_get.assert_has_calls([ + mock.call("a"), + mock.call("b"), + mock.call()(context), + mock.call()(context) + ]) + + @mock.patch("rally.benchmark.context.base.Context.get_by_name") + def test_validate(self, mock_get): + config = { + "ctx1": mock.MagicMock(), + "ctx2": mock.MagicMock() + } + + base.ContextManager.validate(config) + mock_get.assert_has_calls([ + mock.call("ctx1"), + mock.call().validate(config["ctx1"], non_hidden=False), + mock.call("ctx2"), + mock.call().validate(config["ctx2"], non_hidden=False) + ]) + + @mock.patch("rally.benchmark.context.base.Context.get_by_name") + def test_validate_non_hidden(self, mock_get): + config = { + "ctx1": mock.MagicMock(), + "ctx2": mock.MagicMock() + } + + base.ContextManager.validate(config, non_hidden=True) + mock_get.assert_has_calls([ + mock.call("ctx1"), + mock.call().validate(config["ctx1"], non_hidden=True), + mock.call("ctx2"), + mock.call().validate(config["ctx2"], non_hidden=True) + ]) + + def test_validate__non_existing_context(self): + config = { + "nonexisting": {"nonexisting": 2} + } + self.assertRaises(exceptions.NoSuchContext, + base.ContextManager.validate, config) + + def test__magic(self): + func = lambda x, y: x + y + + result = base.ContextManager._magic([], func, 2, 3) + self.assertEqual(result, 5) + + def test__magic_with_ctx(self): + ctx = [mock.MagicMock(), mock.MagicMock()] + func = lambda x, y: x + y + + result = base.ContextManager._magic(ctx, func, 2, 3) + self.assertEqual(result, 5) + + expected = [mock.call.__enter__(), mock.call.setup(), + mock.call.__exit__(None, None, None)] + for c in ctx: + ctx[0].assert_has_calls(expected) diff --git a/tests/benchmark/context/test_cleaner.py b/tests/benchmark/context/test_cleaner.py index e6a98b14ec..07230c06f3 100644 --- a/tests/benchmark/context/test_cleaner.py +++ b/tests/benchmark/context/test_cleaner.py @@ -32,7 +32,6 @@ class ResourceCleanerTestCase(test.TestCase): "admin": None, "users": [], "tenants": [], - "scenario_name": "NovaServers.boot_server_from_volume_and_delete" } resource_cleaner = cleaner_ctx.ResourceCleaner(context) with resource_cleaner: @@ -40,6 +39,7 @@ class ResourceCleanerTestCase(test.TestCase): def test_with_statement(self): fake_user_ctx = fakes.FakeUserContext({}).context + fake_user_ctx["config"] = {"cleanup": ["nova"]} res_cleaner = cleaner_ctx.ResourceCleaner(fake_user_ctx) res_cleaner._cleanup_users_resources = mock.MagicMock() @@ -56,7 +56,7 @@ class ResourceCleanerTestCase(test.TestCase): def test_cleaner_admin(self, mock_del_keystone, mock_clients): context = { "task": mock.MagicMock(), - "scenario_name": 'NovaServers.boot_server_from_volume_and_delete', + "config": {"cleanup": ["cinder", "nova"]}, "admin": {"endpoint": mock.MagicMock()}, } res_cleaner = cleaner_ctx.ResourceCleaner(context) @@ -70,21 +70,18 @@ class ResourceCleanerTestCase(test.TestCase): mock_clients.return_value.keystone.assert_called_once_with() mock_del_keystone.assert_called_once_with('keystone') - @mock.patch("rally.benchmark.scenarios.nova.servers.NovaServers." - "boot_server_from_volume_and_delete") @mock.patch("%s.osclients.Clients" % BASE) @mock.patch("%s.utils.delete_nova_resources" % BASE) @mock.patch("%s.utils.delete_glance_resources" % BASE) @mock.patch("%s.utils.delete_cinder_resources" % BASE) def test_cleaner_users_all_services(self, mock_del_cinder, mock_del_glance, mock_del_nova, - mock_clients, mock_scenario_method): - del mock_scenario_method.cleanup_services + mock_clients): context = { "task": mock.MagicMock(), "users": [{"endpoint": mock.MagicMock()}, {"endpoint": mock.MagicMock()}], - "scenario_name": "NovaServers.boot_server_from_volume_and_delete", + "config": {"cleanup": ["cinder", "nova", "glance"]}, "tenants": [mock.MagicMock()] } res_cleaner = cleaner_ctx.ResourceCleaner(context) @@ -100,18 +97,31 @@ class ResourceCleanerTestCase(test.TestCase): self.assertEqual(mock_del_glance.call_count, 2) self.assertEqual(mock_del_cinder.call_count, 2) + @mock.patch("%s.ResourceCleaner._cleanup_users_resources" % BASE) + def test_cleaner_users_default_behavior(self, mock_cleanup): + context = { + "task": mock.MagicMock(), + "users": [{"endpoint": mock.MagicMock()}, + {"endpoint": mock.MagicMock()}], + } + res_cleaner = cleaner_ctx.ResourceCleaner(context) + + with res_cleaner: + res_cleaner.setup() + + self.assertEqual(mock_cleanup.call_count, 0) + @mock.patch("%s.osclients.Clients" % BASE) @mock.patch("%s.utils.delete_nova_resources" % BASE) @mock.patch("%s.utils.delete_glance_resources" % BASE) @mock.patch("%s.utils.delete_cinder_resources" % BASE) def test_cleaner_users_by_service(self, mock_del_cinder, mock_del_glance, mock_del_nova, mock_clients): - context = { "task": mock.MagicMock(), "users": [{"endpoint": mock.MagicMock()}, {"endpoint": mock.MagicMock()}], - "scenario_name": 'NovaServers.boot_server_from_volume_and_delete', + "config": {"cleanup": ["cinder", "nova"]}, "tenants": [mock.MagicMock()] } res_cleaner = cleaner_ctx.ResourceCleaner(context) diff --git a/tests/benchmark/context/test_keypair.py b/tests/benchmark/context/test_keypair.py index 0a45b79985..b207ecd94c 100644 --- a/tests/benchmark/context/test_keypair.py +++ b/tests/benchmark/context/test_keypair.py @@ -26,15 +26,16 @@ class KeyPairContextTestCase(test.TestCase): def setUp(self): super(KeyPairContextTestCase, self).setUp() self.users = 2 + task = mock.MagicMock() self.ctx_with_keys = { "users": [ {"keypair": "key", "endpoint": "endpoint"}, ] * self.users, - "task": {} + "task": task } self.ctx_without_keys = { - "users": [{'endpoint': 'endpoint'}] * self.users, - "task": {} + "users": [{'endpoint': 'endpoint'}] * self.users, + "task": task } @mock.patch("%s.keypair.Keypair._generate_keypair" % CTX) diff --git a/tests/benchmark/context/test_quotas.py b/tests/benchmark/context/test_quotas.py index 0b20f40f0c..80e54dc538 100644 --- a/tests/benchmark/context/test_quotas.py +++ b/tests/benchmark/context/test_quotas.py @@ -97,7 +97,7 @@ class QuotasTestCase(test.TestCase): {"endpoint": mock.MagicMock(), "id": mock.MagicMock()} ], "admin": {"endpoint": mock.MagicMock()}, - "task": {} + "task": mock.MagicMock() } def test_quotas_schemas(self): @@ -128,7 +128,7 @@ class QuotasTestCase(test.TestCase): # Test invalid values ctx["config"]["quotas"][service][key] = self.unlimited - 1 try: - quotas.Quotas.validate(ctx["config"]) + quotas.Quotas.validate(ctx["config"]["quotas"]) except jsonschema.ValidationError: pass else: @@ -137,7 +137,7 @@ class QuotasTestCase(test.TestCase): ctx["config"]["quotas"][service][key] = 2.5 try: - quotas.Quotas.validate(ctx["config"]) + quotas.Quotas.validate(ctx["config"]["quotas"]) except jsonschema.ValidationError: pass else: @@ -146,7 +146,7 @@ class QuotasTestCase(test.TestCase): ctx["config"]["quotas"][service][key] = "-1" try: - quotas.Quotas.validate(ctx["config"]) + quotas.Quotas.validate(ctx["config"]["quotas"]) except jsonschema.ValidationError: pass else: @@ -157,20 +157,20 @@ class QuotasTestCase(test.TestCase): ctx["config"]["quotas"][service][key] = \ random.randint(0, 1000000) try: - quotas.Quotas.validate(ctx["config"]) + quotas.Quotas.validate(ctx["config"]["quotas"]) except jsonschema.ValidationError: self.fail("Positive integers are valid quota values") ctx["config"]["quotas"][service][key] = self.unlimited try: - quotas.Quotas.validate(ctx["config"]) + quotas.Quotas.validate(ctx["config"]["quotas"]) except jsonschema.ValidationError: self.fail("%d is a valid quota value" % self.unlimited) # Test additional keys are refused ctx["config"]["quotas"][service]["additional"] = self.unlimited try: - quotas.Quotas.validate(ctx["config"]) + quotas.Quotas.validate(ctx["config"]["quotas"]) except jsonschema.ValidationError: pass else: @@ -180,7 +180,7 @@ class QuotasTestCase(test.TestCase): # Test valid keys are optional ctx["config"]["quotas"][service] = {} try: - quotas.Quotas.validate(ctx["config"]) + quotas.Quotas.validate(ctx["config"]["quotas"]) except jsonschema.ValidationError: self.fail("Valid quota keys are optional") diff --git a/tests/benchmark/runners/test_base.py b/tests/benchmark/runners/test_base.py index 5b00bfe6b5..17654bff09 100644 --- a/tests/benchmark/runners/test_base.py +++ b/tests/benchmark/runners/test_base.py @@ -18,6 +18,7 @@ import mock from rally.benchmark.runners import base from rally.benchmark.runners import continuous +from rally.benchmark.scenarios import base as base_scenario from rally import consts from rally import exceptions from tests import fakes @@ -180,17 +181,19 @@ class ScenarioRunnerTestCase(test.TestCase): task = mock.MagicMock() endpoints = [mock.MagicMock(), mock.MagicMock()] - runner = base.ScenarioRunner.get_runner(task, endpoints, "new_runner") + config = {"type": "new_runner", "a": 123} + runner = base.ScenarioRunner.get_runner(task, endpoints, config) self.assertEqual(runner.task, task) self.assertEqual(runner.endpoints, endpoints) self.assertEqual(runner.admin_user, endpoints[0]) + self.assertEqual(runner.config, config) self.assertIsInstance(runner, NewRunner) def test_get_runner_no_such(self): self.assertRaises(exceptions.NoSuchRunner, base.ScenarioRunner.get_runner, - None, None, "NoSuchRunner") + None, None, {"type": "NoSuchRunner"}) @mock.patch("rally.benchmark.runners.base.jsonschema.validate") def test_validate_default_runner(self, mock_validate): @@ -200,10 +203,39 @@ class ScenarioRunnerTestCase(test.TestCase): config, continuous.ContinuousScenarioRunner.CONFIG_SCHEMA) - @mock.patch("rally.benchmark.runners.base.ScenarioRunner._run_as_admin") - def test_run_scenario_runner_results_exception(self, mock_run_method): + @mock.patch("rally.benchmark.runners.base.base_ctx.ContextManager") + def test_run(self, mock_ctx_manager): runner = continuous.ContinuousScenarioRunner(mock.MagicMock(), - self.fake_endpoints) + self.fake_endpoints, + mock.MagicMock()) + mock_ctx_manager.run.return_value = base.ScenarioRunnerResult([]) + scenario_name = "NovaServers.boot_server_from_volume_and_delete" + result = runner.run(scenario_name, {"some_ctx": 2}, [1, 2, 3]) + + self.assertEqual(result, mock_ctx_manager.run.return_value) + + cls_name, method_name = scenario_name.split(".", 1) + cls = base_scenario.Scenario.get_by_name(cls_name) + + context_obj = { + "task": runner.task, + "admin": {"endpoint": runner.admin_user}, + "scenario_name": scenario_name, + "config": { + "cleanup": ["nova", "cinder"], "some_ctx": 2, "users": {} + } + } + + expected = [context_obj, runner._run_scenario, cls, method_name, + context_obj, [1, 2, 3]] + mock_ctx_manager.run.assert_called_once_with(*expected) + + @mock.patch("rally.benchmark.runners.base.base_ctx.ContextManager") + def test_run__scenario_runner_results_exception(self, mock_ctx_manager): + runner = continuous.ContinuousScenarioRunner(mock.MagicMock(), + self.fake_endpoints, + mock.MagicMock()) self.assertRaises(exceptions.InvalidRunnerResult, - runner.run, "NovaServers.boot_server_from_volume_" - "and_delete", mock.MagicMock()) + runner.run, + "NovaServers.boot_server_from_volume_and_delete", + mock.MagicMock(), {}) diff --git a/tests/benchmark/runners/test_continuous.py b/tests/benchmark/runners/test_continuous.py index d8e54f96ad..856ab049b8 100644 --- a/tests/benchmark/runners/test_continuous.py +++ b/tests/benchmark/runners/test_continuous.py @@ -41,8 +41,9 @@ class ContinuousScenarioRunnerTestCase(test.TestCase): def test_run_scenario_continuously_for_times(self): context = fakes.FakeUserContext({"task": None}).context + runner = continuous.ContinuousScenarioRunner( - None, [context["admin"]["endpoint"]]) + None, [context["admin"]["endpoint"]], {}) times = 4 concurrent = 2 timeout = 2 @@ -56,7 +57,7 @@ class ContinuousScenarioRunnerTestCase(test.TestCase): def test_run_scenario_continuously_for_times_exception(self): context = fakes.FakeUserContext({"task": None}).context runner = continuous.ContinuousScenarioRunner( - None, [context["admin"]["endpoint"]]) + None, [context["admin"]["endpoint"]], {}) times = 4 concurrent = 2 timeout = 2 @@ -72,7 +73,7 @@ class ContinuousScenarioRunnerTestCase(test.TestCase): self.skipTest("This test produce a lot of races so we should fix it " "before running inside in gates") runner = continuous.ContinuousScenarioRunner(mock.MagicMock(), - [mock.MagicMock()]) + [mock.MagicMock()], {}) duration = 0 active_users = 4 timeout = 5 diff --git a/tests/benchmark/runners/test_periodic.py b/tests/benchmark/runners/test_periodic.py index c2b9a0bc1d..0ceb42d337 100644 --- a/tests/benchmark/runners/test_periodic.py +++ b/tests/benchmark/runners/test_periodic.py @@ -48,29 +48,24 @@ class PeriodicScenarioRunnerTestCase(test.TestCase): def test_run_scenario(self): context = fakes.FakeUserContext({}).context + config = {"times": 3, "period": 0, "timeout": 5} runner = periodic.PeriodicScenarioRunner( - None, [context["admin"]["endpoint"]]) - times = 3 - period = 0 + None, [context["admin"]["endpoint"]], config) - result = runner._run_scenario(fakes.FakeScenario, "do_it", context, {}, - {"times": times, "period": period, - "timeout": 5}) - self.assertEqual(len(result), times) + result = runner._run_scenario(fakes.FakeScenario, "do_it", context, {}) + self.assertEqual(len(result), config["times"]) self.assertIsNotNone(base.ScenarioRunnerResult(result)) def test_run_scenario_exception(self): context = fakes.FakeUserContext({}).context + + config = {"times": 4, "period": 0} runner = periodic.PeriodicScenarioRunner( - None, [context["admin"]["endpoint"]]) - times = 4 - period = 0 + None, [context["admin"]["endpoint"]], config) result = runner._run_scenario(fakes.FakeScenario, - "something_went_wrong", context, {}, - {"times": times, "period": period, - "timeout": 5}) - self.assertEqual(len(result), times) + "something_went_wrong", context, {}) + self.assertEqual(len(result), config["times"]) self.assertIsNotNone(base.ScenarioRunnerResult(result)) @mock.patch("rally.benchmark.runners.periodic.base.ScenarioRunnerResult") @@ -79,19 +74,17 @@ class PeriodicScenarioRunnerTestCase(test.TestCase): def test_run_scenario_internal_logic(self, mock_time, mock_pool, mock_result): context = fakes.FakeUserContext({}).context + config = {"times": 4, "period": 0, "timeout": 5} runner = periodic.PeriodicScenarioRunner( - None, [context["admin"]["endpoint"]]) - times = 4 - period = 0 + None, [context["admin"]["endpoint"]], config) mock_pool_inst = mock.MagicMock() mock_pool.ThreadPool.return_value = mock_pool_inst - runner._run_scenario(fakes.FakeScenario, "do_it", context, {}, - {"times": times, "period": period, "timeout": 5}) + runner._run_scenario(fakes.FakeScenario, "do_it", context, {}) exptected_pool_inst_call = [] - for i in range(times): + for i in range(config["times"]): args = ( base._run_scenario_once, ((i, fakes.FakeScenario, "do_it", @@ -99,7 +92,7 @@ class PeriodicScenarioRunnerTestCase(test.TestCase): ) exptected_pool_inst_call.append(mock.call.apply_async(*args)) - for i in range(times): + for i in range(config["times"]): call = mock.call.apply_async().get(timeout=5) exptected_pool_inst_call.append(call) @@ -117,5 +110,5 @@ class PeriodicScenarioRunnerTestCase(test.TestCase): runner = base.ScenarioRunner.get_runner(mock.MagicMock(), self.fake_endpoints, - "periodic") + {"type": "periodic"}) self.assertTrue(runner is not None) diff --git a/tests/benchmark/runners/test_serial.py b/tests/benchmark/runners/test_serial.py index d31a418ab6..c73b13395d 100644 --- a/tests/benchmark/runners/test_serial.py +++ b/tests/benchmark/runners/test_serial.py @@ -40,9 +40,9 @@ class SerialScenarioRunnerTestCase(test.TestCase): expected_results = [result for i in range(times)] runner = serial.SerialScenarioRunner(mock.MagicMock(), - self.fake_endpoints) + self.fake_endpoints, + {"times": times}) results = runner._run_scenario(fakes.FakeScenario, "do_it", - fakes.FakeUserContext({}).context, - {}, {"type": "serial", "times": times}) + fakes.FakeUserContext({}).context, {}) self.assertEqual(mock_run_once.call_count, times) self.assertEqual(results, expected_results) diff --git a/tests/benchmark/scenarios/nova/test_utils.py b/tests/benchmark/scenarios/nova/test_utils.py index fc60c3e2a9..ceeb37d412 100644 --- a/tests/benchmark/scenarios/nova/test_utils.py +++ b/tests/benchmark/scenarios/nova/test_utils.py @@ -80,7 +80,7 @@ class NovaScenarioTestCase(test.TestCase): @mock.patch(NOVA_UTILS + '.NovaScenario.clients') def test__boot_server(self, mock_clients): mock_clients("nova").servers.create.return_value = self.server - nova_scenario = utils.NovaScenario() + nova_scenario = utils.NovaScenario(context={}) return_server = nova_scenario._boot_server('server_name', 'image_id', 'flavor_id') self.wait_for.mock.assert_called_once_with( diff --git a/tests/benchmark/scenarios/test_base.py b/tests/benchmark/scenarios/test_base.py index 5ac427e91d..b5a0759cc9 100644 --- a/tests/benchmark/scenarios/test_base.py +++ b/tests/benchmark/scenarios/test_base.py @@ -14,7 +14,9 @@ # under the License. import mock +import traceback +from rally.benchmark.context import base as base_ctx from rally.benchmark.scenarios import base from rally.benchmark import validation from rally import consts @@ -173,3 +175,17 @@ class ScenarioTestCase(test.TestCase): scenario = base.Scenario(admin_clients=clients) self.assertEqual(clients.nova(), scenario.admin_clients("nova")) self.assertEqual(clients.glance(), scenario.admin_clients("glance")) + + def test_scenario_context_are_valid(self): + scenarios = base.Scenario.list_benchmark_scenarios() + + for scenario in scenarios: + cls_name, method_name = scenario.split(".", 1) + cls = base.Scenario.get_by_name(cls_name) + context = getattr(cls, method_name).context + try: + base_ctx.ContextManager.validate(context) + except Exception: + print(traceback.format_exc()) + self.assertTrue(False, + "Scenario `%s` has wrong context" % scenario) diff --git a/tests/benchmark/test_engine.py b/tests/benchmark/test_engine.py index 934212d297..e87b811520 100644 --- a/tests/benchmark/test_engine.py +++ b/tests/benchmark/test_engine.py @@ -121,16 +121,17 @@ class BenchmarkEngineTestCase(test.TestCase): eng._validate_config_scenarios_name, config) @mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner.validate") - @mock.patch("rally.benchmark.engine.base_ctx.Context.validate") + @mock.patch("rally.benchmark.engine.base_ctx.ContextManager.validate") def test__validate_config_syntax(self, mock_context, mock_runner): config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]} eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock()) eng._validate_config_syntax(config) mock_runner.assert_has_calls([mock.call({}), mock.call("b")]) - mock_context.assert_has_calls([mock.call("a"), mock.call({})]) + mock_context.assert_has_calls([mock.call("a", non_hidden=True), + mock.call({}, non_hidden=True)]) @mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner") - @mock.patch("rally.benchmark.engine.base_ctx.Context.validate") + @mock.patch("rally.benchmark.engine.base_ctx.ContextManager.validate") def test__validate_config_syntax__wrong_runner(self, mock_context, mock_runner): config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]} @@ -142,7 +143,7 @@ class BenchmarkEngineTestCase(test.TestCase): eng._validate_config_syntax, config) @mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner.validate") - @mock.patch("rally.benchmark.engine.base_ctx.Context") + @mock.patch("rally.benchmark.engine.base_ctx.ContextManager") def test__validate_config_syntax__wrong_context(self, mock_context, mock_runner): config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]} diff --git a/tests/doc/test_task_samples.py b/tests/doc/test_task_samples.py index 3089e90f22..e7c763e183 100644 --- a/tests/doc/test_task_samples.py +++ b/tests/doc/test_task_samples.py @@ -43,7 +43,7 @@ class TaskSampleTestCase(test.TestCase): eng = engine.BenchmarkEngine(task_config, mock.MagicMock()) eng.validate() - except Exception : + except Exception: print(traceback.format_exc()) self.assertTrue(False, "Wrong task config %s" % full_path)