From d12632579fa11baf1ba4e0132bfcc22a274e3cc9 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Tue, 11 Apr 2017 17:50:17 +0300 Subject: [PATCH] Split validation at 3 layers We can group validation by 3 types: syntax (check that config is valid for plugin in terms of jsonschema, required arguments, etc), platform (check that deployment has credentials for launching the plugin on specific environment), semantic(all others which operates users: for example validate that specific service is enabled). Splitting validation by these groups is required to support "validators for validators". I mean that if we check that syntax of config is valid and that we have credentials for all required platforms, we able to execute more complex validators and ensure that they will not failed on "random" failures like "AttributeError", "KeyError" and etc. Change-Id: I1ea38403af6cfbd6b7765ae3c0caca12759bc571 --- rally/cli/cliutils.py | 3 +- rally/common/validation.py | 55 ++- .../openstack/context/keystone/users.py | 2 + rally/task/engine.py | 172 +++++----- rally/task/validation.py | 2 + tests/unit/common/test_validation.py | 7 +- tests/unit/doc/test_task_samples.py | 7 +- tests/unit/rally_jobs/test_jobs.py | 7 +- tests/unit/task/test_engine.py | 318 ++++++++---------- 9 files changed, 304 insertions(+), 269 deletions(-) diff --git a/rally/cli/cliutils.py b/rally/cli/cliutils.py index 2a0a9fa047..49db3da47d 100644 --- a/rally/cli/cliutils.py +++ b/rally/cli/cliutils.py @@ -664,7 +664,8 @@ def run(argv, categories): exceptions.RallyException, jsonschema.ValidationError) as e: if logging.is_debug(): LOG.exception(e) - print(e) + else: + print(e) return 1 except sqlalchemy.exc.OperationalError as e: if logging.is_debug(): diff --git a/rally/common/validation.py b/rally/common/validation.py index ca0eafc206..57101f8ea8 100644 --- a/rally/common/validation.py +++ b/rally/common/validation.py @@ -18,12 +18,16 @@ import traceback import six +from rally.common import logging from rally.common.plugin import plugin from rally import exceptions configure = plugin.configure +LOG = logging.getLogger(__name__) + + @plugin.base() @six.add_metaclass(abc.ABCMeta) class Validator(plugin.Plugin): @@ -76,7 +80,12 @@ class RequiredPlatformValidator(Validator): if self.admin and credentials.get("admin") is None: return self.fail("No admin credential for %s" % self.platform) if self.users and len(credentials.get("users", ())) == 0: - return self.fail("No user credentials for %s" % self.platform) + if credentials.get("admin") is not None: + LOG.debug("Plugin %s requires 'users' for launching. There " + "are no specified users, assumes that 'users' " + "context can create them via admin user.") + else: + return self.fail("No user credentials for %s" % self.platform) def add(name, **kwargs): @@ -153,7 +162,7 @@ class ValidatablePluginMixin(object): @classmethod def validate(cls, name, credentials, config, plugin_cfg, - namespace=None, allow_hidden=False): + namespace=None, allow_hidden=False, vtype=None): """Execute all validators stored in meta of plugin. Iterate during all validators stored in the meta of Validator @@ -165,6 +174,10 @@ class ValidatablePluginMixin(object): :param credentials: credentials dict for all platforms :param config: dict with configuration of specified workload :param plugin_cfg: dict, with exact configuration of the plugin + :param allow_hidden: do not ignore hidden plugins + :param vtype: Type of validation. Allowed types: syntax, platform, + semantic. HINT: To specify several types use tuple or list with + types :returns: list of ValidationResult(is_valid=False) instances """ try: @@ -175,21 +188,45 @@ class ValidatablePluginMixin(object): cls.__name__, name) return [ValidationResult(is_valid=False, msg=msg)] + if vtype is None: + semantic = True + syntax = True + platform = True + else: + if not isinstance(vtype, (list, tuple)): + vtype = [vtype] + wrong_types = set(vtype) - {"semantic", "syntax", "platform"} + if wrong_types: + raise ValueError("Wrong type of validation: %s" % + ", ".join(wrong_types)) + semantic = "semantic" in vtype + syntax = "syntax" in vtype + platform = "platform" in vtype + + syntax_validators = [] platform_validators = [] regular_validators = [] plugin_validators = cls._load_validators(plugin) for validator, args, kwargs in plugin_validators: if issubclass(validator, RequiredPlatformValidator): - platform_validators.append((validator, args, kwargs)) + if platform: + platform_validators.append((validator, args, kwargs)) else: - regular_validators.append((validator, args, kwargs)) - - # Load platform validators from each validator - platform_validators.extend(cls._load_validators(validator)) + validators_of_validators = cls._load_validators(validator) + if validators_of_validators: + if semantic: + regular_validators.append((validator, args, kwargs)) + if platform: + # Load platform validators from each validator + platform_validators.extend(validators_of_validators) + else: + if syntax: + syntax_validators.append((validator, args, kwargs)) results = [] - for validators in (platform_validators, regular_validators): + for validators in (syntax_validators, platform_validators, + regular_validators): for validator_cls, args, kwargs in validators: try: validator = validator_cls(*args, **kwargs) @@ -207,6 +244,8 @@ class ValidatablePluginMixin(object): etype=type(exc).__name__, etraceback=traceback.format_exc()) if not result.is_valid: + LOG.debug("Result of validator '%s' is not successful for " + "plugin %s.", validator_cls.get_name(), name) results.append(result) if results: diff --git a/rally/plugins/openstack/context/keystone/users.py b/rally/plugins/openstack/context/keystone/users.py index 317217ef98..8c3288cfd7 100644 --- a/rally/plugins/openstack/context/keystone/users.py +++ b/rally/plugins/openstack/context/keystone/users.py @@ -30,6 +30,7 @@ from rally.plugins.openstack.services.identity import identity from rally.plugins.openstack.wrappers import network from rally.task import context from rally.task import utils +from rally.task import validation from rally.common import opts opts.register() @@ -45,6 +46,7 @@ PROJECT_DOMAIN_DESCR = "ID of domain in which projects will be created." USER_DOMAIN_DESCR = "ID of domain in which users will be created." +@validation.add("required_platform", platform="openstack", admin=True) @context.configure(name="users", namespace="openstack", order=100) class UserGenerator(context.Context): """Context class for generating temporary users/tenants for benchmarks.""" diff --git a/rally/task/engine.py b/rally/task/engine.py index e7904b8bad..009753994c 100644 --- a/rally/task/engine.py +++ b/rally/task/engine.py @@ -261,80 +261,92 @@ class TaskEngine(object): self.deployment = deployment self.abort_on_sla_failure = abort_on_sla_failure - @logging.log_task_wrapper(LOG.info, - _("Task validation of scenarios names.")) - def _validate_config_scenarios_name(self, config): - available = set(s.get_name() for s in scenario.Scenario.get_all()) + def _validate_workload(self, workload, credentials=None, vtype=None): + scenario_cls = scenario.Scenario.get(workload.name) + namespace = scenario_cls.get_namespace() + scenario_context = copy.deepcopy(scenario_cls.get_default_context()) - specified = set() - for subtask in config.subtasks: - for s in subtask.workloads: - specified.add(s.name) + results = [] - if not specified.issubset(available): - names = ", ".join(specified - available) - raise exceptions.NotFoundScenarios(names=names) + results.extend(scenario.Scenario.validate( + name=workload.name, + credentials=credentials, + config=workload.to_dict(), + plugin_cfg=None, + vtype=vtype)) + + if workload.runner: + results.extend(runner.ScenarioRunner.validate( + name=workload.runner["type"], + credentials=credentials, + config=None, + plugin_cfg=workload.runner, + namespace=namespace, + vtype=vtype)) + + for context_name, context_conf in workload.context.items(): + results.extend(context.Context.validate( + name=context_name, + credentials=credentials, + config=None, + plugin_cfg=context_conf, + namespace=namespace, + vtype=vtype)) + + for context_name, context_conf in scenario_context.items(): + results.extend(context.Context.validate( + name=context_name, + credentials=credentials, + config=None, + plugin_cfg=context_conf, + namespace=namespace, + allow_hidden=True, + vtype=vtype)) + + for sla_name, sla_conf in workload.sla.items(): + results.extend(sla.SLA.validate( + name=sla_name, + credentials=credentials, + config=None, + plugin_cfg=sla_conf, + vtype=vtype)) + + for hook_conf in workload.hooks: + results.extend(hook.Hook.validate( + name=hook_conf["name"], + credentials=credentials, + config=None, + plugin_cfg=hook_conf["args"], + vtype=vtype)) + + trigger_conf = hook_conf["trigger"] + results.extend(trigger.Trigger.validate( + name=trigger_conf["name"], + credentials=credentials, + config=None, + plugin_cfg=trigger_conf["args"], + vtype=vtype)) + + if results: + msg = "\n ".join([str(r) for r in results]) + kw = workload.make_exception_args(msg) + raise exceptions.InvalidTaskConfig(**kw) @logging.log_task_wrapper(LOG.info, _("Task validation of syntax.")) def _validate_config_syntax(self, config): for subtask in config.subtasks: for workload in subtask.workloads: - scenario_cls = scenario.Scenario.get(workload.name) - namespace = scenario_cls.get_namespace() - scenario_context = copy.deepcopy( - scenario_cls.get_default_context()) + self._validate_workload(workload, vtype="syntax") - results = [] - if workload.runner: - results.extend(runner.ScenarioRunner.validate( - name=workload.runner["type"], - credentials=None, - config=None, - plugin_cfg=workload.runner, - namespace=namespace)) - - for context_name, context_conf in workload.context.items(): - results.extend(context.Context.validate( - name=context_name, - credentials=None, - config=None, - plugin_cfg=context_conf, - namespace=namespace)) - - for context_name, context_conf in scenario_context.items(): - results.extend(context.Context.validate( - name=context_name, - credentials=None, - config=None, - plugin_cfg=context_conf, - namespace=namespace, - allow_hidden=True)) - - for sla_name, sla_conf in workload.sla.items(): - results.extend(sla.SLA.validate( - name=sla_name, - credentials=None, - config=None, - plugin_cfg=sla_conf)) - - for hook_conf in workload.hooks: - results.extend(hook.Hook.validate( - name=hook_conf["name"], - credentials=None, - config=None, - plugin_cfg=hook_conf["args"])) - - trigger_conf = hook_conf["trigger"] - results.extend(trigger.Trigger.validate( - name=trigger_conf["name"], - credentials=None, - config=None, - plugin_cfg=trigger_conf["args"])) - - if results: - msg = "\n ".join([str(r) for r in results]) - kw = workload.make_exception_args(msg) - raise exceptions.InvalidTaskConfig(**kw) + @logging.log_task_wrapper(LOG.info, _("Task validation of required " + "platforms.")) + def _validate_config_platforms(self, config): + credentials = self.deployment.get_all_credentials() + credentials = dict((p, creds[0]) for p, creds in credentials.items()) + for subtask in config.subtasks: + for workload in subtask.workloads: + self._validate_workload(workload, vtype="platform", + credentials=credentials) def _validate_config_semantic_helper(self, admin, user_context, workloads, platform): @@ -342,15 +354,9 @@ class TaskEngine(object): ctx.setup() users = ctx.context["users"] for workload in workloads: - results = scenario.Scenario.validate( - name=workload.name, - credentials={platform: {"admin": admin, "users": users}}, - config=workload.to_dict(), - plugin_cfg=None) - if results: - msg = "\n ".join([str(r) for r in results]) - kw = workload.make_exception_args(msg) - raise exceptions.InvalidTaskConfig(**kw) + credentials = {platform: {"admin": admin, "users": users}} + self._validate_workload(workload, credentials=credentials, + vtype="semantic") @logging.log_task_wrapper(LOG.info, _("Task validation of semantic.")) def _validate_config_semantic(self, config): @@ -405,19 +411,25 @@ class TaskEngine(object): platform) @logging.log_task_wrapper(LOG.info, _("Task validation.")) - def validate(self): - """Perform full task configuration validation.""" + def validate(self, only_syntax=False): + """Perform full task configuration validation. + + :param only_syntax: Check only syntax of task configuration + """ self.task.update_status(consts.TaskStatus.VALIDATING) try: - self._validate_config_scenarios_name(self.config) self._validate_config_syntax(self.config) + if only_syntax: + return + self._validate_config_platforms(self.config) self._validate_config_semantic(self.config) except Exception as e: exception_info = json.dumps(traceback.format_exc(), indent=2, separators=(",", ": ")) self.task.set_failed(type(e).__name__, str(e), exception_info) - if logging.is_debug(): + if (logging.is_debug() and + not isinstance(e, exceptions.InvalidTaskConfig)): LOG.exception(e) raise exceptions.InvalidTaskException(str(e)) @@ -782,5 +794,5 @@ class Workload(object): def make_exception_args(self, reason): return {"name": self.name, "pos": self.pos, - "config": self.to_dict(), + "config": json.dumps(self.to_dict()), "reason": reason} diff --git a/rally/task/validation.py b/rally/task/validation.py index 8354a76e8f..a2ec42a16d 100755 --- a/rally/task/validation.py +++ b/rally/task/validation.py @@ -39,6 +39,8 @@ ValidationResult = validation.ValidationResult add = validation.add +@validation.add("required_platform", platform="openstack", admin=True, + users=True) @validation.configure(name="old_validator", namespace="openstack") class OldValidator(validation.Validator): """Legacy validator for OpenStack scenarios""" diff --git a/tests/unit/common/test_validation.py b/tests/unit/common/test_validation.py index 84240bd943..6b2da057b7 100644 --- a/tests/unit/common/test_validation.py +++ b/tests/unit/common/test_validation.py @@ -118,7 +118,12 @@ class RequiredPlatformValidatorTestCase(test.TestCase): {"kwargs": {"platform": "foo", "admin": True}, "credentials": {"foo": {"admin": "fake_admin"}}}, {"kwargs": {"platform": "foo", "admin": True, "users": True}, - "credentials": {"foo": {"admin": "fake_admin"}}, + "credentials": {"foo": {"admin": "fake_admin"}}}, + {"kwargs": {"platform": "foo", "admin": True, "users": True}, + "credentials": {"foo": {"users": ["fake_user"]}}, + "error_msg": "No admin credential for foo"}, + {"kwargs": {"platform": "foo", "users": True}, + "credentials": {"foo": {}}, "error_msg": "No user credentials for foo"}, {"kwargs": {"platform": "foo", "admin": True, "users": True}, "credentials": {"foo": {"admin": "fake_admin", diff --git a/tests/unit/doc/test_task_samples.py b/tests/unit/doc/test_task_samples.py index 13ff1b7df0..83488b3140 100644 --- a/tests/unit/doc/test_task_samples.py +++ b/tests/unit/doc/test_task_samples.py @@ -40,10 +40,7 @@ class TaskSampleTestCase(test.TestCase): if os.environ.get("TOX_ENV_NAME") == "cover": self.skipTest("There is no need to check samples in coverage job.") - @mock.patch("rally.task.engine.TaskEngine" - "._validate_config_semantic") - def test_schema_is_valid(self, - mock_task_engine__validate_config_semantic): + def test_schema_is_valid(self): scenarios = set() for dirname, dirnames, filenames in os.walk(self.samples_path): @@ -61,7 +58,7 @@ class TaskSampleTestCase(test.TestCase): (task_file.read())) eng = engine.TaskEngine(task_config, mock.MagicMock(), mock.Mock()) - eng.validate() + eng.validate(only_syntax=True) except Exception: print(traceback.format_exc()) self.fail("Invalid task file: %s" % full_path) diff --git a/tests/unit/rally_jobs/test_jobs.py b/tests/unit/rally_jobs/test_jobs.py index 6a17846c99..c176db58da 100644 --- a/tests/unit/rally_jobs/test_jobs.py +++ b/tests/unit/rally_jobs/test_jobs.py @@ -29,10 +29,7 @@ class RallyJobsTestCase(test.TestCase): rally_jobs_path = os.path.join( os.path.dirname(rally.__file__), "..", "rally-jobs") - @mock.patch("rally.task.engine.TaskEngine" - "._validate_config_semantic") - def test_schema_is_valid( - self, mock_task_engine__validate_config_semantic): + def test_schema_is_valid(self): discover.load_plugins(os.path.join(self.rally_jobs_path, "plugins")) files = {f for f in os.listdir(self.rally_jobs_path) @@ -64,7 +61,7 @@ class RallyJobsTestCase(test.TestCase): eng = engine.TaskEngine(task, mock.MagicMock(), mock.Mock()) - eng.validate() + eng.validate(only_syntax=True) except Exception: print(traceback.format_exc()) self.fail("Wrong task input file: %s" % full_path) diff --git a/tests/unit/task/test_engine.py b/tests/unit/task/test_engine.py index 1c56ced049..41cb44a0c7 100644 --- a/tests/unit/task/test_engine.py +++ b/tests/unit/task/test_engine.py @@ -16,6 +16,7 @@ """Tests for the Test engine.""" import collections +import json import threading import mock @@ -62,18 +63,15 @@ class TaskEngineTestCase(test.TestCase): mock.Mock()) mock_validate = mock.MagicMock() - eng._validate_config_scenarios_name = mock_validate.names eng._validate_config_syntax = mock_validate.syntax + eng._validate_config_platforms = mock_validate.platforms eng._validate_config_semantic = mock_validate.semantic eng.validate() - expected_calls = [ - mock.call.names(config), - mock.call.syntax(config), - mock.call.semantic(config) - ] - mock_validate.assert_has_calls(expected_calls) + mock_validate.syntax.assert_called_once_with(config) + mock_validate.platforms.assert_called_once_with(config) + mock_validate.semantic.assert_called_once_with(config) def test_validate__wrong_schema(self): config = { @@ -84,85 +82,36 @@ class TaskEngineTestCase(test.TestCase): engine.TaskEngine, config, task, mock.Mock()) self.assertTrue(task.set_failed.called) - @mock.patch("rally.task.engine.TaskConfig") - def test_validate__wrong_scenarios_name(self, mock_task_config): - task = mock.MagicMock() - eng = engine.TaskEngine(mock.MagicMock(), task, mock.Mock()) - eng._validate_config_scenarios_name = mock.MagicMock( - side_effect=exceptions.NotFoundScenarios) - - self.assertRaises(exceptions.InvalidTaskException, eng.validate) - self.assertTrue(task.set_failed.called) - @mock.patch("rally.task.engine.TaskConfig") def test_validate__wrong_syntax(self, mock_task_config): task = mock.MagicMock() eng = engine.TaskEngine(mock.MagicMock(), task, mock.Mock()) - eng._validate_config_scenarios_name = mock.MagicMock() eng._validate_config_syntax = mock.MagicMock( side_effect=exceptions.InvalidTaskConfig) + eng._validate_config_platforms = mock.Mock() self.assertRaises(exceptions.InvalidTaskException, eng.validate) + self.assertTrue(task.set_failed.called) + # the next validation step should not be processed + self.assertFalse(eng._validate_config_platforms.called) @mock.patch("rally.task.engine.TaskConfig") def test_validate__wrong_semantic(self, mock_task_config): task = mock.MagicMock() eng = engine.TaskEngine(mock.MagicMock(), task, mock.Mock()) - eng._validate_config_scenarios_name = mock.MagicMock() eng._validate_config_syntax = mock.MagicMock() + eng._validate_config_platforms = mock.MagicMock() eng._validate_config_semantic = mock.MagicMock( side_effect=exceptions.InvalidTaskConfig) self.assertRaises(exceptions.InvalidTaskException, eng.validate) self.assertTrue(task.set_failed.called) - - @mock.patch("rally.task.engine.TaskConfig") - @mock.patch("rally.task.engine.scenario.Scenario.get_all") - def test__validate_config_scenarios_name( - self, mock_scenario_get_all, mock_task_config): - - mock_task_instance = mock.MagicMock() - mock_subtask = mock.MagicMock() - mock_subtask.workloads = [ - engine.Workload({"name": "a"}, 0), - engine.Workload({"name": "b"}, 1) - ] - mock_task_instance.subtasks = [mock_subtask] - - mock_scenario_get_all.return_value = [ - mock.MagicMock(get_name=lambda: "e"), - mock.MagicMock(get_name=lambda: "b"), - mock.MagicMock(get_name=lambda: "a") - ] - eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(), - mock.Mock()) - eng._validate_config_scenarios_name(mock_task_instance) - - @mock.patch("rally.task.engine.TaskConfig") - @mock.patch("rally.task.engine.scenario.Scenario") - def test__validate_config_scenarios_name_non_exsisting( - self, mock_scenario, mock_task_config): - - mock_task_instance = mock.MagicMock() - mock_subtask = mock.MagicMock() - mock_subtask.workloads = [ - engine.Workload({"name": "exist"}, 0), - engine.Workload({"name": "nonexist1"}, 1), - engine.Workload({"name": "nonexist2"}, 2) - ] - mock_task_instance.subtasks = [mock_subtask] - mock_scenario.get_all.return_value = [ - mock.Mock(get_name=lambda: "exist"), - mock.Mock(get_name=lambda: "aaa")] - eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(), - mock.Mock()) - - exc = self.assertRaises(exceptions.NotFoundScenarios, - eng._validate_config_scenarios_name, - mock_task_instance) - self.assertEqual("There are no benchmark scenarios with names: " - "`nonexist2, nonexist1`.", str(exc)) + # all steps of validation are called, which means that the last one is + # failed + self.assertTrue(eng._validate_config_syntax) + self.assertTrue(eng._validate_config_platforms) + self.assertTrue(eng._validate_config_semantic) @mock.patch("rally.task.engine.scenario.Scenario.get") @mock.patch("rally.task.sla.SLA.validate") @@ -171,15 +120,15 @@ class TaskEngineTestCase(test.TestCase): @mock.patch("rally.task.engine.TaskConfig") @mock.patch("rally.task.engine.runner.ScenarioRunner.validate") @mock.patch("rally.task.engine.context.Context.validate") - def test__validate_config_syntax( + def test__validate_workload( self, mock_context_validate, mock_scenario_runner_validate, mock_task_config, mock_hook_validate, mock_trigger_validate, mock_sla_validate, - mock_scenario_get - ): + mock_scenario_get): + mock_context_validate.return_value = [] mock_sla_validate.return_value = [] mock_hook_validate.return_value = [] @@ -188,87 +137,81 @@ class TaskEngineTestCase(test.TestCase): scenario_cls = mock_scenario_get.return_value scenario_cls.get_namespace.return_value = "default" scenario_cls.get_default_context.return_value = default_context - mock_task_instance = mock.MagicMock() - mock_subtask = mock.MagicMock() + + scenario_name = "Foo.bar" + runner_type = "MegaRunner" hook_conf = {"name": "c", "args": "c_args", "trigger": {"name": "d", "args": "d_args"}} - mock_subtask.workloads = [ - engine.Workload({"name": "sca", "context": {"a": "a_conf"}}, 0), - engine.Workload({"name": "sca", "runner": {"type": "b"}, - "sla": {"foo_sla": "sla_conf"}}, 1), - engine.Workload({"name": "sca", "hooks": [hook_conf]}, 2), - ] - mock_task_instance.subtasks = [mock_subtask] + workload = engine.Workload({"name": scenario_name, + "runner": {"type": runner_type}, + "context": {"a": "a_conf"}, + "hooks": [hook_conf], + "sla": {"foo_sla": "sla_conf"}}, 2) + eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(), mock.Mock()) - eng._validate_config_syntax(mock_task_instance) + + eng._validate_workload(workload) + mock_scenario_runner_validate.assert_called_once_with( - name="b", credentials=None, config=None, - plugin_cfg={"type": "b"}, namespace="default") - mock_context_validate.assert_has_calls( - [mock.call(name="a", - credentials=None, - config=None, - plugin_cfg="a_conf", - namespace="default"), - mock.call(name="foo", - credentials=None, - config=None, - plugin_cfg="foo_conf", - namespace="default", - allow_hidden=True), - mock.call(name="foo", - credentials=None, - config=None, - plugin_cfg="foo_conf", - namespace="default", - allow_hidden=True), - mock.call(name="foo", - credentials=None, - config=None, - plugin_cfg="foo_conf", - namespace="default", - allow_hidden=True)], - any_order=True - ) + name=runner_type, credentials=None, config=None, + plugin_cfg={"type": runner_type}, namespace="default", vtype=None) + self.assertEqual([mock.call(name="a", + credentials=None, + config=None, + plugin_cfg="a_conf", + namespace="default", + vtype=None), + mock.call(name="foo", + credentials=None, + config=None, + plugin_cfg="foo_conf", + namespace="default", + allow_hidden=True, + vtype=None)], + mock_context_validate.call_args_list) mock_sla_validate.assert_called_once_with( config=None, credentials=None, - name="foo_sla", plugin_cfg="sla_conf") + name="foo_sla", plugin_cfg="sla_conf", vtype=None) mock_hook_validate.assert_called_once_with( - config=None, credentials=None, name="c", plugin_cfg="c_args") + config=None, credentials=None, name="c", plugin_cfg="c_args", + vtype=None) mock_trigger_validate.assert_called_once_with( - config=None, credentials=None, name="d", plugin_cfg="d_args") + config=None, credentials=None, name="d", plugin_cfg="d_args", + vtype=None) + @mock.patch("rally.task.engine.json.dumps") @mock.patch("rally.task.engine.scenario.Scenario.get") @mock.patch("rally.task.engine.TaskConfig") @mock.patch("rally.task.engine.runner.ScenarioRunner.validate") - def test__validate_config_syntax__wrong_runner( - self, mock_scenario_runner_validate, - mock_task_config, mock_scenario_get): - result = validation.ValidationResult(False, "context_error") + def test___validate_workload__wrong_runner( + self, mock_scenario_runner_validate, mock_task_config, + mock_scenario_get, mock_dumps): + mock_dumps.return_value = "" + result = validation.ValidationResult(False, "There is no such runner") mock_scenario_runner_validate.return_value = [result] scenario_cls = mock_scenario_get.return_value scenario_cls.get_default_context.return_value = {} - mock_task_instance = mock.MagicMock() - mock_subtask = mock.MagicMock() - mock_subtask.workloads = [ - engine.Workload({"name": "sca"}, 0), - engine.Workload({"name": "sca", "runner": {"type": "b"}}, 1) - ] - mock_task_instance.subtasks = [mock_subtask] + workload = engine.Workload({"name": "sca", "runner": {"type": "b"}}, 0) eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(), mock.Mock()) - self.assertRaises(exceptions.InvalidTaskConfig, - eng._validate_config_syntax, mock_task_instance) + e = self.assertRaises(exceptions.InvalidTaskConfig, + eng._validate_workload, workload) + self.assertEqual("Input task is invalid!\n\nSubtask sca[0] has wrong " + "configuration\nSubtask configuration:\n" + "\n\nReason(s):\n" + " There is no such runner", e.format_message()) + @mock.patch("rally.task.engine.json.dumps") @mock.patch("rally.task.engine.scenario.Scenario.get") @mock.patch("rally.task.engine.TaskConfig") @mock.patch("rally.task.engine.context.Context.validate") def test__validate_config_syntax__wrong_context( - self, mock_context_validate, - mock_task_config, mock_scenario_get): + self, mock_context_validate, mock_task_config, mock_scenario_get, + mock_dumps): + mock_dumps.return_value = "" result = validation.ValidationResult(False, "context_error") mock_context_validate.return_value = [result] scenario_cls = mock_scenario_get.return_value @@ -276,21 +219,27 @@ class TaskEngineTestCase(test.TestCase): mock_task_instance = mock.MagicMock() mock_subtask = mock.MagicMock() mock_subtask.workloads = [ - engine.Workload({"name": "sca", "context": {"a": "a_conf"}}, 0), - engine.Workload({"name": "sca"}, 1) + engine.Workload({"name": "sca"}, 0), + engine.Workload({"name": "sca", "context": {"a": "a_conf"}}, 1) ] mock_task_instance.subtasks = [mock_subtask] eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(), mock.Mock()) - self.assertRaises(exceptions.InvalidTaskConfig, - eng._validate_config_syntax, mock_task_instance) + e = self.assertRaises(exceptions.InvalidTaskConfig, + eng._validate_config_syntax, mock_task_instance) + self.assertEqual("Input task is invalid!\n\nSubtask sca[1] has wrong " + "configuration\nSubtask configuration:\n\n\n" + "Reason(s):\n context_error", e.format_message()) + @mock.patch("rally.task.engine.json.dumps") @mock.patch("rally.task.engine.scenario.Scenario.get") @mock.patch("rally.task.sla.SLA.validate") @mock.patch("rally.task.engine.TaskConfig") def test__validate_config_syntax__wrong_sla( - self, mock_task_config, mock_sla_validate, mock_scenario_get): + self, mock_task_config, mock_sla_validate, mock_scenario_get, + mock_dumps): + mock_dumps.return_value = "" result = validation.ValidationResult(False, "sla_error") mock_sla_validate.return_value = [result] scenario_cls = mock_scenario_get.return_value @@ -304,14 +253,23 @@ class TaskEngineTestCase(test.TestCase): mock_task_instance.subtasks = [mock_subtask] eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(), mock.Mock()) - self.assertRaises(exceptions.InvalidTaskConfig, - eng._validate_config_syntax, mock_task_instance) + e = self.assertRaises(exceptions.InvalidTaskConfig, + eng._validate_config_syntax, mock_task_instance) + self.assertEqual("Input task is invalid!\n\n" + "Subtask sca[1] has wrong configuration\n" + "Subtask configuration:\n\n\n" + "Reason(s):\n sla_error", e.format_message()) + @mock.patch("rally.task.engine.json.dumps") @mock.patch("rally.task.engine.scenario.Scenario.get") @mock.patch("rally.task.hook.Hook.validate") + @mock.patch("rally.task.trigger.Trigger.validate") @mock.patch("rally.task.engine.TaskConfig") def test__validate_config_syntax__wrong_hook( - self, mock_task_config, mock_hook_validate, mock_scenario_get): + self, mock_task_config, mock_trigger_validate, mock_hook_validate, + mock_scenario_get, mock_dumps): + mock_dumps.return_value = "" + mock_trigger_validate.return_value = [] result = validation.ValidationResult(False, "hook_error") mock_hook_validate.return_value = [result] scenario_cls = mock_scenario_get.return_value @@ -329,16 +287,23 @@ class TaskEngineTestCase(test.TestCase): eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(), mock.Mock()) - self.assertRaises(exceptions.InvalidTaskConfig, - eng._validate_config_syntax, mock_task_instance) + e = self.assertRaises(exceptions.InvalidTaskConfig, + eng._validate_config_syntax, mock_task_instance) + self.assertEqual("Input task is invalid!\n\n" + "Subtask sca[1] has wrong configuration\n" + "Subtask configuration:\n\n\n" + "Reason(s):\n hook_error", e.format_message()) + + @mock.patch("rally.task.engine.json.dumps") @mock.patch("rally.task.engine.scenario.Scenario.get") @mock.patch("rally.task.trigger.Trigger.validate") @mock.patch("rally.task.hook.Hook.validate") @mock.patch("rally.task.engine.TaskConfig") def test__validate_config_syntax__wrong_trigger( self, mock_task_config, mock_hook_validate, mock_trigger_validate, - mock_scenario_get): + mock_scenario_get, mock_dumps): + mock_dumps.return_value = "" result = validation.ValidationResult(False, "trigger_error") mock_trigger_validate.return_value = [result] mock_hook_validate.return_value = [] @@ -357,42 +322,32 @@ class TaskEngineTestCase(test.TestCase): eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(), mock.Mock()) - self.assertRaises(exceptions.InvalidTaskConfig, - eng._validate_config_syntax, mock_task_instance) + e = self.assertRaises(exceptions.InvalidTaskConfig, + eng._validate_config_syntax, mock_task_instance) + + self.assertEqual("Input task is invalid!\n\n" + "Subtask sca[1] has wrong configuration\n" + "Subtask configuration:\n\n\n" + "Reason(s):\n trigger_error", e.format_message()) - @mock.patch("rally.task.engine.scenario.Scenario.validate") @mock.patch("rally.task.engine.TaskConfig") - def test__validate_config_semantic_helper(self, mock_task_config, - mock_scenario_validate): - mock_scenario_validate.return_value = [] + def test__validate_config_semantic_helper(self, mock_task_config): eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(), mock.Mock()) + eng._validate_workload = mock.Mock() workloads = [engine.Workload( {"name": "name", "runner": "runner", "args": "args"}, 0)] + users = [{"foo": "user1"}] user_context = mock.MagicMock() - user_context.__enter__.return_value.context = { - "users": [{"foo": "user1"}]} + user_context.__enter__.return_value.context = {"users": users} + eng._validate_config_semantic_helper( "admin", user_context, workloads, "foo") - mock_scenario_validate.assert_called_once_with( - name="name", config={"runner": "runner", "args": "args"}, - credentials={"foo": {"admin": "admin", - "users": [{"foo": "user1"}]}}, - plugin_cfg=None) - @mock.patch("rally.task.engine.scenario.Scenario.validate") - @mock.patch("rally.task.engine.TaskConfig") - def test__validate_config_semanitc_helper_invalid_arg( - self, mock_task_config, mock_scenario_validate): - eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(), - mock.Mock()) - mock_scenario_validate.return_value = [ - validation.ValidationResult(False, msg="foo")] - user_context = mock.MagicMock() - workloads = [engine.Workload({"name": "name"}, 0)] - self.assertRaises(exceptions.InvalidTaskConfig, - eng._validate_config_semantic_helper, "a", - user_context, workloads, "foo") + eng._validate_workload.assert_called_once_with( + workloads[0], credentials={"foo": {"admin": "admin", + "users": users}}, + vtype="semantic") @mock.patch("rally.task.engine.scenario.Scenario.get") @mock.patch("rally.task.engine.context.Context") @@ -438,6 +393,36 @@ class TaskEngineTestCase(test.TestCase): mock.call(admin, user_context, [wconf2, wconf3], "openstack"), ], any_order=True) + @mock.patch("rally.task.engine.TaskConfig") + @mock.patch("rally.task.engine.TaskEngine._validate_workload") + def test__validate_config_platforms( + self, mock__validate_workload, mock_task_config): + + class FakeDeployment(object): + def __init__(self, credentials): + self._creds = credentials + self.get_all_credentials = mock.Mock() + self.get_all_credentials.return_value = self._creds + + foo_cred1 = {"admin": "admin", "users": ["user1"]} + foo_cred2 = {"admin": "admin", "users": ["user1"]} + deployment = FakeDeployment({"foo": [foo_cred1, foo_cred2]}) + + workload1 = "workload1" + workload2 = "workload2" + subtasks = [mock.Mock(workloads=[workload1]), + mock.Mock(workloads=[workload2])] + config = mock.Mock(subtasks=subtasks) + eng = engine.TaskEngine({}, mock.MagicMock(), deployment) + + eng._validate_config_platforms(config) + + self.assertEqual( + [mock.call(w, vtype="platform", credentials={"foo": foo_cred1}) + for w in (workload1, workload2)], + mock__validate_workload.call_args_list) + deployment.get_all_credentials.assert_called_once_with() + @mock.patch("rally.common.objects.Task.get_status") @mock.patch("rally.task.engine.TaskConfig") @mock.patch("rally.task.engine.ResultConsumer") @@ -1147,13 +1132,8 @@ class WorkloadTestCase(test.TestCase): "name": "n", "pos": 0, "reason": "r", - "config": { - "runner": "r", - "context": "c", - "sla": "s", - "hooks": "h", - "args": "a" - } + "config": json.dumps({"runner": "r", "context": "c", "sla": "s", + "hooks": "h", "args": "a"}) } self.assertEqual(expected_args,