Split validation at 3 layers
We can group validation by 3 types: syntax (check that config is valid for plugin in terms of jsonschema, required arguments, etc), platform (check that deployment has credentials for launching the plugin on specific environment), semantic(all others which operates users: for example validate that specific service is enabled). Splitting validation by these groups is required to support "validators for validators". I mean that if we check that syntax of config is valid and that we have credentials for all required platforms, we able to execute more complex validators and ensure that they will not failed on "random" failures like "AttributeError", "KeyError" and etc. Change-Id: I1ea38403af6cfbd6b7765ae3c0caca12759bc571
This commit is contained in:
parent
3d113b5a23
commit
d12632579f
@ -664,7 +664,8 @@ def run(argv, categories):
|
||||
exceptions.RallyException, jsonschema.ValidationError) as e:
|
||||
if logging.is_debug():
|
||||
LOG.exception(e)
|
||||
print(e)
|
||||
else:
|
||||
print(e)
|
||||
return 1
|
||||
except sqlalchemy.exc.OperationalError as e:
|
||||
if logging.is_debug():
|
||||
|
@ -18,12 +18,16 @@ import traceback
|
||||
|
||||
import six
|
||||
|
||||
from rally.common import logging
|
||||
from rally.common.plugin import plugin
|
||||
from rally import exceptions
|
||||
|
||||
configure = plugin.configure
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@plugin.base()
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class Validator(plugin.Plugin):
|
||||
@ -76,7 +80,12 @@ class RequiredPlatformValidator(Validator):
|
||||
if self.admin and credentials.get("admin") is None:
|
||||
return self.fail("No admin credential for %s" % self.platform)
|
||||
if self.users and len(credentials.get("users", ())) == 0:
|
||||
return self.fail("No user credentials for %s" % self.platform)
|
||||
if credentials.get("admin") is not None:
|
||||
LOG.debug("Plugin %s requires 'users' for launching. There "
|
||||
"are no specified users, assumes that 'users' "
|
||||
"context can create them via admin user.")
|
||||
else:
|
||||
return self.fail("No user credentials for %s" % self.platform)
|
||||
|
||||
|
||||
def add(name, **kwargs):
|
||||
@ -153,7 +162,7 @@ class ValidatablePluginMixin(object):
|
||||
|
||||
@classmethod
|
||||
def validate(cls, name, credentials, config, plugin_cfg,
|
||||
namespace=None, allow_hidden=False):
|
||||
namespace=None, allow_hidden=False, vtype=None):
|
||||
"""Execute all validators stored in meta of plugin.
|
||||
|
||||
Iterate during all validators stored in the meta of Validator
|
||||
@ -165,6 +174,10 @@ class ValidatablePluginMixin(object):
|
||||
:param credentials: credentials dict for all platforms
|
||||
:param config: dict with configuration of specified workload
|
||||
:param plugin_cfg: dict, with exact configuration of the plugin
|
||||
:param allow_hidden: do not ignore hidden plugins
|
||||
:param vtype: Type of validation. Allowed types: syntax, platform,
|
||||
semantic. HINT: To specify several types use tuple or list with
|
||||
types
|
||||
:returns: list of ValidationResult(is_valid=False) instances
|
||||
"""
|
||||
try:
|
||||
@ -175,21 +188,45 @@ class ValidatablePluginMixin(object):
|
||||
cls.__name__, name)
|
||||
return [ValidationResult(is_valid=False, msg=msg)]
|
||||
|
||||
if vtype is None:
|
||||
semantic = True
|
||||
syntax = True
|
||||
platform = True
|
||||
else:
|
||||
if not isinstance(vtype, (list, tuple)):
|
||||
vtype = [vtype]
|
||||
wrong_types = set(vtype) - {"semantic", "syntax", "platform"}
|
||||
if wrong_types:
|
||||
raise ValueError("Wrong type of validation: %s" %
|
||||
", ".join(wrong_types))
|
||||
semantic = "semantic" in vtype
|
||||
syntax = "syntax" in vtype
|
||||
platform = "platform" in vtype
|
||||
|
||||
syntax_validators = []
|
||||
platform_validators = []
|
||||
regular_validators = []
|
||||
|
||||
plugin_validators = cls._load_validators(plugin)
|
||||
for validator, args, kwargs in plugin_validators:
|
||||
if issubclass(validator, RequiredPlatformValidator):
|
||||
platform_validators.append((validator, args, kwargs))
|
||||
if platform:
|
||||
platform_validators.append((validator, args, kwargs))
|
||||
else:
|
||||
regular_validators.append((validator, args, kwargs))
|
||||
|
||||
# Load platform validators from each validator
|
||||
platform_validators.extend(cls._load_validators(validator))
|
||||
validators_of_validators = cls._load_validators(validator)
|
||||
if validators_of_validators:
|
||||
if semantic:
|
||||
regular_validators.append((validator, args, kwargs))
|
||||
if platform:
|
||||
# Load platform validators from each validator
|
||||
platform_validators.extend(validators_of_validators)
|
||||
else:
|
||||
if syntax:
|
||||
syntax_validators.append((validator, args, kwargs))
|
||||
|
||||
results = []
|
||||
for validators in (platform_validators, regular_validators):
|
||||
for validators in (syntax_validators, platform_validators,
|
||||
regular_validators):
|
||||
for validator_cls, args, kwargs in validators:
|
||||
try:
|
||||
validator = validator_cls(*args, **kwargs)
|
||||
@ -207,6 +244,8 @@ class ValidatablePluginMixin(object):
|
||||
etype=type(exc).__name__,
|
||||
etraceback=traceback.format_exc())
|
||||
if not result.is_valid:
|
||||
LOG.debug("Result of validator '%s' is not successful for "
|
||||
"plugin %s.", validator_cls.get_name(), name)
|
||||
results.append(result)
|
||||
|
||||
if results:
|
||||
|
@ -30,6 +30,7 @@ from rally.plugins.openstack.services.identity import identity
|
||||
from rally.plugins.openstack.wrappers import network
|
||||
from rally.task import context
|
||||
from rally.task import utils
|
||||
from rally.task import validation
|
||||
|
||||
from rally.common import opts
|
||||
opts.register()
|
||||
@ -45,6 +46,7 @@ PROJECT_DOMAIN_DESCR = "ID of domain in which projects will be created."
|
||||
USER_DOMAIN_DESCR = "ID of domain in which users will be created."
|
||||
|
||||
|
||||
@validation.add("required_platform", platform="openstack", admin=True)
|
||||
@context.configure(name="users", namespace="openstack", order=100)
|
||||
class UserGenerator(context.Context):
|
||||
"""Context class for generating temporary users/tenants for benchmarks."""
|
||||
|
@ -261,80 +261,92 @@ class TaskEngine(object):
|
||||
self.deployment = deployment
|
||||
self.abort_on_sla_failure = abort_on_sla_failure
|
||||
|
||||
@logging.log_task_wrapper(LOG.info,
|
||||
_("Task validation of scenarios names."))
|
||||
def _validate_config_scenarios_name(self, config):
|
||||
available = set(s.get_name() for s in scenario.Scenario.get_all())
|
||||
def _validate_workload(self, workload, credentials=None, vtype=None):
|
||||
scenario_cls = scenario.Scenario.get(workload.name)
|
||||
namespace = scenario_cls.get_namespace()
|
||||
scenario_context = copy.deepcopy(scenario_cls.get_default_context())
|
||||
|
||||
specified = set()
|
||||
for subtask in config.subtasks:
|
||||
for s in subtask.workloads:
|
||||
specified.add(s.name)
|
||||
results = []
|
||||
|
||||
if not specified.issubset(available):
|
||||
names = ", ".join(specified - available)
|
||||
raise exceptions.NotFoundScenarios(names=names)
|
||||
results.extend(scenario.Scenario.validate(
|
||||
name=workload.name,
|
||||
credentials=credentials,
|
||||
config=workload.to_dict(),
|
||||
plugin_cfg=None,
|
||||
vtype=vtype))
|
||||
|
||||
if workload.runner:
|
||||
results.extend(runner.ScenarioRunner.validate(
|
||||
name=workload.runner["type"],
|
||||
credentials=credentials,
|
||||
config=None,
|
||||
plugin_cfg=workload.runner,
|
||||
namespace=namespace,
|
||||
vtype=vtype))
|
||||
|
||||
for context_name, context_conf in workload.context.items():
|
||||
results.extend(context.Context.validate(
|
||||
name=context_name,
|
||||
credentials=credentials,
|
||||
config=None,
|
||||
plugin_cfg=context_conf,
|
||||
namespace=namespace,
|
||||
vtype=vtype))
|
||||
|
||||
for context_name, context_conf in scenario_context.items():
|
||||
results.extend(context.Context.validate(
|
||||
name=context_name,
|
||||
credentials=credentials,
|
||||
config=None,
|
||||
plugin_cfg=context_conf,
|
||||
namespace=namespace,
|
||||
allow_hidden=True,
|
||||
vtype=vtype))
|
||||
|
||||
for sla_name, sla_conf in workload.sla.items():
|
||||
results.extend(sla.SLA.validate(
|
||||
name=sla_name,
|
||||
credentials=credentials,
|
||||
config=None,
|
||||
plugin_cfg=sla_conf,
|
||||
vtype=vtype))
|
||||
|
||||
for hook_conf in workload.hooks:
|
||||
results.extend(hook.Hook.validate(
|
||||
name=hook_conf["name"],
|
||||
credentials=credentials,
|
||||
config=None,
|
||||
plugin_cfg=hook_conf["args"],
|
||||
vtype=vtype))
|
||||
|
||||
trigger_conf = hook_conf["trigger"]
|
||||
results.extend(trigger.Trigger.validate(
|
||||
name=trigger_conf["name"],
|
||||
credentials=credentials,
|
||||
config=None,
|
||||
plugin_cfg=trigger_conf["args"],
|
||||
vtype=vtype))
|
||||
|
||||
if results:
|
||||
msg = "\n ".join([str(r) for r in results])
|
||||
kw = workload.make_exception_args(msg)
|
||||
raise exceptions.InvalidTaskConfig(**kw)
|
||||
|
||||
@logging.log_task_wrapper(LOG.info, _("Task validation of syntax."))
|
||||
def _validate_config_syntax(self, config):
|
||||
for subtask in config.subtasks:
|
||||
for workload in subtask.workloads:
|
||||
scenario_cls = scenario.Scenario.get(workload.name)
|
||||
namespace = scenario_cls.get_namespace()
|
||||
scenario_context = copy.deepcopy(
|
||||
scenario_cls.get_default_context())
|
||||
self._validate_workload(workload, vtype="syntax")
|
||||
|
||||
results = []
|
||||
if workload.runner:
|
||||
results.extend(runner.ScenarioRunner.validate(
|
||||
name=workload.runner["type"],
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg=workload.runner,
|
||||
namespace=namespace))
|
||||
|
||||
for context_name, context_conf in workload.context.items():
|
||||
results.extend(context.Context.validate(
|
||||
name=context_name,
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg=context_conf,
|
||||
namespace=namespace))
|
||||
|
||||
for context_name, context_conf in scenario_context.items():
|
||||
results.extend(context.Context.validate(
|
||||
name=context_name,
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg=context_conf,
|
||||
namespace=namespace,
|
||||
allow_hidden=True))
|
||||
|
||||
for sla_name, sla_conf in workload.sla.items():
|
||||
results.extend(sla.SLA.validate(
|
||||
name=sla_name,
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg=sla_conf))
|
||||
|
||||
for hook_conf in workload.hooks:
|
||||
results.extend(hook.Hook.validate(
|
||||
name=hook_conf["name"],
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg=hook_conf["args"]))
|
||||
|
||||
trigger_conf = hook_conf["trigger"]
|
||||
results.extend(trigger.Trigger.validate(
|
||||
name=trigger_conf["name"],
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg=trigger_conf["args"]))
|
||||
|
||||
if results:
|
||||
msg = "\n ".join([str(r) for r in results])
|
||||
kw = workload.make_exception_args(msg)
|
||||
raise exceptions.InvalidTaskConfig(**kw)
|
||||
@logging.log_task_wrapper(LOG.info, _("Task validation of required "
|
||||
"platforms."))
|
||||
def _validate_config_platforms(self, config):
|
||||
credentials = self.deployment.get_all_credentials()
|
||||
credentials = dict((p, creds[0]) for p, creds in credentials.items())
|
||||
for subtask in config.subtasks:
|
||||
for workload in subtask.workloads:
|
||||
self._validate_workload(workload, vtype="platform",
|
||||
credentials=credentials)
|
||||
|
||||
def _validate_config_semantic_helper(self, admin, user_context,
|
||||
workloads, platform):
|
||||
@ -342,15 +354,9 @@ class TaskEngine(object):
|
||||
ctx.setup()
|
||||
users = ctx.context["users"]
|
||||
for workload in workloads:
|
||||
results = scenario.Scenario.validate(
|
||||
name=workload.name,
|
||||
credentials={platform: {"admin": admin, "users": users}},
|
||||
config=workload.to_dict(),
|
||||
plugin_cfg=None)
|
||||
if results:
|
||||
msg = "\n ".join([str(r) for r in results])
|
||||
kw = workload.make_exception_args(msg)
|
||||
raise exceptions.InvalidTaskConfig(**kw)
|
||||
credentials = {platform: {"admin": admin, "users": users}}
|
||||
self._validate_workload(workload, credentials=credentials,
|
||||
vtype="semantic")
|
||||
|
||||
@logging.log_task_wrapper(LOG.info, _("Task validation of semantic."))
|
||||
def _validate_config_semantic(self, config):
|
||||
@ -405,19 +411,25 @@ class TaskEngine(object):
|
||||
platform)
|
||||
|
||||
@logging.log_task_wrapper(LOG.info, _("Task validation."))
|
||||
def validate(self):
|
||||
"""Perform full task configuration validation."""
|
||||
def validate(self, only_syntax=False):
|
||||
"""Perform full task configuration validation.
|
||||
|
||||
:param only_syntax: Check only syntax of task configuration
|
||||
"""
|
||||
self.task.update_status(consts.TaskStatus.VALIDATING)
|
||||
try:
|
||||
self._validate_config_scenarios_name(self.config)
|
||||
self._validate_config_syntax(self.config)
|
||||
if only_syntax:
|
||||
return
|
||||
self._validate_config_platforms(self.config)
|
||||
self._validate_config_semantic(self.config)
|
||||
except Exception as e:
|
||||
exception_info = json.dumps(traceback.format_exc(), indent=2,
|
||||
separators=(",", ": "))
|
||||
self.task.set_failed(type(e).__name__,
|
||||
str(e), exception_info)
|
||||
if logging.is_debug():
|
||||
if (logging.is_debug() and
|
||||
not isinstance(e, exceptions.InvalidTaskConfig)):
|
||||
LOG.exception(e)
|
||||
raise exceptions.InvalidTaskException(str(e))
|
||||
|
||||
@ -782,5 +794,5 @@ class Workload(object):
|
||||
def make_exception_args(self, reason):
|
||||
return {"name": self.name,
|
||||
"pos": self.pos,
|
||||
"config": self.to_dict(),
|
||||
"config": json.dumps(self.to_dict()),
|
||||
"reason": reason}
|
||||
|
@ -39,6 +39,8 @@ ValidationResult = validation.ValidationResult
|
||||
add = validation.add
|
||||
|
||||
|
||||
@validation.add("required_platform", platform="openstack", admin=True,
|
||||
users=True)
|
||||
@validation.configure(name="old_validator", namespace="openstack")
|
||||
class OldValidator(validation.Validator):
|
||||
"""Legacy validator for OpenStack scenarios"""
|
||||
|
@ -118,7 +118,12 @@ class RequiredPlatformValidatorTestCase(test.TestCase):
|
||||
{"kwargs": {"platform": "foo", "admin": True},
|
||||
"credentials": {"foo": {"admin": "fake_admin"}}},
|
||||
{"kwargs": {"platform": "foo", "admin": True, "users": True},
|
||||
"credentials": {"foo": {"admin": "fake_admin"}},
|
||||
"credentials": {"foo": {"admin": "fake_admin"}}},
|
||||
{"kwargs": {"platform": "foo", "admin": True, "users": True},
|
||||
"credentials": {"foo": {"users": ["fake_user"]}},
|
||||
"error_msg": "No admin credential for foo"},
|
||||
{"kwargs": {"platform": "foo", "users": True},
|
||||
"credentials": {"foo": {}},
|
||||
"error_msg": "No user credentials for foo"},
|
||||
{"kwargs": {"platform": "foo", "admin": True, "users": True},
|
||||
"credentials": {"foo": {"admin": "fake_admin",
|
||||
|
@ -40,10 +40,7 @@ class TaskSampleTestCase(test.TestCase):
|
||||
if os.environ.get("TOX_ENV_NAME") == "cover":
|
||||
self.skipTest("There is no need to check samples in coverage job.")
|
||||
|
||||
@mock.patch("rally.task.engine.TaskEngine"
|
||||
"._validate_config_semantic")
|
||||
def test_schema_is_valid(self,
|
||||
mock_task_engine__validate_config_semantic):
|
||||
def test_schema_is_valid(self):
|
||||
scenarios = set()
|
||||
|
||||
for dirname, dirnames, filenames in os.walk(self.samples_path):
|
||||
@ -61,7 +58,7 @@ class TaskSampleTestCase(test.TestCase):
|
||||
(task_file.read()))
|
||||
eng = engine.TaskEngine(task_config,
|
||||
mock.MagicMock(), mock.Mock())
|
||||
eng.validate()
|
||||
eng.validate(only_syntax=True)
|
||||
except Exception:
|
||||
print(traceback.format_exc())
|
||||
self.fail("Invalid task file: %s" % full_path)
|
||||
|
@ -29,10 +29,7 @@ class RallyJobsTestCase(test.TestCase):
|
||||
rally_jobs_path = os.path.join(
|
||||
os.path.dirname(rally.__file__), "..", "rally-jobs")
|
||||
|
||||
@mock.patch("rally.task.engine.TaskEngine"
|
||||
"._validate_config_semantic")
|
||||
def test_schema_is_valid(
|
||||
self, mock_task_engine__validate_config_semantic):
|
||||
def test_schema_is_valid(self):
|
||||
discover.load_plugins(os.path.join(self.rally_jobs_path, "plugins"))
|
||||
|
||||
files = {f for f in os.listdir(self.rally_jobs_path)
|
||||
@ -64,7 +61,7 @@ class RallyJobsTestCase(test.TestCase):
|
||||
|
||||
eng = engine.TaskEngine(task, mock.MagicMock(),
|
||||
mock.Mock())
|
||||
eng.validate()
|
||||
eng.validate(only_syntax=True)
|
||||
except Exception:
|
||||
print(traceback.format_exc())
|
||||
self.fail("Wrong task input file: %s" % full_path)
|
||||
|
@ -16,6 +16,7 @@
|
||||
"""Tests for the Test engine."""
|
||||
|
||||
import collections
|
||||
import json
|
||||
import threading
|
||||
|
||||
import mock
|
||||
@ -62,18 +63,15 @@ class TaskEngineTestCase(test.TestCase):
|
||||
mock.Mock())
|
||||
mock_validate = mock.MagicMock()
|
||||
|
||||
eng._validate_config_scenarios_name = mock_validate.names
|
||||
eng._validate_config_syntax = mock_validate.syntax
|
||||
eng._validate_config_platforms = mock_validate.platforms
|
||||
eng._validate_config_semantic = mock_validate.semantic
|
||||
|
||||
eng.validate()
|
||||
|
||||
expected_calls = [
|
||||
mock.call.names(config),
|
||||
mock.call.syntax(config),
|
||||
mock.call.semantic(config)
|
||||
]
|
||||
mock_validate.assert_has_calls(expected_calls)
|
||||
mock_validate.syntax.assert_called_once_with(config)
|
||||
mock_validate.platforms.assert_called_once_with(config)
|
||||
mock_validate.semantic.assert_called_once_with(config)
|
||||
|
||||
def test_validate__wrong_schema(self):
|
||||
config = {
|
||||
@ -84,85 +82,36 @@ class TaskEngineTestCase(test.TestCase):
|
||||
engine.TaskEngine, config, task, mock.Mock())
|
||||
self.assertTrue(task.set_failed.called)
|
||||
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
def test_validate__wrong_scenarios_name(self, mock_task_config):
|
||||
task = mock.MagicMock()
|
||||
eng = engine.TaskEngine(mock.MagicMock(), task, mock.Mock())
|
||||
eng._validate_config_scenarios_name = mock.MagicMock(
|
||||
side_effect=exceptions.NotFoundScenarios)
|
||||
|
||||
self.assertRaises(exceptions.InvalidTaskException, eng.validate)
|
||||
self.assertTrue(task.set_failed.called)
|
||||
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
def test_validate__wrong_syntax(self, mock_task_config):
|
||||
task = mock.MagicMock()
|
||||
eng = engine.TaskEngine(mock.MagicMock(), task, mock.Mock())
|
||||
eng._validate_config_scenarios_name = mock.MagicMock()
|
||||
eng._validate_config_syntax = mock.MagicMock(
|
||||
side_effect=exceptions.InvalidTaskConfig)
|
||||
eng._validate_config_platforms = mock.Mock()
|
||||
|
||||
self.assertRaises(exceptions.InvalidTaskException, eng.validate)
|
||||
|
||||
self.assertTrue(task.set_failed.called)
|
||||
# the next validation step should not be processed
|
||||
self.assertFalse(eng._validate_config_platforms.called)
|
||||
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
def test_validate__wrong_semantic(self, mock_task_config):
|
||||
task = mock.MagicMock()
|
||||
eng = engine.TaskEngine(mock.MagicMock(), task, mock.Mock())
|
||||
eng._validate_config_scenarios_name = mock.MagicMock()
|
||||
eng._validate_config_syntax = mock.MagicMock()
|
||||
eng._validate_config_platforms = mock.MagicMock()
|
||||
eng._validate_config_semantic = mock.MagicMock(
|
||||
side_effect=exceptions.InvalidTaskConfig)
|
||||
|
||||
self.assertRaises(exceptions.InvalidTaskException, eng.validate)
|
||||
self.assertTrue(task.set_failed.called)
|
||||
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
@mock.patch("rally.task.engine.scenario.Scenario.get_all")
|
||||
def test__validate_config_scenarios_name(
|
||||
self, mock_scenario_get_all, mock_task_config):
|
||||
|
||||
mock_task_instance = mock.MagicMock()
|
||||
mock_subtask = mock.MagicMock()
|
||||
mock_subtask.workloads = [
|
||||
engine.Workload({"name": "a"}, 0),
|
||||
engine.Workload({"name": "b"}, 1)
|
||||
]
|
||||
mock_task_instance.subtasks = [mock_subtask]
|
||||
|
||||
mock_scenario_get_all.return_value = [
|
||||
mock.MagicMock(get_name=lambda: "e"),
|
||||
mock.MagicMock(get_name=lambda: "b"),
|
||||
mock.MagicMock(get_name=lambda: "a")
|
||||
]
|
||||
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
|
||||
mock.Mock())
|
||||
eng._validate_config_scenarios_name(mock_task_instance)
|
||||
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
@mock.patch("rally.task.engine.scenario.Scenario")
|
||||
def test__validate_config_scenarios_name_non_exsisting(
|
||||
self, mock_scenario, mock_task_config):
|
||||
|
||||
mock_task_instance = mock.MagicMock()
|
||||
mock_subtask = mock.MagicMock()
|
||||
mock_subtask.workloads = [
|
||||
engine.Workload({"name": "exist"}, 0),
|
||||
engine.Workload({"name": "nonexist1"}, 1),
|
||||
engine.Workload({"name": "nonexist2"}, 2)
|
||||
]
|
||||
mock_task_instance.subtasks = [mock_subtask]
|
||||
mock_scenario.get_all.return_value = [
|
||||
mock.Mock(get_name=lambda: "exist"),
|
||||
mock.Mock(get_name=lambda: "aaa")]
|
||||
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
|
||||
mock.Mock())
|
||||
|
||||
exc = self.assertRaises(exceptions.NotFoundScenarios,
|
||||
eng._validate_config_scenarios_name,
|
||||
mock_task_instance)
|
||||
self.assertEqual("There are no benchmark scenarios with names: "
|
||||
"`nonexist2, nonexist1`.", str(exc))
|
||||
# all steps of validation are called, which means that the last one is
|
||||
# failed
|
||||
self.assertTrue(eng._validate_config_syntax)
|
||||
self.assertTrue(eng._validate_config_platforms)
|
||||
self.assertTrue(eng._validate_config_semantic)
|
||||
|
||||
@mock.patch("rally.task.engine.scenario.Scenario.get")
|
||||
@mock.patch("rally.task.sla.SLA.validate")
|
||||
@ -171,15 +120,15 @@ class TaskEngineTestCase(test.TestCase):
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
@mock.patch("rally.task.engine.runner.ScenarioRunner.validate")
|
||||
@mock.patch("rally.task.engine.context.Context.validate")
|
||||
def test__validate_config_syntax(
|
||||
def test__validate_workload(
|
||||
self, mock_context_validate,
|
||||
mock_scenario_runner_validate,
|
||||
mock_task_config,
|
||||
mock_hook_validate,
|
||||
mock_trigger_validate,
|
||||
mock_sla_validate,
|
||||
mock_scenario_get
|
||||
):
|
||||
mock_scenario_get):
|
||||
|
||||
mock_context_validate.return_value = []
|
||||
mock_sla_validate.return_value = []
|
||||
mock_hook_validate.return_value = []
|
||||
@ -188,87 +137,81 @@ class TaskEngineTestCase(test.TestCase):
|
||||
scenario_cls = mock_scenario_get.return_value
|
||||
scenario_cls.get_namespace.return_value = "default"
|
||||
scenario_cls.get_default_context.return_value = default_context
|
||||
mock_task_instance = mock.MagicMock()
|
||||
mock_subtask = mock.MagicMock()
|
||||
|
||||
scenario_name = "Foo.bar"
|
||||
runner_type = "MegaRunner"
|
||||
hook_conf = {"name": "c",
|
||||
"args": "c_args",
|
||||
"trigger": {"name": "d", "args": "d_args"}}
|
||||
mock_subtask.workloads = [
|
||||
engine.Workload({"name": "sca", "context": {"a": "a_conf"}}, 0),
|
||||
engine.Workload({"name": "sca", "runner": {"type": "b"},
|
||||
"sla": {"foo_sla": "sla_conf"}}, 1),
|
||||
engine.Workload({"name": "sca", "hooks": [hook_conf]}, 2),
|
||||
]
|
||||
mock_task_instance.subtasks = [mock_subtask]
|
||||
workload = engine.Workload({"name": scenario_name,
|
||||
"runner": {"type": runner_type},
|
||||
"context": {"a": "a_conf"},
|
||||
"hooks": [hook_conf],
|
||||
"sla": {"foo_sla": "sla_conf"}}, 2)
|
||||
|
||||
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
|
||||
mock.Mock())
|
||||
eng._validate_config_syntax(mock_task_instance)
|
||||
|
||||
eng._validate_workload(workload)
|
||||
|
||||
mock_scenario_runner_validate.assert_called_once_with(
|
||||
name="b", credentials=None, config=None,
|
||||
plugin_cfg={"type": "b"}, namespace="default")
|
||||
mock_context_validate.assert_has_calls(
|
||||
[mock.call(name="a",
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg="a_conf",
|
||||
namespace="default"),
|
||||
mock.call(name="foo",
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg="foo_conf",
|
||||
namespace="default",
|
||||
allow_hidden=True),
|
||||
mock.call(name="foo",
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg="foo_conf",
|
||||
namespace="default",
|
||||
allow_hidden=True),
|
||||
mock.call(name="foo",
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg="foo_conf",
|
||||
namespace="default",
|
||||
allow_hidden=True)],
|
||||
any_order=True
|
||||
)
|
||||
name=runner_type, credentials=None, config=None,
|
||||
plugin_cfg={"type": runner_type}, namespace="default", vtype=None)
|
||||
self.assertEqual([mock.call(name="a",
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg="a_conf",
|
||||
namespace="default",
|
||||
vtype=None),
|
||||
mock.call(name="foo",
|
||||
credentials=None,
|
||||
config=None,
|
||||
plugin_cfg="foo_conf",
|
||||
namespace="default",
|
||||
allow_hidden=True,
|
||||
vtype=None)],
|
||||
mock_context_validate.call_args_list)
|
||||
mock_sla_validate.assert_called_once_with(
|
||||
config=None, credentials=None,
|
||||
name="foo_sla", plugin_cfg="sla_conf")
|
||||
name="foo_sla", plugin_cfg="sla_conf", vtype=None)
|
||||
mock_hook_validate.assert_called_once_with(
|
||||
config=None, credentials=None, name="c", plugin_cfg="c_args")
|
||||
config=None, credentials=None, name="c", plugin_cfg="c_args",
|
||||
vtype=None)
|
||||
mock_trigger_validate.assert_called_once_with(
|
||||
config=None, credentials=None, name="d", plugin_cfg="d_args")
|
||||
config=None, credentials=None, name="d", plugin_cfg="d_args",
|
||||
vtype=None)
|
||||
|
||||
@mock.patch("rally.task.engine.json.dumps")
|
||||
@mock.patch("rally.task.engine.scenario.Scenario.get")
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
@mock.patch("rally.task.engine.runner.ScenarioRunner.validate")
|
||||
def test__validate_config_syntax__wrong_runner(
|
||||
self, mock_scenario_runner_validate,
|
||||
mock_task_config, mock_scenario_get):
|
||||
result = validation.ValidationResult(False, "context_error")
|
||||
def test___validate_workload__wrong_runner(
|
||||
self, mock_scenario_runner_validate, mock_task_config,
|
||||
mock_scenario_get, mock_dumps):
|
||||
mock_dumps.return_value = "<JSON>"
|
||||
result = validation.ValidationResult(False, "There is no such runner")
|
||||
mock_scenario_runner_validate.return_value = [result]
|
||||
scenario_cls = mock_scenario_get.return_value
|
||||
scenario_cls.get_default_context.return_value = {}
|
||||
mock_task_instance = mock.MagicMock()
|
||||
mock_subtask = mock.MagicMock()
|
||||
mock_subtask.workloads = [
|
||||
engine.Workload({"name": "sca"}, 0),
|
||||
engine.Workload({"name": "sca", "runner": {"type": "b"}}, 1)
|
||||
]
|
||||
mock_task_instance.subtasks = [mock_subtask]
|
||||
workload = engine.Workload({"name": "sca", "runner": {"type": "b"}}, 0)
|
||||
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
|
||||
mock.Mock())
|
||||
|
||||
self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_config_syntax, mock_task_instance)
|
||||
e = self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_workload, workload)
|
||||
self.assertEqual("Input task is invalid!\n\nSubtask sca[0] has wrong "
|
||||
"configuration\nSubtask configuration:\n"
|
||||
"<JSON>\n\nReason(s):\n"
|
||||
" There is no such runner", e.format_message())
|
||||
|
||||
@mock.patch("rally.task.engine.json.dumps")
|
||||
@mock.patch("rally.task.engine.scenario.Scenario.get")
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
@mock.patch("rally.task.engine.context.Context.validate")
|
||||
def test__validate_config_syntax__wrong_context(
|
||||
self, mock_context_validate,
|
||||
mock_task_config, mock_scenario_get):
|
||||
self, mock_context_validate, mock_task_config, mock_scenario_get,
|
||||
mock_dumps):
|
||||
mock_dumps.return_value = "<JSON>"
|
||||
result = validation.ValidationResult(False, "context_error")
|
||||
mock_context_validate.return_value = [result]
|
||||
scenario_cls = mock_scenario_get.return_value
|
||||
@ -276,21 +219,27 @@ class TaskEngineTestCase(test.TestCase):
|
||||
mock_task_instance = mock.MagicMock()
|
||||
mock_subtask = mock.MagicMock()
|
||||
mock_subtask.workloads = [
|
||||
engine.Workload({"name": "sca", "context": {"a": "a_conf"}}, 0),
|
||||
engine.Workload({"name": "sca"}, 1)
|
||||
engine.Workload({"name": "sca"}, 0),
|
||||
engine.Workload({"name": "sca", "context": {"a": "a_conf"}}, 1)
|
||||
]
|
||||
mock_task_instance.subtasks = [mock_subtask]
|
||||
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
|
||||
mock.Mock())
|
||||
|
||||
self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_config_syntax, mock_task_instance)
|
||||
e = self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_config_syntax, mock_task_instance)
|
||||
self.assertEqual("Input task is invalid!\n\nSubtask sca[1] has wrong "
|
||||
"configuration\nSubtask configuration:\n<JSON>\n\n"
|
||||
"Reason(s):\n context_error", e.format_message())
|
||||
|
||||
@mock.patch("rally.task.engine.json.dumps")
|
||||
@mock.patch("rally.task.engine.scenario.Scenario.get")
|
||||
@mock.patch("rally.task.sla.SLA.validate")
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
def test__validate_config_syntax__wrong_sla(
|
||||
self, mock_task_config, mock_sla_validate, mock_scenario_get):
|
||||
self, mock_task_config, mock_sla_validate, mock_scenario_get,
|
||||
mock_dumps):
|
||||
mock_dumps.return_value = "<JSON>"
|
||||
result = validation.ValidationResult(False, "sla_error")
|
||||
mock_sla_validate.return_value = [result]
|
||||
scenario_cls = mock_scenario_get.return_value
|
||||
@ -304,14 +253,23 @@ class TaskEngineTestCase(test.TestCase):
|
||||
mock_task_instance.subtasks = [mock_subtask]
|
||||
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
|
||||
mock.Mock())
|
||||
self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_config_syntax, mock_task_instance)
|
||||
e = self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_config_syntax, mock_task_instance)
|
||||
self.assertEqual("Input task is invalid!\n\n"
|
||||
"Subtask sca[1] has wrong configuration\n"
|
||||
"Subtask configuration:\n<JSON>\n\n"
|
||||
"Reason(s):\n sla_error", e.format_message())
|
||||
|
||||
@mock.patch("rally.task.engine.json.dumps")
|
||||
@mock.patch("rally.task.engine.scenario.Scenario.get")
|
||||
@mock.patch("rally.task.hook.Hook.validate")
|
||||
@mock.patch("rally.task.trigger.Trigger.validate")
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
def test__validate_config_syntax__wrong_hook(
|
||||
self, mock_task_config, mock_hook_validate, mock_scenario_get):
|
||||
self, mock_task_config, mock_trigger_validate, mock_hook_validate,
|
||||
mock_scenario_get, mock_dumps):
|
||||
mock_dumps.return_value = "<JSON>"
|
||||
mock_trigger_validate.return_value = []
|
||||
result = validation.ValidationResult(False, "hook_error")
|
||||
mock_hook_validate.return_value = [result]
|
||||
scenario_cls = mock_scenario_get.return_value
|
||||
@ -329,16 +287,23 @@ class TaskEngineTestCase(test.TestCase):
|
||||
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
|
||||
mock.Mock())
|
||||
|
||||
self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_config_syntax, mock_task_instance)
|
||||
e = self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_config_syntax, mock_task_instance)
|
||||
|
||||
self.assertEqual("Input task is invalid!\n\n"
|
||||
"Subtask sca[1] has wrong configuration\n"
|
||||
"Subtask configuration:\n<JSON>\n\n"
|
||||
"Reason(s):\n hook_error", e.format_message())
|
||||
|
||||
@mock.patch("rally.task.engine.json.dumps")
|
||||
@mock.patch("rally.task.engine.scenario.Scenario.get")
|
||||
@mock.patch("rally.task.trigger.Trigger.validate")
|
||||
@mock.patch("rally.task.hook.Hook.validate")
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
def test__validate_config_syntax__wrong_trigger(
|
||||
self, mock_task_config, mock_hook_validate, mock_trigger_validate,
|
||||
mock_scenario_get):
|
||||
mock_scenario_get, mock_dumps):
|
||||
mock_dumps.return_value = "<JSON>"
|
||||
result = validation.ValidationResult(False, "trigger_error")
|
||||
mock_trigger_validate.return_value = [result]
|
||||
mock_hook_validate.return_value = []
|
||||
@ -357,42 +322,32 @@ class TaskEngineTestCase(test.TestCase):
|
||||
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
|
||||
mock.Mock())
|
||||
|
||||
self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_config_syntax, mock_task_instance)
|
||||
e = self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_config_syntax, mock_task_instance)
|
||||
|
||||
self.assertEqual("Input task is invalid!\n\n"
|
||||
"Subtask sca[1] has wrong configuration\n"
|
||||
"Subtask configuration:\n<JSON>\n\n"
|
||||
"Reason(s):\n trigger_error", e.format_message())
|
||||
|
||||
@mock.patch("rally.task.engine.scenario.Scenario.validate")
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
def test__validate_config_semantic_helper(self, mock_task_config,
|
||||
mock_scenario_validate):
|
||||
mock_scenario_validate.return_value = []
|
||||
def test__validate_config_semantic_helper(self, mock_task_config):
|
||||
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
|
||||
mock.Mock())
|
||||
eng._validate_workload = mock.Mock()
|
||||
workloads = [engine.Workload(
|
||||
{"name": "name", "runner": "runner", "args": "args"}, 0)]
|
||||
users = [{"foo": "user1"}]
|
||||
user_context = mock.MagicMock()
|
||||
user_context.__enter__.return_value.context = {
|
||||
"users": [{"foo": "user1"}]}
|
||||
user_context.__enter__.return_value.context = {"users": users}
|
||||
|
||||
eng._validate_config_semantic_helper(
|
||||
"admin", user_context, workloads, "foo")
|
||||
mock_scenario_validate.assert_called_once_with(
|
||||
name="name", config={"runner": "runner", "args": "args"},
|
||||
credentials={"foo": {"admin": "admin",
|
||||
"users": [{"foo": "user1"}]}},
|
||||
plugin_cfg=None)
|
||||
|
||||
@mock.patch("rally.task.engine.scenario.Scenario.validate")
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
def test__validate_config_semanitc_helper_invalid_arg(
|
||||
self, mock_task_config, mock_scenario_validate):
|
||||
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
|
||||
mock.Mock())
|
||||
mock_scenario_validate.return_value = [
|
||||
validation.ValidationResult(False, msg="foo")]
|
||||
user_context = mock.MagicMock()
|
||||
workloads = [engine.Workload({"name": "name"}, 0)]
|
||||
self.assertRaises(exceptions.InvalidTaskConfig,
|
||||
eng._validate_config_semantic_helper, "a",
|
||||
user_context, workloads, "foo")
|
||||
eng._validate_workload.assert_called_once_with(
|
||||
workloads[0], credentials={"foo": {"admin": "admin",
|
||||
"users": users}},
|
||||
vtype="semantic")
|
||||
|
||||
@mock.patch("rally.task.engine.scenario.Scenario.get")
|
||||
@mock.patch("rally.task.engine.context.Context")
|
||||
@ -438,6 +393,36 @@ class TaskEngineTestCase(test.TestCase):
|
||||
mock.call(admin, user_context, [wconf2, wconf3], "openstack"),
|
||||
], any_order=True)
|
||||
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
@mock.patch("rally.task.engine.TaskEngine._validate_workload")
|
||||
def test__validate_config_platforms(
|
||||
self, mock__validate_workload, mock_task_config):
|
||||
|
||||
class FakeDeployment(object):
|
||||
def __init__(self, credentials):
|
||||
self._creds = credentials
|
||||
self.get_all_credentials = mock.Mock()
|
||||
self.get_all_credentials.return_value = self._creds
|
||||
|
||||
foo_cred1 = {"admin": "admin", "users": ["user1"]}
|
||||
foo_cred2 = {"admin": "admin", "users": ["user1"]}
|
||||
deployment = FakeDeployment({"foo": [foo_cred1, foo_cred2]})
|
||||
|
||||
workload1 = "workload1"
|
||||
workload2 = "workload2"
|
||||
subtasks = [mock.Mock(workloads=[workload1]),
|
||||
mock.Mock(workloads=[workload2])]
|
||||
config = mock.Mock(subtasks=subtasks)
|
||||
eng = engine.TaskEngine({}, mock.MagicMock(), deployment)
|
||||
|
||||
eng._validate_config_platforms(config)
|
||||
|
||||
self.assertEqual(
|
||||
[mock.call(w, vtype="platform", credentials={"foo": foo_cred1})
|
||||
for w in (workload1, workload2)],
|
||||
mock__validate_workload.call_args_list)
|
||||
deployment.get_all_credentials.assert_called_once_with()
|
||||
|
||||
@mock.patch("rally.common.objects.Task.get_status")
|
||||
@mock.patch("rally.task.engine.TaskConfig")
|
||||
@mock.patch("rally.task.engine.ResultConsumer")
|
||||
@ -1147,13 +1132,8 @@ class WorkloadTestCase(test.TestCase):
|
||||
"name": "n",
|
||||
"pos": 0,
|
||||
"reason": "r",
|
||||
"config": {
|
||||
"runner": "r",
|
||||
"context": "c",
|
||||
"sla": "s",
|
||||
"hooks": "h",
|
||||
"args": "a"
|
||||
}
|
||||
"config": json.dumps({"runner": "r", "context": "c", "sla": "s",
|
||||
"hooks": "h", "args": "a"})
|
||||
}
|
||||
|
||||
self.assertEqual(expected_args,
|
||||
|
Loading…
Reference in New Issue
Block a user