openstack/rally
Code Issues Proposed changes

Merge "[validation] validate scenario default_context"

Browse Source
This commit is contained in:
Jenkins 2017-03-01 12:44:56 +00:00 committed by Gerrit Code Review
commit 4dcb597949
13 changed files with 135 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
rally/common/plugin/plugin.py
View File

@ -59,15 +59,18 @@ def base():
return wrapper
def configure(name, namespace="default"):
def configure(name, namespace="default", hidden=False):
"""Use this decorator to configure plugin's attributes.
:param name: name of plugin that is used for searching purpose
:param namespace: plugin namespace
:param hidden: if True the plugin will be marked as hidden and can be
loaded only explicitly
"""
def decorator(plugin):
plugin._configure(name, namespace)
plugin._meta_set("hidden", hidden)
return plugin
return decorator
@ -204,7 +207,7 @@ class Plugin(meta.MetaMixin, info.InfoMixin):
return cls
@classmethod
def get(cls, name, namespace=None):
def get(cls, name, namespace=None, allow_hidden=False):
"""Return plugin by its name from specified namespace.
This method iterates over all subclasses of cls and returns plugin
@ -215,15 +218,20 @@ class Plugin(meta.MetaMixin, info.InfoMixin):
:param name: Plugin's name
:param namespace: Namespace where to search for plugins
:param allow_hidden: if False and found plugin is hidden then
PluginNotFound will be raised
"""
potential_result = []
for p in cls.get_all(namespace=namespace):
for p in cls.get_all(namespace=namespace, allow_hidden=True):
if p.get_name() == name:
potential_result.append(p)
if len(potential_result) == 1:
return potential_result[0]
plugin = potential_result[0]
if allow_hidden or not plugin.is_hidden():
return plugin
elif potential_result:
hint = _LE("Try to choose the correct Plugin base or namespace to "
"search in.")
@ -240,18 +248,26 @@ class Plugin(meta.MetaMixin, info.InfoMixin):
name=name, namespace=namespace or "any of")
@classmethod
def get_all(cls, namespace=None):
def get_all(cls, namespace=None, allow_hidden=False):
"""Return all subclass plugins of plugin.
All plugins that are not configured will be ignored.
:param namespace: return only plugins from specified namespace.
:param allow_hidden: if False return only non hidden plugins
"""
plugins = []
for p in discover.itersubclasses(cls):
if issubclass(p, Plugin) and p._meta_is_inited(raise_exc=False):
if not namespace or namespace == p.get_namespace():
if not issubclass(p, Plugin):
continue
if not p._meta_is_inited(raise_exc=False):
continue
if namespace and namespace != p.get_namespace():
continue
if not allow_hidden and p.is_hidden():
continue
plugins.append(getattr(p, "func_ref", p))
return plugins
@ -266,6 +282,11 @@ class Plugin(meta.MetaMixin, info.InfoMixin):
""""Return namespace of plugin, e.g. default or openstack."""
return cls._meta_get("namespace")
@classmethod
def is_hidden(cls):
"""Return True if plugin is hidden."""
return cls._meta_get("hidden", False)
@classmethod
def is_deprecated(cls):
"""Return deprecation details for deprecated plugins."""

5
rally/plugins/openstack/context/api_versions.py
View File

@ -221,9 +221,8 @@ class OpenStackAPIVersions(context.Context):
pass
@classmethod
def validate(cls, config, non_hidden=False):
super(OpenStackAPIVersions, cls).validate(config,
non_hidden=non_hidden)
def validate(cls, config):
super(OpenStackAPIVersions, cls).validate(config)
for client in config:
client_cls = osclients.OSClient.get(client)
if ("service_type" in config[client] and

4
rally/plugins/openstack/context/cleanup/admin.py
View File

@ -31,8 +31,8 @@ class AdminCleanup(base.CleanupMixin, context.Context):
"""Context class for admin resources cleanup."""
@classmethod
def validate(cls, config, non_hidden=False):
super(AdminCleanup, cls).validate(config, non_hidden)
def validate(cls, config):
super(AdminCleanup, cls).validate(config)
missing = set(config)
missing -= manager.list_resource_names(admin_required=True)

4
rally/plugins/openstack/context/cleanup/user.py
View File

@ -31,8 +31,8 @@ class UserCleanup(base.CleanupMixin, context.Context):
"""Context class for user resources cleanup."""
@classmethod
def validate(cls, config, non_hidden=False):
super(UserCleanup, cls).validate(config, non_hidden)
def validate(cls, config):
super(UserCleanup, cls).validate(config)
missing = set(config)
missing -= manager.list_resource_names(admin_required=False)

20
rally/task/context.py
View File

@ -21,7 +21,6 @@ import six
from rally.common import logging
from rally.common.plugin import plugin
from rally.common import utils
from rally import exceptions
from rally.task import functional
LOG = logging.getLogger(__name__)
@ -42,9 +41,8 @@ def configure(name, order, hidden=False):
task config
"""
def wrapper(cls):
cls = plugin.configure(name=name)(cls)
cls = plugin.configure(name=name, hidden=hidden)(cls)
cls._meta_set("order", order)
cls._meta_set("hidden", hidden)
return cls
return wrapper
@ -102,12 +100,7 @@ class BaseContext(plugin.Plugin, functional.FunctionalMixin,
return not self.__eq__(other)
@classmethod
def validate(cls, config, non_hidden=False):
# TODO(boris-42): This is going to be replaced with common validation
# mechanism (generalization of scenario validation)
if non_hidden and cls._meta_get("hidden"):
raise exceptions.PluginNotFound(name=cls.get_name(),
namespace="context")
def validate(cls, config):
jsonschema.validate(config, cls.CONFIG_SCHEMA)
@classmethod
@ -161,13 +154,14 @@ class ContextManager(object):
self.context_obj = context_obj
@staticmethod
def validate(ctx, non_hidden=False):
def validate(ctx, allow_hidden=False):
for name, config in ctx.items():
Context.get(name).validate(config, non_hidden=non_hidden)
Context.get(name, allow_hidden=allow_hidden).validate(config)
def _get_sorted_context_lst(self):
ctxlst = map(Context.get, self.context_obj["config"])
return sorted(map(lambda ctx: ctx(self.context_obj), ctxlst))
return sorted([
Context.get(ctx_name, allow_hidden=True)(self.context_obj)
for ctx_name in self.context_obj["config"].keys()])
def setup(self):
"""Creates benchmark environment from config."""

14
rally/task/engine.py
View File

@ -284,10 +284,12 @@ class TaskEngine(object):
def _validate_config_syntax(self, config):
for subtask in config.subtasks:
for pos, workload in enumerate(subtask.workloads):
scenario_context = self._get_defualt_context(workload.name)
try:
runner.ScenarioRunner.validate(workload.runner)
context.ContextManager.validate(
workload.context, non_hidden=True)
context.ContextManager.validate(workload.context)
context.ContextManager.validate(scenario_context,
allow_hidden=True)
sla.SLA.validate(workload.sla)
for hook_conf in workload.hooks:
hook.Hook.validate(hook_conf)
@ -362,9 +364,13 @@ class TaskEngine(object):
config = config or {"type": "serial"}
return runner.ScenarioRunner.get(config["type"])(self.task, config)
def _prepare_context(self, ctx, name, credential):
scenario_context = copy.deepcopy(
@staticmethod
def _get_defualt_context(name):
return copy.deepcopy(
scenario.Scenario.get(name)._meta_get("default_context"))
def _prepare_context(self, ctx, name, credential):
scenario_context = self._get_defualt_context(name)
if self.existing_users and "users" not in ctx:
scenario_context.setdefault("existing_users", self.existing_users)
elif "users" not in ctx:

21
rally/verification/context.py
View File

@ -12,10 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import functools
from rally.common.plugin import plugin
from rally.task import context
configure = context.configure
# all VerifierContexts should be always hidden
configure = functools.partial(context.configure, hidden=True)
@plugin.base()
@ -28,14 +31,7 @@ class VerifierContext(context.BaseContext):
self.verifier = self.context["verifier"]
@classmethod
def _meta_get(cls, key, default=None):
# It should be always hidden
if key == "hidden":
return True
return super(VerifierContext, cls)._meta_get(key, default)
@classmethod
def validate(cls, config, non_hidden=False):
def validate(cls, config):
# do not validate jsonschema.
pass
@ -43,10 +39,11 @@ class VerifierContext(context.BaseContext):
class ContextManager(context.ContextManager):
@staticmethod
def validate(ctx, non_hidden=False):
def validate(ctx):
for name, config in ctx.items():
VerifierContext.get(name).validate(config, non_hidden=non_hidden)
VerifierContext.get(name, allow_hidden=True).validate(config)
def _get_sorted_context_lst(self):
return sorted([VerifierContext.get(name)(self.context_obj)
return sorted([
VerifierContext.get(name, allow_hidden=True)(self.context_obj)
for name in self.context_obj["config"].keys()])

19
tests/unit/common/plugin/test_plugin.py
View File

@ -41,6 +41,7 @@ class PluginModuleTestCase(test.TestCase):
return a
self.assertEqual("configure_func_plugin_test", func.get_name())
self.assertFalse(func.is_hidden())
self.assertEqual(42, func(42))
def test_deprecated_cls(self):
@ -148,6 +149,11 @@ class SomePlugin(BasePlugin):
pass
@plugin.configure(name="test_hidden_plugin", hidden=True)
class HiddenPlugin(BasePlugin):
pass
@plugin.deprecated("some_reason", "0.1.1")
@plugin.configure(name="test_deprecated_plugin")
class DeprecatedPlugin(BasePlugin):
@ -174,6 +180,15 @@ class PluginTestCase(test.TestCase):
self.assertEqual(SomePlugin,
BasePlugin.get("test_some_plugin"))
def test_get_hidden(self):
self.assertEqual(HiddenPlugin,
BasePlugin.get("test_hidden_plugin",
allow_hidden=True))
def test_get_hidden_not_found(self):
self.assertRaises(exceptions.PluginNotFound,
BasePlugin.get, "test_hidden_plugin")
def test_get_not_found(self):
self.assertRaises(exceptions.PluginNotFound,
BasePlugin.get, "non_existing")
@ -198,6 +213,10 @@ class PluginTestCase(test.TestCase):
set(BasePlugin.get_all()))
self.assertEqual([], SomePlugin.get_all())
def test_get_all_hidden(self):
self.assertEqual(set([SomePlugin, DeprecatedPlugin, HiddenPlugin]),
set(BasePlugin.get_all(allow_hidden=True)))
def test_is_deprecated(self):
self.assertFalse(SomePlugin.is_deprecated())
self.assertEqual(DeprecatedPlugin.is_deprecated(),

30
tests/unit/plugins/openstack/context/test_api_versions.py
View File

@ -40,44 +40,44 @@ class OpenStackServicesTestCase(test.TestCase):
})
def test_validate_wrong_configs(self):
# Non-existing clients should be caught
self.assertRaises(
exceptions.PluginNotFound,
api_versions.OpenStackAPIVersions.validate,
{"invalid": {"service_type": "some_type"}},
"Non-existing clients should be caught.")
{"invalid": {"service_type": "some_type"}})
# Additional properties should be restricted
self.assertRaises(
jsonschema.ValidationError,
api_versions.OpenStackAPIVersions.validate,
{"nova": {"some_key": "some_value"}},
"Additional properties should be restricted.")
{"nova": {"some_key": "some_value"}})
# Setting service_type is allowed only
# for those clients, which support it
self.assertRaises(
exceptions.ValidationError,
api_versions.OpenStackAPIVersions.validate,
{"keystone": {"service_type": "identity"}},
"Setting service_type is allowed only for those clients, which "
"support it.")
{"keystone": {"service_type": "identity"}})
# Setting service_name is allowed only
# for those clients, which support it
self.assertRaises(
exceptions.ValidationError,
api_versions.OpenStackAPIVersions.validate,
{"keystone": {"service_name": "keystone"}},
"Setting service_name is allowed only for those clients, which "
"support it.")
{"keystone": {"service_name": "keystone"}})
# Setting version is allowed only
# for those clients, which support it
self.assertRaises(
exceptions.ValidationError,
api_versions.OpenStackAPIVersions.validate,
{"keystone": {"version": 1}},
"Setting version is allowed only for those clients, which "
"support it.")
{"keystone": {"version": 1}})
# Unsupported version should be caught
self.assertRaises(
exceptions.ValidationError,
api_versions.OpenStackAPIVersions.validate,
{"nova": {"version": 666}},
"Unsupported version should be caught.")
{"nova": {"version": 666}})
def test_setup_with_wrong_service_name(self):
context = {

27
tests/unit/task/test_context.py
View File

@ -69,12 +69,6 @@ class BaseContextTestCase(test.TestCase):
self.assertRaises(jsonschema.ValidationError,
fakes.FakeContext.validate, {"nonexisting": 2})
def test_validate__hidden(self):
fakes.FakeHiddenContext.validate({"test": 2})
self.assertRaises(exceptions.PluginNotFound,
fakes.FakeHiddenContext.validate,
{"test": 2}, non_hidden=True)
def test_setup_is_abstract(self):
@context.configure("test_abstract_setup", 0)
@ -154,22 +148,22 @@ class ContextManagerTestCase(test.TestCase):
context.ContextManager.validate(config)
for ctx in ("ctx1", "ctx2"):
mock_context_get.assert_has_calls([
mock.call(ctx),
mock.call().validate(config[ctx], non_hidden=False),
mock.call(ctx, allow_hidden=False),
mock.call().validate(config[ctx]),
])
@mock.patch("rally.task.context.Context.get")
def test_validate_non_hidden(self, mock_context_get):
def test_validate_hidden(self, mock_context_get):
config = {
"ctx1": mock.MagicMock(),
"ctx2": mock.MagicMock()
}
context.ContextManager.validate(config, non_hidden=True)
context.ContextManager.validate(config, allow_hidden=True)
for ctx in ("ctx1", "ctx2"):
mock_context_get.assert_has_calls([
mock.call(ctx),
mock.call().validate(config[ctx], non_hidden=True),
mock.call(ctx, allow_hidden=True),
mock.call().validate(config[ctx]),
])
def test_validate__non_existing_context(self):
@ -191,7 +185,8 @@ class ContextManagerTestCase(test.TestCase):
self.assertEqual(result, ctx_object)
mock_context_get.assert_has_calls(
[mock.call("a"), mock.call("b")], any_order=True)
[mock.call("a", allow_hidden=True),
mock.call("b", allow_hidden=True)], any_order=True)
mock_context.assert_has_calls(
[mock.call(ctx_object), mock.call(ctx_object)], any_order=True)
self.assertEqual([mock_context(), mock_context()], manager._visited)
@ -208,7 +203,8 @@ class ContextManagerTestCase(test.TestCase):
manager = context.ContextManager(ctx_object)
manager.cleanup()
mock_context_get.assert_has_calls(
[mock.call("a"), mock.call("b")], any_order=True)
[mock.call("a", allow_hidden=True),
mock.call("b", allow_hidden=True)], any_order=True)
mock_context.assert_has_calls(
[mock.call(ctx_object), mock.call(ctx_object)], any_order=True)
mock_context.return_value.assert_has_calls(
@ -225,7 +221,8 @@ class ContextManagerTestCase(test.TestCase):
manager.cleanup()
mock_context_get.assert_has_calls(
[mock.call("a"), mock.call("b")], any_order=True)
[mock.call("a", allow_hidden=True),
mock.call("b", allow_hidden=True)], any_order=True)
mock_context.assert_has_calls(
[mock.call(ctx_object), mock.call(ctx_object)], any_order=True)
mock_context.return_value.assert_has_calls(

24
tests/unit/task/test_engine.py
View File

@ -157,6 +157,7 @@ class TaskEngineTestCase(test.TestCase):
eng._validate_config_scenarios_name,
mock_task_instance)
@mock.patch("rally.task.engine.scenario.Scenario.get")
@mock.patch("rally.task.hook.Hook.validate")
@mock.patch("rally.task.engine.TaskConfig")
@mock.patch("rally.task.engine.runner.ScenarioRunner.validate")
@ -165,8 +166,11 @@ class TaskEngineTestCase(test.TestCase):
self, mock_context_manager_validate,
mock_scenario_runner_validate,
mock_task_config,
mock_hook_validate
mock_hook_validate,
mock_scenario_get
):
default_context = {"foo": 1}
mock_scenario_get.return_value._meta_get.return_value = default_context
mock_task_instance = mock.MagicMock()
mock_subtask = mock.MagicMock()
mock_subtask.workloads = [
@ -180,16 +184,24 @@ class TaskEngineTestCase(test.TestCase):
mock_scenario_runner_validate.assert_has_calls(
[mock.call({}), mock.call("b")], any_order=True)
mock_context_manager_validate.assert_has_calls(
[mock.call("a", non_hidden=True), mock.call({}, non_hidden=True)],
any_order=True)
[mock.call("a"),
mock.call(default_context, allow_hidden=True),
mock.call({}),
mock.call(default_context, allow_hidden=True),
mock.call({}),
mock.call(default_context, allow_hidden=True)],
any_order=True
)
mock_hook_validate.assert_called_once_with("c")
@mock.patch("rally.task.engine.scenario.Scenario.get")
@mock.patch("rally.task.engine.TaskConfig")
@mock.patch("rally.task.engine.runner.ScenarioRunner")
@mock.patch("rally.task.engine.context.ContextManager.validate")
def test__validate_config_syntax__wrong_runner(
self, mock_context_manager_validate,
mock_scenario_runner, mock_task_config):
mock_scenario_runner, mock_task_config, mock_scenario_get):
mock_scenario_get.return_value._meta_get.return_value = {}
mock_task_instance = mock.MagicMock()
mock_subtask = mock.MagicMock()
mock_subtask.workloads = [
@ -204,12 +216,14 @@ class TaskEngineTestCase(test.TestCase):
self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_syntax, mock_task_instance)
@mock.patch("rally.task.engine.scenario.Scenario.get")
@mock.patch("rally.task.engine.TaskConfig")
@mock.patch("rally.task.engine.runner.ScenarioRunner.validate")
@mock.patch("rally.task.engine.context.ContextManager")
def test__validate_config_syntax__wrong_context(
self, mock_context_manager, mock_scenario_runner_validate,
mock_task_config):
mock_task_config, mock_scenario_get):
mock_scenario_get.return_value._meta_get.return_value = {}
mock_task_instance = mock.MagicMock()
mock_subtask = mock.MagicMock()
mock_subtask.workloads = [

3
tests/unit/task/test_scenario.py
View File

@ -263,7 +263,8 @@ class ScenarioTestCase(test.TestCase):
def test_scenario_context_are_valid(self):
for s in scenario.Scenario.get_all():
try:
context.ContextManager.validate(s._meta_get("default_context"))
context.ContextManager.validate(s._meta_get("default_context"),
allow_hidden=True)
except Exception:
print(traceback.format_exc())
self.fail("Scenario `%s` has wrong context" % scenario)

11
tests/unit/verification/test_context.py
View File

@ -28,17 +28,15 @@ class FakeContext(context.VerifierContext):
class VerifierContextTestCase(test.TestCase):
def test__meta_get(self):
data = {"key1": "value1", "key2": "value2", "hidden": False}
data = {"key1": "value1", "key2": "value2"}
for k, v in data.items():
FakeContext._meta_set(k, v)
for k, v in data.items():
if k != "hidden":
self.assertEqual(v, FakeContext._meta_get(k))
self.assertTrue(FakeContext._meta_get("hidden"))
self.assertNotEqual(data["hidden"], FakeContext._meta_get("hidden"))
self.assertTrue(FakeContext.is_hidden())
class ContextManagerTestCase(test.TestCase):
@ -48,8 +46,9 @@ class ContextManagerTestCase(test.TestCase):
context.ContextManager.validate(config)
self.assertEqual([mock.call(k) for k, v in config.items()],
self.assertEqual([mock.call(k, allow_hidden=True)
for k, v in config.items()],
mock_verifier_context.get.call_args_list)
self.assertEqual(
[mock.call(v, non_hidden=False) for k, v in config.items()],
[mock.call(v) for k, v in config.items()],
mock_verifier_context.get.return_value.validate.call_args_list)