From 2aadcf4ebe2ced9c06b1768f036b436dcf0e8fc1 Mon Sep 17 00:00:00 2001 From: Boris Pavlovic Date: Fri, 12 Jun 2015 20:38:55 +0300 Subject: [PATCH] Add rally.common.plugin.discover module Move all related to plugins from utils to discover module and fix the code and tests to make it work Change-Id: Ifaed1fadfa0760f121cb1584cb4d8ae3fee8768d --- rally/benchmark/scenarios/base.py | 7 +- rally/cli/cliutils.py | 9 +- rally/cli/commands/info.py | 11 +- rally/common/plugin/discover.py | 102 +++++++++++++++++ rally/common/plugin/plugin.py | 4 +- rally/common/utils.py | 86 --------------- rally/deploy/__init__.py | 5 +- rally/deploy/serverprovider/__init__.py | 20 ---- .../openstack/context/cleanup/manager.py | 5 +- tests/unit/cli/commands/test_info.py | 15 +-- tests/unit/common/plugin/test_discover.py | 103 ++++++++++++++++++ tests/unit/common/test_utils.py | 91 ---------------- .../serverprovider/providers/test_existing.py | 11 +- .../deploy/serverprovider/test_provider.py | 25 ++--- .../openstack/context/cleanup/test_manager.py | 4 +- .../context/cleanup/test_resources.py | 4 +- tests/unit/rally_jobs/test_jobs.py | 4 +- tests/unit/test_docstrings.py | 13 ++- 18 files changed, 264 insertions(+), 255 deletions(-) create mode 100644 rally/common/plugin/discover.py create mode 100644 tests/unit/common/plugin/test_discover.py diff --git a/rally/benchmark/scenarios/base.py b/rally/benchmark/scenarios/base.py index 217f755d2a..491fd2cdd5 100644 --- a/rally/benchmark/scenarios/base.py +++ b/rally/benchmark/scenarios/base.py @@ -22,6 +22,7 @@ import time from rally.benchmark import functional from rally.common import costilius from rally.common import log as logging +from rally.common.plugin import discover from rally.common import utils from rally import consts from rally import exceptions @@ -72,7 +73,7 @@ class Scenario(functional.FunctionalMixin): @staticmethod def get_by_name(name): """Returns Scenario class by name.""" - for scenario in utils.itersubclasses(Scenario): + for scenario in discover.itersubclasses(Scenario): if name == scenario.__name__: return scenario raise exceptions.NoSuchScenario(name=name) @@ -97,7 +98,7 @@ class Scenario(functional.FunctionalMixin): if Scenario.is_scenario(scenario_cls, scenario_name): return getattr(scenario_cls, scenario_name) else: - for scenario_cls in utils.itersubclasses(Scenario): + for scenario_cls in discover.itersubclasses(Scenario): if Scenario.is_scenario(scenario_cls, name): return getattr(scenario_cls, name) raise exceptions.NoSuchScenario(name=name) @@ -112,7 +113,7 @@ class Scenario(functional.FunctionalMixin): :param scenario_cls: the base class for searching scenarios in :returns: List of strings """ - scenario_classes = (list(utils.itersubclasses(scenario_cls)) + + scenario_classes = (list(discover.itersubclasses(scenario_cls)) + [scenario_cls]) benchmark_scenarios = [ ["%s.%s" % (scenario.__name__, func) diff --git a/rally/cli/cliutils.py b/rally/cli/cliutils.py index 28998e7655..6aec97981e 100644 --- a/rally/cli/cliutils.py +++ b/rally/cli/cliutils.py @@ -30,6 +30,7 @@ import six from rally.common.i18n import _ from rally.common import log as logging +from rally.common.plugin import discover from rally.common import utils from rally.common import version from rally import exceptions @@ -518,11 +519,11 @@ def run(argv, categories): return(1) try: - utils.load_plugins("/opt/rally/plugins/") - utils.load_plugins(os.path.expanduser("~/.rally/plugins/")) - utils.import_modules_from_package("rally.plugins") + discover.load_plugins("/opt/rally/plugins/") + discover.load_plugins(os.path.expanduser("~/.rally/plugins/")) + discover.import_modules_from_package("rally.plugins") for path in CONF.plugin_paths or []: - utils.load_plugins(path) + discover.load_plugins(path) validate_deprecated_args(argv, fn) diff --git a/rally/cli/commands/info.py b/rally/cli/commands/info.py index 6acd352bdd..c45bdb423e 100644 --- a/rally/cli/commands/info.py +++ b/rally/cli/commands/info.py @@ -52,9 +52,10 @@ from __future__ import print_function from rally.benchmark.scenarios import base as scenario_base from rally.benchmark import sla from rally.cli import cliutils +from rally.common.plugin import discover from rally.common import utils from rally import deploy -from rally.deploy import serverprovider +from rally.deploy.serverprovider import provider from rally import exceptions @@ -205,7 +206,7 @@ class InfoCommands(object): def ServerProviders(self): """Get information about server providers available in Rally.""" - providers = self._get_descriptions(serverprovider.ProviderFactory) + providers = self._get_descriptions(provider.ProviderFactory) info = (self._make_header("Rally - Server providers") + "\n\n" "Rally is an OpenStack benchmarking system. Before starting " @@ -238,7 +239,7 @@ class InfoCommands(object): def _get_descriptions(self, base_cls, subclass_filter=None): descriptions = [] - subclasses = utils.itersubclasses(base_cls) + subclasses = discover.itersubclasses(base_cls) if subclass_filter: subclasses = filter(subclass_filter, subclasses) for entity in subclasses: @@ -265,7 +266,7 @@ class InfoCommands(object): deploy_engines = [cls.get_name() for cls in deploy.EngineFactory.get_all()] server_providers = [cls.get_name() for cls in - serverprovider.ProviderFactory.get_all()] + provider.ProviderFactory.get_all()] candidates = (scenarios + scenario_groups + scenario_methods + sla_info + deploy_engines + server_providers) @@ -351,7 +352,7 @@ class InfoCommands(object): def _get_server_provider_info(self, query): try: - server_provider = serverprovider.ProviderFactory.get(query) + server_provider = provider.ProviderFactory.get(query) header = "%s (server provider)" % server_provider.get_name() info = self._make_header(header) info += "\n\n" diff --git a/rally/common/plugin/discover.py b/rally/common/plugin/discover.py new file mode 100644 index 0000000000..6f755005e8 --- /dev/null +++ b/rally/common/plugin/discover.py @@ -0,0 +1,102 @@ +# Copyright 2015: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import imp +import os +import sys + +from oslo_utils import importutils + +import rally +from rally.common.i18n import _ +from rally.common import log as logging + +LOG = logging.getLogger(__name__) + + +def itersubclasses(cls, seen=None): + """Generator over all subclasses of a given class in depth first order.""" + + seen = seen or set() + try: + subs = cls.__subclasses__() + except TypeError: # fails only when cls is type + subs = cls.__subclasses__(cls) + for sub in subs: + if sub not in seen: + seen.add(sub) + yield sub + for sub in itersubclasses(sub, seen): + yield sub + + +def import_modules_from_package(package): + """Import modules from package and append into sys.modules + + :param: package - Full package name. For example: rally.deploy.engines + """ + path = [os.path.dirname(rally.__file__), ".."] + package.split(".") + path = os.path.join(*path) + for root, dirs, files in os.walk(path): + for filename in files: + if filename.startswith("__") or not filename.endswith(".py"): + continue + new_package = ".".join(root.split(os.sep)).split("....")[1] + module_name = "%s.%s" % (new_package, filename[:-3]) + if module_name not in sys.modules: + sys.modules[module_name] = importutils.import_module( + module_name) + + +def load_plugins(dir_or_file): + if os.path.isdir(dir_or_file): + directory = dir_or_file + LOG.info(_("Loading plugins from directories %s/*") % directory) + + to_load = [] + for root, dirs, files in os.walk(directory): + to_load.extend((plugin[:-3], root) + for plugin in files if plugin.endswith(".py")) + for plugin, directory in to_load: + if directory not in sys.path: + sys.path.append(directory) + + fullpath = os.path.join(directory, plugin) + try: + fp, pathname, descr = imp.find_module(plugin, [directory]) + imp.load_module(plugin, fp, pathname, descr) + fp.close() + LOG.info(_("\t Loaded module with plugins: %s.py") % fullpath) + except Exception as e: + LOG.warning( + "\t Failed to load module with plugins %(path)s.py: %(e)s" + % {"path": fullpath, "e": e}) + if logging.is_debug(): + LOG.exception(e) + elif os.path.isfile(dir_or_file): + plugin_file = dir_or_file + LOG.info(_("Loading plugins from file %s") % plugin_file) + if plugin_file not in sys.path: + sys.path.append(plugin_file) + try: + plugin_name = os.path.splitext(plugin_file.split("/")[-1])[0] + imp.load_source(plugin_name, plugin_file) + LOG.info(_("\t Loaded module with plugins: %s.py") % plugin_name) + except Exception as e: + LOG.warning(_( + "\t Failed to load module with plugins %(path)s: %(e)s") + % {"path": plugin_file, "e": e}) + if logging.is_debug(): + LOG.exception(e) diff --git a/rally/common/plugin/plugin.py b/rally/common/plugin/plugin.py index b3cf8bf068..67f62ff028 100644 --- a/rally/common/plugin/plugin.py +++ b/rally/common/plugin/plugin.py @@ -13,8 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +from rally.common.plugin import discover from rally.common.plugin import meta -from rally.common import utils from rally import exceptions @@ -184,7 +184,7 @@ class Plugin(meta.MetaMixin): """ plugins = [] - for p in utils.itersubclasses(cls): + for p in discover.itersubclasses(cls): if issubclass(p, Plugin) and p._meta_is_inited(raise_exc=False): if not namespace or namespace == p.get_namespace(): plugins.append(getattr(p, "func_ref", p)) diff --git a/rally/common/utils.py b/rally/common/utils.py index f74849535c..145f81c506 100644 --- a/rally/common/utils.py +++ b/rally/common/utils.py @@ -14,21 +14,17 @@ # under the License. import functools -import imp import inspect import multiprocessing -import os import random import re import string import sys import time -from oslo_utils import importutils from six import moves from sphinx.util import docstrings -import rally from rally.common.i18n import _ from rally.common import log as logging from rally import exceptions @@ -139,46 +135,6 @@ class RAMInt(object): self.__int.value = 0 -def itersubclasses(cls, _seen=None): - """Generator over all subclasses of a given class in depth first order.""" - - if not isinstance(cls, type): - raise TypeError(_("itersubclasses must be called with " - "new-style classes, not %.100r") % cls) - _seen = _seen or set() - try: - subs = cls.__subclasses__() - except TypeError: # fails only when cls is type - subs = cls.__subclasses__(cls) - for sub in subs: - if sub not in _seen: - _seen.add(sub) - yield sub - for sub in itersubclasses(sub, _seen): - yield sub - - -def try_append_module(name, modules): - if name not in modules: - modules[name] = importutils.import_module(name) - - -def import_modules_from_package(package): - """Import modules from package and append into sys.modules - - :param: package - Full package name. For example: rally.deploy.engines - """ - path = [os.path.dirname(rally.__file__), ".."] + package.split(".") - path = os.path.join(*path) - for root, dirs, files in os.walk(path): - for filename in files: - if filename.startswith("__") or not filename.endswith(".py"): - continue - new_package = ".".join(root.split(os.sep)).split("....")[1] - module_name = "%s.%s" % (new_package, filename[:-3]) - try_append_module(module_name, sys.modules) - - def _log_wrapper(obj, log_function, msg, **kw): """A logging wrapper for any method of a class. @@ -274,48 +230,6 @@ def log_deprecated_args(message, rally_version, deprecated_args, return decorator -def load_plugins(dir_or_file): - if os.path.isdir(dir_or_file): - directory = dir_or_file - LOG.info("Loading plugins from directories %s/*" % directory) - - to_load = [] - for root, dirs, files in os.walk(directory): - to_load.extend((plugin[:-3], root) - for plugin in files if plugin.endswith(".py")) - for plugin, directory in to_load: - if directory not in sys.path: - sys.path.append(directory) - - fullpath = os.path.join(directory, plugin) - try: - fp, pathname, descr = imp.find_module(plugin, [directory]) - imp.load_module(plugin, fp, pathname, descr) - fp.close() - LOG.info("\t Loaded module with plugins: %s.py" % fullpath) - except Exception as e: - LOG.warning( - "\t Failed to load module with plugins %(path)s.py: %(e)s" - % {"path": fullpath, "e": e}) - if logging.is_debug(): - LOG.exception(e) - elif os.path.isfile(dir_or_file): - plugin_file = dir_or_file - LOG.info("Loading plugins from file %s" % plugin_file) - if plugin_file not in sys.path: - sys.path.append(plugin_file) - try: - plugin_name = os.path.splitext(plugin_file.split("/")[-1])[0] - imp.load_source(plugin_name, plugin_file) - LOG.info("\t Loaded module with plugins: %s.py" % plugin_name) - except Exception as e: - LOG.warning( - "\t Failed to load module with plugins %(path)s: %(e)s" - % {"path": plugin_file, "e": e}) - if logging.is_debug(): - LOG.exception(e) - - def get_method_class(func): """Return the class that defined the given method. diff --git a/rally/deploy/__init__.py b/rally/deploy/__init__.py index bdab2a6ab2..b889c0ac4d 100644 --- a/rally/deploy/__init__.py +++ b/rally/deploy/__init__.py @@ -14,7 +14,8 @@ # under the License. from rally.deploy.engine import * # noqa -from rally.common import utils +from rally.common.plugin import discover -utils.import_modules_from_package("rally.deploy.engines") +discover.import_modules_from_package("rally.deploy.engines") +discover.import_modules_from_package("rally.deploy.serverprovider") diff --git a/rally/deploy/serverprovider/__init__.py b/rally/deploy/serverprovider/__init__.py index 3a94fb4dcb..e69de29bb2 100644 --- a/rally/deploy/serverprovider/__init__.py +++ b/rally/deploy/serverprovider/__init__.py @@ -1,20 +0,0 @@ -# Copyright 2013: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from rally.common import utils -from rally.deploy.serverprovider.provider import * # noqa - - -utils.import_modules_from_package("rally.deploy.serverprovider.providers") diff --git a/rally/plugins/openstack/context/cleanup/manager.py b/rally/plugins/openstack/context/cleanup/manager.py index b60334cda3..f764bbe7b5 100644 --- a/rally/plugins/openstack/context/cleanup/manager.py +++ b/rally/plugins/openstack/context/cleanup/manager.py @@ -18,6 +18,7 @@ import time from rally.common import broker from rally.common.i18n import _ from rally.common import log as logging +from rally.common.plugin import discover from rally.common import utils as rutils from rally import osclients from rally.plugins.openstack.context.cleanup import base @@ -196,7 +197,7 @@ def list_resource_names(admin_required=None): True -> returns only admin ResourceManagers False -> returns only non admin ResourceManagers """ - res_mgrs = rutils.itersubclasses(base.ResourceManager) + res_mgrs = discover.itersubclasses(base.ResourceManager) if admin_required is not None: res_mgrs = filter(lambda cls: cls._admin_required == admin_required, res_mgrs) @@ -221,7 +222,7 @@ def find_resource_managers(names=None, admin_required=None): names = set(names or []) resource_managers = [] - for manager in rutils.itersubclasses(base.ResourceManager): + for manager in discover.itersubclasses(base.ResourceManager): if admin_required is not None: if admin_required != manager._admin_required: continue diff --git a/tests/unit/cli/commands/test_info.py b/tests/unit/cli/commands/test_info.py index f587f167c7..6968cb2c79 100644 --- a/tests/unit/cli/commands/test_info.py +++ b/tests/unit/cli/commands/test_info.py @@ -20,7 +20,7 @@ from rally.benchmark import sla from rally.cli.commands import info from rally import deploy from rally.deploy.engines import existing as existing_cloud -from rally.deploy import serverprovider +from rally.deploy.serverprovider import provider from rally.deploy.serverprovider.providers import existing as existing_servers from rally import exceptions from rally.plugins.common.scenarios.dummy import dummy @@ -31,8 +31,9 @@ from tests.unit import test SCENARIO = "rally.cli.commands.info.scenario_base.Scenario" SLA = "rally.cli.commands.info.sla.SLA" ENGINE = "rally.cli.commands.info.deploy.EngineFactory" -PROVIDER = "rally.cli.commands.info.serverprovider.ProviderFactory" +PROVIDER = "rally.cli.commands.info.provider.ProviderFactory" UTILS = "rally.cli.commands.info.utils" +DISCOVER = "rally.cli.commands.info.discover" COMMANDS = "rally.cli.commands.info.InfoCommands" @@ -101,29 +102,29 @@ class InfoCommandsTestCase(test.TestCase): mock_ServerProviders.assert_called_once_with() self.assertIsNone(status) - @mock.patch(UTILS + ".itersubclasses", return_value=[dummy.Dummy]) + @mock.patch(DISCOVER + ".itersubclasses", return_value=[dummy.Dummy]) def test_BenchmarkScenarios(self, mock_itersubclasses): status = self.info.BenchmarkScenarios() mock_itersubclasses.assert_called_with(scenario_base.Scenario) self.assertIsNone(status) - @mock.patch(UTILS + ".itersubclasses", + @mock.patch(DISCOVER + ".itersubclasses", return_value=[failure_rate.FailureRate]) def test_SLA(self, mock_itersubclasses): status = self.info.SLA() mock_itersubclasses.assert_called_with(sla.SLA) self.assertIsNone(status) - @mock.patch(UTILS + ".itersubclasses", + @mock.patch(DISCOVER + ".itersubclasses", return_value=[existing_cloud.ExistingCloud]) def test_DeploymentEngines(self, mock_itersubclasses): status = self.info.DeploymentEngines() mock_itersubclasses.assert_called_with(deploy.EngineFactory) self.assertIsNone(status) - @mock.patch(UTILS + ".itersubclasses", + @mock.patch(DISCOVER + ".itersubclasses", return_value=[existing_servers.ExistingServers]) def test_ServerProviders(self, mock_itersubclasses): status = self.info.ServerProviders() - mock_itersubclasses.assert_called_with(serverprovider.ProviderFactory) + mock_itersubclasses.assert_called_with(provider.ProviderFactory) self.assertIsNone(status) diff --git a/tests/unit/common/plugin/test_discover.py b/tests/unit/common/plugin/test_discover.py new file mode 100644 index 0000000000..62328e1ebd --- /dev/null +++ b/tests/unit/common/plugin/test_discover.py @@ -0,0 +1,103 @@ +# Copyright 2015: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from rally.common.plugin import discover +from tests.unit import test + + +DISCOVER = "rally.common.plugin.discover" + + +class IterSubclassesTestCase(test.TestCase): + + def test_itersubclasses(self): + class A(object): + pass + + class B(A): + pass + + class C(A): + pass + + class D(C): + pass + + self.assertEqual([B, C, D], list(discover.itersubclasses(A))) + + +class LoadExtraModulesTestCase(test.TestCase): + + @mock.patch("%s.os.path.isdir" % DISCOVER, return_value=True) + @mock.patch("%s.imp.load_module" % DISCOVER) + @mock.patch("%s.imp.find_module" % DISCOVER, + return_value=(mock.MagicMock(), None, None)) + @mock.patch("%s.os.walk" % DISCOVER, return_value=[ + ("/somewhere", ("/subdir", ), ("plugin1.py", )), + ("/somewhere/subdir", ("/subsubdir", ), ("plugin2.py", + "withoutextension")), + ("/somewhere/subdir/subsubdir", [], ("plugin3.py", ))]) + def test_load_plugins_from_dir_successful(self, mock_oswalk, + mock_find_module, + mock_load_module, mock_isdir): + test_path = "/somewhere" + discover.load_plugins(test_path) + expected = [ + mock.call("plugin1", ["/somewhere"]), + mock.call("plugin2", ["/somewhere/subdir"]), + mock.call("plugin3", ["/somewhere/subdir/subsubdir"]) + ] + self.assertEqual(expected, mock_find_module.mock_calls) + self.assertEqual(3, len(mock_load_module.mock_calls)) + + @mock.patch("%s.os.path.isfile" % DISCOVER, return_value=True) + @mock.patch("%s.imp.load_source" % DISCOVER) + def test_load_plugins_from_file_successful(self, mock_load_source, + mock_isfile): + discover.load_plugins("/somewhere/plugin.py") + expected = [mock.call("plugin", "/somewhere/plugin.py")] + self.assertEqual(expected, mock_load_source.mock_calls) + + @mock.patch("%s.os" % DISCOVER) + def test_load_plugins_from_nonexisting_and_empty_dir(self, mock_os): + # test no fails for nonexisting directory + mock_os.path.isdir.return_value = False + discover.load_plugins("/somewhere") + # test no fails for empty directory + mock_os.path.isdir.return_value = True + mock_os.walk.return_value = [] + discover.load_plugins("/somewhere") + + @mock.patch("%s.os.path.isfile" % DISCOVER, return_value=True) + def test_load_plugins_from_file_fails(self, mock_isfile): + discover.load_plugins("/somwhere/plugin.py") + + @mock.patch("%s.os.path.isfile" % DISCOVER, return_value=False) + def test_load_plugins_from_nonexisting_file(self, mock_isfile): + # test no fails for nonexisting file + discover.load_plugins("/somewhere/plugin.py") + + @mock.patch("%s.imp.load_module" % DISCOVER, side_effect=Exception()) + @mock.patch("%s.imp.find_module" % DISCOVER) + @mock.patch("%s.os.path" % DISCOVER, return_value=True) + @mock.patch("%s.os.walk" % DISCOVER, + return_value=[("/etc/.rally/plugins", [], ("load_it.py", ))]) + def test_load_plugins_fails(self, mock_oswalk, mock_ospath, + mock_load_module, mock_find_module): + # test no fails if module is broken + # TODO(olkonami): check exception is handled correct + discover.load_plugins("/somwhere") diff --git a/tests/unit/common/test_utils.py b/tests/unit/common/test_utils.py index 879a842af2..dd398eb316 100644 --- a/tests/unit/common/test_utils.py +++ b/tests/unit/common/test_utils.py @@ -13,8 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -"""Test for Rally utils.""" - from __future__ import print_function import string import sys @@ -108,31 +106,6 @@ class TimerTestCase(test.TestCase): self.assertEqual(timer.error[0], type(Exception())) -class IterSubclassesTestCase(test.TestCase): - - def test_itersubclasses(self): - class A(object): - pass - - class B(A): - pass - - class C(A): - pass - - class D(C): - pass - - self.assertEqual([B, C, D], list(utils.itersubclasses(A))) - - -class ImportModulesTestCase(test.TestCase): - def test_try_append_module_into_sys_modules(self): - modules = {} - utils.try_append_module("rally.common.version", modules) - self.assertIn("rally.common.version", modules) - - class LogTestCase(test.TestCase): def test_log_task_wrapper(self): @@ -200,70 +173,6 @@ class LogTestCase(test.TestCase): "Deprecated test (args `z' deprecated in Rally v0.0.1)") -class LoadExtraModulesTestCase(test.TestCase): - - @mock.patch("rally.common.utils.os.path.isdir", return_value=True) - @mock.patch("rally.common.utils.imp.load_module") - @mock.patch("rally.common.utils.imp.find_module", - return_value=(mock.MagicMock(), None, None)) - @mock.patch("rally.common.utils.os.walk", return_value=[ - ("/somewhere", ("/subdir", ), ("plugin1.py", )), - ("/somewhere/subdir", ("/subsubdir", ), ("plugin2.py", - "withoutextension")), - ("/somewhere/subdir/subsubdir", [], ("plugin3.py", ))]) - @mock.patch("rally.common.utils.os.path.isdir", return_value=True) - def test_load_plugins_from_dir_successful(self, mock_exists, - mock_oswalk, mock_find_module, - mock_load_module, mock_isdir): - test_path = "/somewhere" - utils.load_plugins(test_path) - expected = [ - mock.call("plugin1", ["/somewhere"]), - mock.call("plugin2", ["/somewhere/subdir"]), - mock.call("plugin3", ["/somewhere/subdir/subsubdir"]) - ] - self.assertEqual(expected, mock_find_module.mock_calls) - self.assertEqual(3, len(mock_load_module.mock_calls)) - - @mock.patch("rally.common.utils.os.path.isfile", return_value=True) - @mock.patch("rally.common.utils.imp.load_source") - def test_load_plugins_from_file_successful(self, mock_load_source, - mock_isfile): - utils.load_plugins("/somewhere/plugin.py") - expected = [mock.call("plugin", "/somewhere/plugin.py")] - self.assertEqual(expected, mock_load_source.mock_calls) - - @mock.patch("rally.common.utils.os") - def test_load_plugins_from_nonexisting_and_empty_dir(self, mock_os): - # test no fails for nonexisting directory - mock_os.path.isdir.return_value = False - utils.load_plugins("/somewhere") - # test no fails for empty directory - mock_os.path.isdir.return_value = True - mock_os.walk.return_value = [] - utils.load_plugins("/somewhere") - - @mock.patch("rally.common.utils.os.path.isfile", return_value=True) - def test_load_plugins_from_file_fails(self, mock_isfile): - utils.load_plugins("/somwhere/plugin.py") - - @mock.patch("rally.common.utils.os.path.isfile", return_value=False) - def test_load_plugins_from_nonexisting_file(self, mock_isfile): - # test no fails for nonexisting file - utils.load_plugins("/somewhere/plugin.py") - - @mock.patch("rally.common.utils.imp.load_module", side_effect=Exception()) - @mock.patch("rally.common.utils.imp.find_module") - @mock.patch("rally.common.utils.os.path", return_value=True) - @mock.patch("rally.common.utils.os.walk", - return_value=[("/etc/.rally/plugins", [], ("load_it.py", ))]) - def test_load_plugins_fails(self, mock_oswalk, mock_ospath, - mock_load_module, mock_find_module): - # test no fails if module is broken - # TODO(olkonami): check exception is handled correct - utils.load_plugins("/somwhere") - - def module_level_method(): pass diff --git a/tests/unit/deploy/serverprovider/providers/test_existing.py b/tests/unit/deploy/serverprovider/providers/test_existing.py index 693f37ebe5..3907113261 100644 --- a/tests/unit/deploy/serverprovider/providers/test_existing.py +++ b/tests/unit/deploy/serverprovider/providers/test_existing.py @@ -15,14 +15,11 @@ import jsonschema -from rally.deploy import serverprovider +from rally.deploy.serverprovider import provider from rally.deploy.serverprovider.providers import existing from tests.unit import test -ProviderFactory = serverprovider.ProviderFactory - - class ExistingServersTestCase(test.TestCase): def setUp(self): super(ExistingServersTestCase, self).setUp() @@ -31,9 +28,9 @@ class ExistingServersTestCase(test.TestCase): {"user": "user", "host": "host2"}]} def test_create_servers(self): - provider = serverprovider.ProviderFactory.get_provider(self.config, - None) - credentials = provider.create_servers() + _provider = provider.ProviderFactory.get_provider(self.config, + None) + credentials = _provider.create_servers() self.assertEqual(["host1", "host2"], [s.host for s in credentials]) self.assertEqual(["user", "user"], [s.user for s in credentials]) diff --git a/tests/unit/deploy/serverprovider/test_provider.py b/tests/unit/deploy/serverprovider/test_provider.py index cf628756d8..a8724b7891 100644 --- a/tests/unit/deploy/serverprovider/test_provider.py +++ b/tests/unit/deploy/serverprovider/test_provider.py @@ -18,14 +18,11 @@ import mock import six from rally.common import sshutils -from rally.deploy import serverprovider +from rally.deploy.serverprovider import provider from rally import exceptions from tests.unit import test -ProviderFactory = serverprovider.ProviderFactory - - class ProviderMixIn(object): def create_servers(self, image_uuid=None, amount=1): pass @@ -34,7 +31,7 @@ class ProviderMixIn(object): pass -class ProviderA(ProviderMixIn, ProviderFactory): +class ProviderA(ProviderMixIn, provider.ProviderFactory): """Fake server provider. Used for tests. @@ -42,7 +39,7 @@ class ProviderA(ProviderMixIn, ProviderFactory): pass -class ProviderB(ProviderMixIn, ProviderFactory): +class ProviderB(ProviderMixIn, provider.ProviderFactory): """Fake server provider. Used for tests. @@ -63,18 +60,18 @@ FAKE_PROVIDERS = [ProviderA, ProviderB, ProviderC] class ProviderTestCase(test.TestCase): - @mock.patch.object(ProviderFactory, "validate") + @mock.patch.object(provider.ProviderFactory, "validate") def test_init(self, fake_validate): ProviderA(None, None) fake_validate.assert_called_once_with() def test_get_provider_not_found(self): self.assertRaises(exceptions.PluginNotFound, - ProviderFactory.get_provider, + provider.ProviderFactory.get_provider, {"type": "fail"}, None) def test_vm_prvoider_factory_is_abstract(self): - self.assertRaises(TypeError, ProviderFactory) + self.assertRaises(TypeError, provider.ProviderFactory) class ServerTestCase(test.TestCase): @@ -84,15 +81,15 @@ class ServerTestCase(test.TestCase): self.keys = ["host", "user", "key", "password"] def test_init_server_dto(self): - server = serverprovider.Server(*self.vals) + server = provider.Server(*self.vals) for k, v in six.iteritems(dict(zip(self.keys, self.vals))): self.assertEqual(getattr(server, k), v) self.assertIsInstance(server.ssh, sshutils.SSH) def test_credentials(self): - server_one = serverprovider.Server(*self.vals) + server_one = provider.Server(*self.vals) creds = server_one.get_credentials() - server_two = serverprovider.Server.from_credentials(creds) + server_two = provider.Server.from_credentials(creds) for k in self.keys: self.assertEqual(getattr(server_one, k), getattr(server_two, k)) @@ -101,8 +98,8 @@ class ResourceManagerTestCase(test.TestCase): def setUp(self): super(ResourceManagerTestCase, self).setUp() self.deployment = mock.Mock() - self.resources = serverprovider.ResourceManager(self.deployment, - "provider") + self.resources = provider.ResourceManager(self.deployment, + "provider") def test_create(self): self.resources.create("info", type="type") diff --git a/tests/unit/plugins/openstack/context/cleanup/test_manager.py b/tests/unit/plugins/openstack/context/cleanup/test_manager.py index e491e5deb1..599f3907cf 100644 --- a/tests/unit/plugins/openstack/context/cleanup/test_manager.py +++ b/tests/unit/plugins/openstack/context/cleanup/test_manager.py @@ -275,7 +275,7 @@ class ResourceManagerTestCase(test.TestCase): mock_iter.assert_called_once_with(base.ResourceManager) mock_iter.reset_mock() - @mock.patch("%s.rutils.itersubclasses" % BASE) + @mock.patch("%s.discover.itersubclasses" % BASE) def test_list_resource_names(self, mock_iter): mock_iter.return_value = [ self._get_res_mock(_service="fake", _resource="1", @@ -292,7 +292,7 @@ class ResourceManagerTestCase(test.TestCase): self._list_res_names_helper(["fake", "other", "fake.2", "other.2"], False, mock_iter) - @mock.patch("%s.rutils.itersubclasses" % BASE) + @mock.patch("%s.discover.itersubclasses" % BASE) def test_find_resource_managers(self, mock_iter): mock_iter.return_value = [ self._get_res_mock(_service="fake", _resource="1", _order=1, diff --git a/tests/unit/plugins/openstack/context/cleanup/test_resources.py b/tests/unit/plugins/openstack/context/cleanup/test_resources.py index 76c7269521..eb63e50538 100644 --- a/tests/unit/plugins/openstack/context/cleanup/test_resources.py +++ b/tests/unit/plugins/openstack/context/cleanup/test_resources.py @@ -17,7 +17,7 @@ from boto import exception as boto_exception import mock from neutronclient.common import exceptions as neutron_exceptions -from rally.common import utils +from rally.common.plugin import discover from rally.plugins.openstack.context.cleanup import base from rally.plugins.openstack.context.cleanup import resources from rally.plugins.openstack.scenarios.keystone import utils as keystone_utils @@ -30,7 +30,7 @@ class AllResourceManagerTestCase(test.TestCase): def test_res_manager_special_field(self): - for res_mgr in utils.itersubclasses(base.ResourceManager): + for res_mgr in discover.itersubclasses(base.ResourceManager): manager_name = "%s.%s" % (res_mgr.__module__, res_mgr.__name__) fields = filter(lambda x: not x.startswith("__"), dir(res_mgr)) diff --git a/tests/unit/rally_jobs/test_jobs.py b/tests/unit/rally_jobs/test_jobs.py index 67b59af0c6..aeb1fc5374 100644 --- a/tests/unit/rally_jobs/test_jobs.py +++ b/tests/unit/rally_jobs/test_jobs.py @@ -20,7 +20,7 @@ import yaml from rally import api from rally.benchmark import engine -import rally.common.utils as rutils +from rally.common.plugin import discover from tests.unit import test @@ -31,7 +31,7 @@ class RallyJobsTestCase(test.TestCase): @mock.patch("rally.benchmark.engine.BenchmarkEngine" "._validate_config_semantic") def test_schema_is_valid(self, mock_validate): - rutils.load_plugins(os.path.join(self.rally_jobs_path, "plugins")) + discover.load_plugins(os.path.join(self.rally_jobs_path, "plugins")) for filename in ["rally.yaml", "rally-neutron.yaml", "rally-zaqar.yaml", "rally-designate.yaml"]: diff --git a/tests/unit/test_docstrings.py b/tests/unit/test_docstrings.py index ef0b8b8729..29b0ebf052 100644 --- a/tests/unit/test_docstrings.py +++ b/tests/unit/test_docstrings.py @@ -15,9 +15,10 @@ from rally.benchmark.scenarios import base from rally.benchmark import sla +from rally.common.plugin import discover from rally.common import utils from rally import deploy -from rally.deploy import serverprovider +from rally.deploy.serverprovider import provider from tests.unit import test @@ -41,7 +42,7 @@ class DocstringsTestCase(test.TestCase): def test_all_scenarios_have_docstrings(self): ignored_params = ["self", "scenario_obj"] - for scenario_group in utils.itersubclasses(base.Scenario): + for scenario_group in discover.itersubclasses(base.Scenario): if scenario_group.__module__.startswith("tests."): continue @@ -75,7 +76,7 @@ class DocstringsTestCase(test.TestCase): "param": param}) def test_all_scenario_groups_have_docstrings(self): - for scenario_group in utils.itersubclasses(base.Scenario): + for scenario_group in discover.itersubclasses(base.Scenario): self._assert_class_has_docstrings(scenario_group, long_description=False) @@ -84,9 +85,9 @@ class DocstringsTestCase(test.TestCase): self._assert_class_has_docstrings(deploy_engine) def test_all_server_providers_have_docstrings(self): - for provider in serverprovider.ProviderFactory.get_all(): - self._assert_class_has_docstrings(provider) + for _provider in provider.ProviderFactory.get_all(): + self._assert_class_has_docstrings(_provider) def test_all_SLA_have_docstrings(self): - for s in utils.itersubclasses(sla.SLA): + for s in discover.itersubclasses(sla.SLA): self._assert_class_has_docstrings(s, long_description=False)