From d0418d48b739b272b565d594a5e5983b80248363 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Mon, 14 May 2018 19:34:09 +0300 Subject: [PATCH] Load options before importing packages Importing modules from the package (rally plugin) can fail while trying to access proper configuration options. To solve this issue, we need to load options before loading modules itselves. Change-Id: I387ed4f6af809e1306d30fb9903d65668eb14839 --- rally/common/plugin/discover.py | 46 ++++++++++++++--------- rally/plugins/__init__.py | 3 +- tests/unit/common/plugin/test_discover.py | 17 ++++++--- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/rally/common/plugin/discover.py b/rally/common/plugin/discover.py index 134fbbfbe4..9577891744 100644 --- a/rally/common/plugin/discover.py +++ b/rally/common/plugin/discover.py @@ -65,8 +65,8 @@ def import_modules_from_package(package): sys.modules[module_name] = importlib.import_module(module_name) -def import_modules_by_entry_point(): - """Import plugins by entry-point 'rally_plugins'.""" +def find_packages_by_entry_point(): + """Find all packages with rally_plugins entry-point""" loaded_packages = [] for package in pkg_resources.working_set: @@ -79,6 +79,30 @@ def import_modules_by_entry_point(): if "path" in entry_map: ep = entry_map["path"] + package_info["plugins_path"] = ep.module_name + if "options" in entry_map: + ep = entry_map["options"] + package_info["options"] = "%s:%s" % ( + ep.module_name, + ep.attrs[0] if ep.attrs else "list_opts", + ) + + if package_info: + package_info.update( + name=package.project_name, + version=package.version) + loaded_packages.append(package_info) + return loaded_packages + + +def import_modules_by_entry_point(_packages=None): + """Import plugins by entry-point 'rally_plugins'.""" + loaded_packages = _packages or find_packages_by_entry_point() + + for package in loaded_packages: + if "plugins_path" in package: + em = pkg_resources.get_entry_map(package["name"]) + ep = em["rally_plugins"]["path"] try: m = ep.load() if hasattr(m, "__path__"): @@ -93,26 +117,12 @@ def import_modules_by_entry_point(): msg = ("\t Failed to load plugins from module '%(module)s' " "(package: '%(package)s')" % {"module": ep.module_name, - "package": "%s %s" % (package.project_name, - package.version)}) + "package": "%s %s" % (package["name"], + package["version"])}) if logging.is_debug(): LOG.exception(msg) else: LOG.warning(msg + (": %s" % six.text_type(e))) - else: - package_info["plugins_path"] = ep.module_name - if "options" in entry_map: - ep = entry_map["options"] - package_info["options"] = "%s:%s" % ( - ep.module_name, - ep.attrs[0] if ep.attrs else "list_opts", - ) - - if package_info: - package_info.update( - name=package.project_name, - version=package.version) - loaded_packages.append(package_info) return loaded_packages diff --git a/rally/plugins/__init__.py b/rally/plugins/__init__.py index b9e3fd0e25..bc226bdfa9 100644 --- a/rally/plugins/__init__.py +++ b/rally/plugins/__init__.py @@ -43,10 +43,11 @@ def load(): discover.import_modules_from_package("rally.plugins.openstack") discover.import_modules_from_package("rally.plugins.workload") - packages = discover.import_modules_by_entry_point() + packages = discover.find_packages_by_entry_point() for package in packages: if "options" in package: opts.register_options_from_path(package["options"]) + discover.import_modules_by_entry_point(_packages=packages) discover.load_plugins("/opt/rally/plugins/") discover.load_plugins(os.path.expanduser("~/.rally/plugins/")) diff --git a/tests/unit/common/plugin/test_discover.py b/tests/unit/common/plugin/test_discover.py index e25440c476..b01da408f3 100644 --- a/tests/unit/common/plugin/test_discover.py +++ b/tests/unit/common/plugin/test_discover.py @@ -193,6 +193,14 @@ class LoadExtraModulesTestCase(test.TestCase): }}) ] + def mock_get_entry_map(name, group=None): + self.assertIsNone(group) + for p in mock_pkg_resources.working_set: + if p.project_name == name: + return p.entry_map + + mock_pkg_resources.get_entry_map.side_effect = mock_get_entry_map + # use random uuid to not have conflicts in sys.modules packages = [[(mock.Mock(), str(uuid.uuid4()), None)] for i in range(3)] mock_walk_packages.side_effect = packages @@ -205,12 +213,9 @@ class LoadExtraModulesTestCase(test.TestCase): for ep_name, ep in entry_map.items(): if ep_name == "path": ep.load.assert_called_once_with() - if package.project_name == "error": - self.assertNotIn(package.project_name, data) - else: - self.assertIn(package.project_name, data) - self.assertEqual(package.version, - data[package.project_name]["version"]) + self.assertIn(package.project_name, data) + self.assertEqual(package.version, + data[package.project_name]["version"]) else: self.assertFalse(ep.load.called) if ep_name == "options":