Make plugins location configurable

The patch introduces CLI option --plugin-paths to specify the list of
folders with plugins.

Co-Authored-By: Roman Vasilets <rvasilets@mirantis.com>
Co-Authored-By: Ilya Shakhat <ishakhat@mirantis.com>

Change-Id: Iee39217bfe3e5c77e6925910f3998942541387a3

rally/cli/cliutils.py

@@ -433,6 +433,16 @@ def run(argv, categories):
                                      handler=parser)
 
     CONF.register_cli_opt(category_opt)
+    help_msg = ("Additional custom plugin locations. Multiple files or "
+                "directories may be specified. All plugins in the specified"
+                " directories and subdirectories will be imported. Plugins in"
+                " /opt/rally/plugins and ~/.rally/plugins will always be "
+                "imported.")
+
+    CONF.register_cli_opt(cfg.ListOpt("plugin-paths",
+                                      default=os.environ.get(
+                                          "RALLY_PLUGIN_PATHS"),
+                                      help=help_msg))
 
     try:
         CONF(argv[1:], project="rally", version=version.version_string())
@@ -511,6 +521,8 @@ def run(argv, categories):
         utils.load_plugins("/opt/rally/plugins/")
         utils.load_plugins(os.path.expanduser("~/.rally/plugins/"))
         utils.import_modules_from_package("rally.plugins")
+        for path in CONF.plugin_paths or []:
+            utils.load_plugins(path)
 
         validate_deprecated_args(argv, fn)
 

rally/common/utils.py

@@ -274,8 +274,9 @@ def log_deprecated_args(message, rally_version, deprecated_args,
     return decorator
 
 
-def load_plugins(directory):
-    if os.path.exists(directory):
+def load_plugins(dir_or_file):
+    if os.path.isdir(dir_or_file):
+        directory = dir_or_file
         LOG.info("Loading plugins from directories %s/*" % directory)
 
         to_load = []
@@ -298,6 +299,21 @@ def load_plugins(directory):
                     % {"path": fullpath, "e": e})
                 if logging.is_debug():
                     LOG.exception(e)
+    elif os.path.isfile(dir_or_file):
+        plugin_file = dir_or_file
+        LOG.info("Loading plugins from file %s" % plugin_file)
+        if plugin_file not in sys.path:
+            sys.path.append(plugin_file)
+        try:
+            plugin_name = os.path.splitext(plugin_file.split("/")[-1])[0]
+            imp.load_source(plugin_name, plugin_file)
+            LOG.info("\t Loaded module with plugins: %s.py" % plugin_name)
+        except Exception as e:
+            LOG.warning(
+                "\t Failed to load module with plugins %(path)s: %(e)s"
+                % {"path": plugin_file, "e": e})
+            if logging.is_debug():
+                LOG.exception(e)
 
 
 def get_method_class(func):

tests/unit/common/test_utils.py

@@ -202,6 +202,7 @@ class LogTestCase(test.TestCase):
 
 class LoadExtraModulesTestCase(test.TestCase):
 
+    @mock.patch("rally.common.utils.os.path.isdir", return_value=True)
     @mock.patch("rally.common.utils.imp.load_module")
     @mock.patch("rally.common.utils.imp.find_module",
                 return_value=(mock.MagicMock(), None, None))
@@ -210,10 +211,10 @@ class LoadExtraModulesTestCase(test.TestCase):
         ("/somewhere/subdir", ("/subsubdir", ), ("plugin2.py",
                                                  "withoutextension")),
         ("/somewhere/subdir/subsubdir", [], ("plugin3.py", ))])
-    @mock.patch("rally.common.utils.os.path.exists", return_value=True)
-    def test_load_plugins_successfull(self, mock_exists,
-                                      mock_oswalk, mock_find_module,
-                                      mock_load_module):
+    @mock.patch("rally.common.utils.os.path.isdir", return_value=True)
+    def test_load_plugins_from_dir_successful(self, mock_exists,
+                                              mock_oswalk, mock_find_module,
+                                              mock_load_module, mock_isdir):
         test_path = "/somewhere"
         utils.load_plugins(test_path)
         expected = [
@@ -221,19 +222,36 @@ class LoadExtraModulesTestCase(test.TestCase):
             mock.call("plugin2", ["/somewhere/subdir"]),
             mock.call("plugin3", ["/somewhere/subdir/subsubdir"])
         ]
-        self.assertEqual(mock_find_module.mock_calls, expected)
-        self.assertEqual(len(mock_load_module.mock_calls), 3)
+        self.assertEqual(expected, mock_find_module.mock_calls)
+        self.assertEqual(3, len(mock_load_module.mock_calls))
+
+    @mock.patch("rally.common.utils.os.path.isfile", return_value=True)
+    @mock.patch("rally.common.utils.imp.load_source")
+    def test_load_plugins_from_file_successful(self, mock_load_source,
+                                               mock_isfile):
+        utils.load_plugins("/somewhere/plugin.py")
+        expected = [mock.call("plugin", "/somewhere/plugin.py")]
+        self.assertEqual(expected, mock_load_source.mock_calls)
 
     @mock.patch("rally.common.utils.os")
     def test_load_plugins_from_nonexisting_and_empty_dir(self, mock_os):
         # test no fails for nonexisting directory
-        mock_os.path.exists.return_value = False
+        mock_os.path.isdir.return_value = False
         utils.load_plugins("/somewhere")
         # test no fails for empty directory
-        mock_os.path.exists.return_value = True
+        mock_os.path.isdir.return_value = True
         mock_os.walk.return_value = []
         utils.load_plugins("/somewhere")
 
+    @mock.patch("rally.common.utils.os.path.isfile", return_value=True)
+    def test_load_plugins_from_file_fails(self, mock_isfile):
+        utils.load_plugins("/somwhere/plugin.py")
+
+    @mock.patch("rally.common.utils.os.path.isfile", return_value=False)
+    def test_load_plugins_from_nonexisting_file(self, mock_isfile):
+        # test no fails for nonexisting file
+        utils.load_plugins("/somewhere/plugin.py")
+
     @mock.patch("rally.common.utils.imp.load_module", side_effect=Exception())
     @mock.patch("rally.common.utils.imp.find_module")
     @mock.patch("rally.common.utils.os.path", return_value=True)