Migrate to stevedore

Use stevedore to load both transport and storage drivers. It will allow
other users to extend Marconi with their own storages / transports more
easily. Stevedore also covers most of the cases around managing plugins.

Change-Id: Ic417c29cef41bdb72c8c446937905509db5ba8cd

marconi/bootstrap.py

@@ -13,18 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from stevedore import driver
+
 from marconi.common import config
 from marconi.common import decorators
 from marconi.common import exceptions
-from marconi.openstack.common import importutils
 from marconi.openstack.common import log
 from marconi import transport  # NOQA.
 
 
 cfg_handle = config.project('marconi')
 cfg = config.namespace('drivers').from_options(
-    transport='marconi.transport.wsgi',
+    transport='wsgi',
-    storage='marconi.storage.sqlite')
+    storage='sqlite')
 
 LOG = log.getLogger(__name__)
 
@@ -42,25 +43,26 @@ class Bootstrap(object):
 
     @decorators.lazy_property(write=False)
     def storage(self):
-        msg = _("Loading Storage Driver")
+        LOG.debug(_("Loading Storage Driver"))
-        LOG.debug(msg)
+        try:
-        storage_module = import_driver(cfg.storage)
+            mgr = driver.DriverManager('marconi.storage',
-        return storage_module.Driver()
+                                       cfg.storage,
+                                       invoke_on_load=True)
+            return mgr.driver
+        except RuntimeError as exc:
+                raise exceptions.InvalidDriver(exc)
 
     @decorators.lazy_property(write=False)
     def transport(self):
-        msg = _("Loading Transport Driver")
+        LOG.debug(_("Loading Transport Driver"))
-        LOG.debug(msg)
+        try:
-        transport_module = import_driver(cfg.transport)
+            mgr = driver.DriverManager('marconi.transport',
-        return transport_module.Driver(self.storage)
+                                       cfg.transport,
+                                       invoke_on_load=True,
+                                       invoke_args=[self.storage])
+            return mgr.driver
+        except RuntimeError as exc:
+                raise exceptions.InvalidDriver(exc)
 
     def run(self):
         self.transport.listen()
-
-
-def import_driver(module_name):
-    try:
-        return importutils.import_module(module_name)
-    except ImportError:
-        raise exceptions.InvalidDriver(
-            'No module named %s' % module_name)

marconi/openstack/common/importutils.py

@@ -1,67 +0,0 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""
-Import related utilities and helper functions.
-"""
-
-import sys
-import traceback
-
-
-def import_class(import_str):
-    """Returns a class from a string including module and class"""
-    mod_str, _sep, class_str = import_str.rpartition('.')
-    try:
-        __import__(mod_str)
-        return getattr(sys.modules[mod_str], class_str)
-    except (ValueError, AttributeError):
-        raise ImportError('Class %s cannot be found (%s)' %
-                          (class_str,
-                           traceback.format_exception(*sys.exc_info())))
-
-
-def import_object(import_str, *args, **kwargs):
-    """Import a class and return an instance of it."""
-    return import_class(import_str)(*args, **kwargs)
-
-
-def import_object_ns(name_space, import_str, *args, **kwargs):
-    """
-    Import a class and return an instance of it, first by trying
-    to find the class in a default namespace, then failing back to
-    a full path if not found in the default namespace.
-    """
-    import_value = "%s.%s" % (name_space, import_str)
-    try:
-        return import_class(import_value)(*args, **kwargs)
-    except ImportError:
-        return import_class(import_str)(*args, **kwargs)
-
-
-def import_module(import_str):
-    """Import a module."""
-    __import__(import_str)
-    return sys.modules[import_str]
-
-
-def try_import(import_str, default=None):
-    """Try to import a module and if it fails return default."""
-    try:
-        return import_module(import_str)
-    except ImportError:
-        return default

marconi/tests/etc/drivers_storage_invalid.conf

@@ -1,5 +1,9 @@
+[DEFAULT]
+debug = False
+verbose = False
+
 [drivers]
-transport = marconi.transport.wsgi
+transport = wsgi
 storage = invalid
 
 [drivers:transport:wsgi]

marconi/tests/etc/drivers_transport_invalid.conf

@@ -1,6 +1,10 @@
+[DEFAULT]
+debug = False
+verbose = False
+
 [drivers]
 transport = invalid
-storage = marconi.storage.sqlite
+storage = sqlite
 
 [drivers:transport:wsgi]
 port = 8888

marconi/tests/etc/keystone_auth.conf

@@ -1,10 +1,12 @@
 [DEFAULT]
 auth_strategy = keystone
 
+debug = False
+verbose = False
 
 [drivers]
-transport = marconi.transport.wsgi
+transport = wsgi
-storage = marconi.storage.sqlite
+storage = sqlite
 
 [drivers:transport:wsgi]
 bind = 0.0.0.0:8888

marconi/tests/etc/wsgi_faulty.conf

@@ -1,6 +1,10 @@
+[DEFAULT]
+debug = False
+verbose = False
+
 [drivers]
-transport = marconi.transport.wsgi
+transport = wsgi
-storage = marconi.tests.util.faulty_storage
+storage = sqlite
 
 [drivers:transport:wsgi]
 port = 8888

marconi/tests/etc/wsgi_mongodb.conf

@@ -1,6 +1,10 @@
+[DEFAULT]
+debug = False
+verbose = False
+
 [drivers]
-transport = marconi.transport.wsgi
+transport = wsgi
-storage = marconi.storage.mongodb
+storage = mongodb
 
 [drivers:transport:wsgi]
 port = 8888

marconi/tests/etc/wsgi_sqlite.conf

@@ -1,6 +1,10 @@
+[DEFAULT]
+debug = False
+verbose = False
+
 [drivers]
-transport = marconi.transport.wsgi
+transport = wsgi
-storage = marconi.storage.sqlite
+storage = sqlite
 
 [drivers:transport:wsgi]
 bind = 0.0.0.0

marconi/tests/transport/wsgi/base.py

@@ -18,6 +18,7 @@ from falcon import testing
 
 import marconi
 from marconi.tests import util
+from marconi.tests.util import faulty_storage
 
 
 class TestBase(util.TestBase):
@@ -35,3 +36,16 @@ class TestBase(util.TestBase):
 
         self.app = boot.transport.app
         self.srmock = testing.StartResponseMock()
+
+
+class TestBaseFaulty(TestBase):
+
+    def setUp(self):
+        self._storage_backup = marconi.Bootstrap.storage
+        faulty = faulty_storage.Driver()
+        setattr(marconi.Bootstrap, "storage", faulty)
+        super(TestBaseFaulty, self).setUp()
+
+    def tearDown(self):
+        setattr(marconi.Bootstrap, "storage", self._storage_backup)
+        super(TestBaseFaulty, self).tearDown()

marconi/tests/transport/wsgi/test_claims.py

@@ -220,7 +220,7 @@ class ClaimsSQLiteTests(ClaimsBaseTest):
     config_filename = 'wsgi_sqlite.conf'
 
 
-class ClaimsFaultyDriverTests(base.TestBase):
+class ClaimsFaultyDriverTests(base.TestBaseFaulty):
 
     config_filename = 'wsgi_faulty.conf'
 

marconi/tests/transport/wsgi/test_messages.py

@@ -244,7 +244,7 @@ class MessagesMongoDBTests(MessagesBaseTest):
         super(MessagesMongoDBTests, self).setUp()
 
 
-class MessagesFaultyDriverTests(base.TestBase):
+class MessagesFaultyDriverTests(base.TestBaseFaulty):
 
     config_filename = 'wsgi_faulty.conf'
 

marconi/tests/transport/wsgi/test_queue_lifecycle.py

@@ -230,7 +230,7 @@ class QueueLifecycleSQLiteTests(QueueLifecycleBaseTest):
     config_filename = 'wsgi_sqlite.conf'
 
 
-class QueueFaultyDriverTests(base.TestBase):
+class QueueFaultyDriverTests(base.TestBaseFaulty):
 
     config_filename = 'wsgi_faulty.conf'
 

marconi/tests/util/base.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import fixtures
 import os
 import testtools
 
@@ -29,6 +30,16 @@ class TestBase(testtools.TestCase):
     test method.
     """
 
+    def setUp(self):
+        super(TestBase, self).setUp()
+        self.useFixture(fixtures.FakeLogger('marconi'))
+
+        stdout = self.useFixture(fixtures.StringStream('stdout')).stream
+        self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
+
+        stderr = self.useFixture(fixtures.StringStream('stderr')).stream
+        self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
+
     def conf_path(self, filename):
         """Returns the full path to the specified Marconi conf file.
 

requirements.txt

@@ -10,3 +10,4 @@ pymongo
 python-keystoneclient
 simplejson
 WebOb
+stevedore>=0.9

setup.cfg

@@ -27,8 +27,15 @@ setup-hooks =
 
 [entry_points]
 console_scripts =
-    marconi-server = marconi.cmd.server:run
     marconi-gc = marconi.cmd.gc:run
+    marconi-server = marconi.cmd.server:run
+
+marconi.storage =
+    sqlite = marconi.storage.sqlite.driver:Driver
+    mongodb = marconi.storage.mongodb.driver:Driver
+
+marconi.transport =
+    wsgi = marconi.transport.wsgi.driver:Driver
 
 [nosetests]
 where=marconi/tests