Merge "Loads all available drivers by default"

congress/api/system/driver_model.py

@@ -40,7 +40,7 @@ class DatasourceDriverModel(base.APIModel):
         drivers = self.bus.get_drivers_info()
         fields = ['id', 'description']
         results = [self.bus.make_datasource_dict(
-                   drivers[driver], fields=fields)
+                   driver, fields=fields)
                    for driver in drivers]
         return {"results": results}
 

congress/common/config.py

@@ -52,6 +52,8 @@ core_opts = [
                help=_('The type of authentication to use')),
     cfg.ListOpt('drivers',
                 default=[],
+                deprecated_for_removal=True,
+                deprecated_reason='automatically loads all configured drivers',
                 help=_('List of driver class paths to import.')),
     cfg.IntOpt('datasource_sync_period', default=60,
                help='The number of seconds to wait between synchronizing '

congress/dse2/dse_node.py

@@ -22,9 +22,9 @@ from oslo_log import log as logging
 import oslo_messaging as messaging
 from oslo_messaging import exceptions as messaging_exceptions
 from oslo_messaging.rpc import dispatcher
-from oslo_utils import importutils
 from oslo_utils import strutils
 from oslo_utils import uuidutils
+import stevedore
 
 from congress.datalog import compile as datalog_compile
 from congress.datasources import constants
@@ -56,6 +56,8 @@ class DseNode(object):
     CONTROL_TOPIC = 'congress-control'
     SERVICE_TOPIC_PREFIX = 'congress-service-'
 
+    loaded_drivers = {}
+
     def node_rpc_target(self, namespace=None, server=None, fanout=False):
         return messaging.Target(exchange=self.EXCHANGE,
                                 topic=self._add_partition(self.CONTROL_TOPIC),
@@ -115,7 +117,7 @@ class DseNode(object):
         self._control_bus = control_bus.DseNodeControlBus(self)
         self.register_service(self._control_bus)
         # load configured drivers
-        self.loaded_drivers = self.load_drivers()
+        # self.loaded_drivers = self.load_drivers()
         self.periodic_tasks = None
         self.sync_thread = None
         self.start()
@@ -503,36 +505,35 @@ class DseNode(object):
             s._published_tables_with_subscriber = tables_with_subs
 
     # Driver CRUD.  Maybe belongs in a subclass of DseNode?
-    # Note(thread-safety): blocking function?
+    @classmethod
-    def load_drivers(self):
+    def load_drivers(cls):
-        """Load all configured drivers and check no name conflict"""
+        """Loads all configured drivers"""
         result = {}
-        for driver_path in cfg.CONF.drivers:
+        mgr = stevedore.extension.ExtensionManager(
-            # Note(thread-safety): blocking call?
+            namespace='congress.datasource.drivers',
-            obj = importutils.import_class(driver_path)
+            invoke_on_load=False)
-            driver = obj.get_datasource_info()
-            if driver['id'] in result:
-                raise exception.BadConfig(_("There is a driver loaded already"
-                                          "with the driver name of %s")
-                                          % driver['id'])
-            driver['module'] = driver_path
-            result[driver['id']] = driver
-        return result
 
-    def get_driver_info(self, driver_name):
+        for driver in mgr:
-        driver = self.loaded_drivers.get(driver_name)
+            result[driver.name] = driver
+
+        cls.loaded_drivers = result
+
+    @classmethod
+    def get_driver_info(cls, driver_name):
+        driver = cls.loaded_drivers.get(driver_name)
         if not driver:
             raise exception.DriverNotFound(id=driver_name)
-        return driver
+        return driver.plugin.get_datasource_info()
 
-    def get_drivers_info(self):
+    @classmethod
-        return self.loaded_drivers
+    def get_drivers_info(cls):
+        drivers = cls.loaded_drivers.values()
+        return [d.plugin.get_datasource_info() for d in drivers]
 
-    def get_driver_schema(self, drivername):
+    @classmethod
-        driver = self.get_driver_info(drivername)
+    def get_driver_schema(cls, drivername):
-        # Note(thread-safety): blocking call?
+        driver = cls.loaded_drivers.get(drivername)
-        obj = importutils.import_class(driver['module'])
+        return driver.plugin.get_schema()
-        return obj.get_schema()
 
     # Datasource CRUD.  Maybe belongs in a subclass of DseNode?
     # Note(thread-safety): blocking function
@@ -608,35 +609,37 @@ class DseNode(object):
             raise exception.InvalidDatasourceName(value=name)
         driver = req['driver']
         config = req['config'] or {}
-        for loaded_driver in self.loaded_drivers.values():
-            if loaded_driver['id'] == driver:
-                specified_options = set(config.keys())
-                valid_options = set(loaded_driver['config'].keys())
-                # Check that all the specified options passed in are
-                # valid configuration options that the driver exposes.
-                invalid_options = specified_options - valid_options
-                if invalid_options:
-                    raise exception.InvalidDriverOption(
-                        invalid_options=invalid_options)
 
-                # check that all the required options are passed in
+        try:
-                required_options = set(
+            loaded_driver = self.get_driver_info(driver)
-                    [k for k, v in loaded_driver['config'].items()
+        except exception.DriverNotFound:
-                     if v == constants.REQUIRED])
+            raise exception.InvalidDriver(driver=req)
-                missing_options = required_options - specified_options
-                if ('project_name' in missing_options and
-                        'tenant_name' in specified_options):
-                    LOG.warning("tenant_name is deprecated, use project_name "
-                                "instead")
-                    missing_options.remove('project_name')
-                if missing_options:
-                    missing_options = ', '.join(missing_options)
-                    raise exception.MissingRequiredConfigOptions(
-                        missing_options=missing_options)
-                return loaded_driver
 
-        # If we get here no datasource driver match was found.
+        specified_options = set(config.keys())
-        raise exception.InvalidDriver(driver=req)
+        valid_options = set(loaded_driver['config'].keys())
+        # Check that all the specified options passed in are
+        # valid configuration options that the driver exposes.
+        invalid_options = specified_options - valid_options
+        if invalid_options:
+            raise exception.InvalidDriverOption(
+                invalid_options=invalid_options)
+
+        # check that all the required options are passed in
+        required_options = set(
+            [k for k, v in loaded_driver['config'].items()
+             if v == constants.REQUIRED])
+        missing_options = required_options - specified_options
+
+        if ('project_name' in missing_options and 'tenant_name' in
+                specified_options):
+            LOG.warning("tenant_name is deprecated, use project_name instead")
+            missing_options.remove('project_name')
+
+        if missing_options:
+            missing_options = ', '.join(missing_options)
+            raise exception.MissingRequiredConfigOptions(
+                missing_options=missing_options)
+        return loaded_driver
 
     # Note (thread-safety): blocking function
     def create_datasource_service(self, datasource):
@@ -657,13 +660,9 @@ class DseNode(object):
             LOG.info("datasource %s not enabled, skip loading",
                      ds_dict['name'])
             return
-
+        driver = self.loaded_drivers.get(ds_dict['driver'])
-        driver_info = self.get_driver_info(ds_dict['driver'])
+        if not driver:
-        # split class_path into module and class name
+            raise exception.DriverNotFound(id=ds_dict['driver'])
-        class_path = driver_info['module']
-        pieces = class_path.split(".")
-        module_name = ".".join(pieces[:-1])
-        class_name = pieces[-1]
 
         if ds_dict['config'] is None:
             args = {'ds_id': ds_dict['id']}
@@ -671,18 +670,14 @@ class DseNode(object):
             args = dict(ds_dict['config'], ds_id=ds_dict['id'])
         kwargs = {'name': ds_dict['name'], 'args': args}
         LOG.info("creating service %s with class %s and args %s",
-                 ds_dict['name'], module_name,
+                 ds_dict['name'], driver.plugin,
                  strutils.mask_password(kwargs, "****"))
-
-        # import the module
         try:
-            # Note(thread-safety): blocking call?
+            service = driver.plugin(**kwargs)
-            module = importutils.import_module(module_name)
-            service = getattr(module, class_name)(**kwargs)
         except Exception:
             msg = ("Error loading instance of module '%s'")
-            LOG.exception(msg, class_path)
+            LOG.exception(msg, driver.plugin)
-            raise exception.DataServiceError(msg % class_path)
+            raise exception.DataServiceError(msg % driver.plugin)
         return service
 
 

congress/harness.py

@@ -73,6 +73,9 @@ def create2(node_id=None, bus_id=None, existing_node=None,
     # create services as required
     services = {}
 
+    # Load all configured drivers
+    dse_node.DseNode.load_drivers()
+
     if datasources:
         LOG.info("Registering congress datasource services on node %s",
                  node.node_id)

congress/tests/api/test_driver_model.py

@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from congress.api import webservice
 from congress.tests.api import base as api_base
 from congress.tests import base
+from congress.tests import helper
 
 
 class TestDriverModel(base.SqlTestCase):
@@ -46,15 +47,11 @@ class TestDriverModel(base.SqlTestCase):
 
     def test_drivers_list(self):
         context = {}
-        expected_ret = {"results": [
+        drivers = helper.supported_drivers()
-            {
+        expected_ret = sorted(drivers, key=lambda d: d['id'])
-                "description": "This is a fake driver used for testing",
+        ret = self.driver_model.get_items({}, context)['results']
-                "id": "fake_datasource"
+        actual_ret = sorted(ret, key=lambda d: d['id'])
-            }
+        self.assertEqual(expected_ret, actual_ret)
-        ]}
-
-        ret = self.driver_model.get_items({}, context)
-        self.assertEqual(expected_ret, ret)
 
     def test_driver_details(self):
         context = {
@@ -75,7 +72,6 @@ class TestDriverModel(base.SqlTestCase):
             },
             "description": "This is a fake driver used for testing",
             "id": "fake_datasource",
-            "module": "congress.tests.fake_datasource.FakeDataSource",
             "secret": ["password"],
             "tables": [{'columns': [
                 {'description': None, 'name': 'id'},

congress/tests/dse2/test_datasource.py

@@ -18,7 +18,6 @@ from __future__ import division
 from __future__ import absolute_import
 
 import mock
-from oslo_config import cfg
 from oslo_db import exception as db_exc
 
 from congress.db import datasources as datasource_db
@@ -194,12 +193,3 @@ class TestDataSource(base.SqlTestCase):
     #     self.assertEqual(
     #         schema,
     #         fake_datasource.FakeDataSource.get_schema())
-
-    def test_duplicate_driver_name_raises(self):
-        # Load the driver twice
-        cfg.CONF.set_override(
-            'drivers',
-            ['congress.tests.fake_datasource.FakeDataSource',
-             'congress.tests.fake_datasource.FakeDataSource'])
-        self.assertRaises(congressException.BadConfig,
-                          self.dseNode.load_drivers)

congress/tests/dse2/test_dse_node.py

@@ -315,14 +315,14 @@ class TestDseNode(base.SqlTestCase):
                            'password': '<hidden>',
                            'tenant_name': 'armax'}}
 
+    @mock.patch.object(dse_node.DseNode, 'validate_create_datasource')
     @mock.patch.object(dse_node.DseNode, 'get_driver_info')
-    def test_missing_driver_datasources(self, mock_driver_info):
+    def test_missing_driver_datasources(self, mock_driver_info, mock_validate):
         services = api_base.setup_config(api=False, policy=False)
         node = services['node']
         ds_manager = services['ds_manager']
         ds = self._get_datasource_request()
-        mock_driver_info.return_value = {'secret': [],
+        mock_driver_info.return_value = {'secret': []}
-                                         'module': mock.MagicMock()}
         ds_manager.add_datasource(ds)
         mock_driver_info.side_effect = [exception.DriverNotFound]
         node.delete_missing_driver_datasources()

congress/tests/helper.py

@@ -458,3 +458,59 @@ class TestFailureException(Exception):
     """
     def __init__(self, *args, **kwargs):
         Exception.__init__(self, *args, **kwargs)
+
+
+def supported_drivers():
+    """Get list of supported drivers by congress"""
+
+    results = [
+        {"id": "monasca",
+         "description": "Datasource driver that interfaces with monasca."},
+        {"id": "plexxi",
+         "description": "Datasource driver that interfaces with PlexxiCore."},
+        {"id": "doctor",
+         "description": "Datasource driver that allows external systems "
+                        "to push data in accordance with OPNFV Doctor "
+                        "Inspector southbound interface specification."},
+        {"id": "aodh",
+         "description": "Datasource driver that interfaces with aodh."},
+        {"id": "neutronv2_qos",
+         "description": "Datasource driver that interfaces with QoS "
+                        "extension of OpenStack Networking aka Neutron."},
+        {"id": "cloudfoundryv2",
+         "description": "Datasource driver that interfaces with cloudfoundry"},
+        {"id": "heat",
+         "description": "Datasource driver that interfaces with OpenStack "
+                        "orchestration aka heat."},
+        {"id": "nova",
+         "description": "Datasource driver that interfaces with OpenStack "
+                        "Compute aka nova."},
+        {"id": "murano",
+         "description": "Datasource driver that interfaces with murano"},
+        {"id": "neutronv2",
+         "description": "Datasource driver that interfaces with OpenStack "
+                        "Networking aka Neutron."},
+        {"id": "swift",
+         "description": "Datasource driver that interfaces with swift."},
+        {"id": "ironic",
+         "description": "Datasource driver that interfaces with OpenStack "
+                        "bare metal aka ironic."},
+        {"id": "cinder",
+         "description": "Datasource driver that interfaces with OpenStack "
+                        "cinder."},
+        {"id": "fake_datasource",
+         "description": "This is a fake driver used for testing"},
+        {"id": "config",
+         "description": "Datasource driver that allows OS configs retrieval."},
+        {"id": "glancev2",
+         "description": "Datasource driver that interfaces with OpenStack "
+                        "Images aka Glance."},
+        {"id": "vcenter",
+         "description": "Datasource driver that interfaces with vcenter"},
+        {"id": "keystonev3",
+         "description": "Datasource driver that interfaces with keystone."},
+        {"id": "keystone",
+         "description": "Datasource driver that interfaces with keystone."},
+        {"id": "mistral",
+         "description": "Datasource driver that interfaces with Mistral."}]
+    return results

setup.cfg

@@ -59,6 +59,28 @@ console_scripts =
     congress-db-manage = congress.db.migration.cli:main
     congress-cfg-validator-agt = congress.cfg_validator.agent.agent:main
 
+congress.datasource.drivers =
+    aodh = congress.datasources.aodh_driver:AodhDriver
+    cinder = congress.datasources.cinder_driver:CinderDriver
+    cloudfoundryv2 = congress.datasources.cloudfoundryv2_driver:CloudFoundryV2Driver
+    config = congress.datasources.cfgvalidator_driver:ValidatorDriver
+    doctor = congress.datasources.doctor_driver:DoctorDriver
+    fake_datasource = congress.tests.fake_datasource:FakeDataSource
+    glancev2 = congress.datasources.glancev2_driver:GlanceV2Driver
+    heat = congress.datasources.heatv1_driver:HeatV1Driver
+    ironic = congress.datasources.ironic_driver:IronicDriver
+    keystone = congress.datasources.keystone_driver:KeystoneDriver
+    keystonev3 = congress.datasources.keystonev3_driver:KeystoneV3Driver
+    mistral = congress.datasources.mistral_driver:MistralDriver
+    monasca = congress.datasources.monasca_driver:MonascaDriver
+    murano = congress.datasources.murano_driver:MuranoDriver
+    neutronv2 = congress.datasources.neutronv2_driver:NeutronV2Driver
+    neutronv2_qos = congress.datasources.neutronv2_qos_driver:NeutronV2QosDriver
+    nova = congress.datasources.nova_driver:NovaDriver
+    plexxi = congress.datasources.plexxi_driver:PlexxiDriver
+    swift = congress.datasources.swift_driver:SwiftDriver
+    vcenter = congress.datasources.vCenter_driver:VCenterDriver
+
 [build_sphinx]
 all_files = 1
 build-dir = doc/build