Use stevedore for scheduler driver

Avoid having to configure the full class path of scheduler driver,
change to load by stevedore driver plugin using entrypoints.

Change 'scheduler_driver' to use entrypoint with the namespace
'nova.scheduler.driver' in 'setup.cfg'. Meanwhile, still maintain the
compatibility for class path configuration until the next major release.

Change all related tests with flag 'scheduler_driver' to use stevedore
entrypoint.

UpgradeImpact - see the reno file attached.
Implements: blueprint scheduler-driver-use-stevedore
Change-Id: I8c169e12d9bfacdbdb1dadf68b8a1fa98c5ea5bc

nova/conf/scheduler.py

@@ -270,20 +270,26 @@ and will be dropped in the N release.
 """)
 
 driver_opt = cfg.StrOpt("scheduler_driver",
-        default="nova.scheduler.filter_scheduler.FilterScheduler",
+        default="filter_scheduler",
         help="""
-The class of the driver used by the scheduler. This should be the full Python
+The class of the driver used by the scheduler. This should be chosen from one
-path to the class to be used. If nothing is specified in this option, the
+of the entrypoints under the namespace 'nova.scheduler.driver' of file
-FilterScheduler is used.
+'setup.cfg'. If nothing is specified in this option, the 'filter_scheduler' is
+used.
+
+This option also supports deprecated full Python path to the class to be used.
+For example, "nova.scheduler.filter_scheduler.FilterScheduler". But note: this
+support will be dropped in the N Release.
 
 Other options are:
 
-    * 'nova.scheduler.caching_scheduler.CachingScheduler' which aggressively
+    * 'caching_scheduler' which aggressively caches the system state for better
-    caches the system state for better individual scheduler performance at
+    individual scheduler performance at the risk of more retries when running
-    the risk of more retries when running multiple schedulers.
+    multiple schedulers.
 
-    * 'nova.scheduler.chance.ChanceScheduler' which simply picks a host at
+    * 'chance_scheduler' which simply picks a host at random.
-    random.
+
+    * 'fake_scheduler' which is used for testing.
 
 * Services that use this:
 

nova/scheduler/manager.py

@@ -19,18 +19,23 @@
 Scheduler Service
 """
 
+from oslo_log import log as logging
 import oslo_messaging as messaging
 from oslo_serialization import jsonutils
 from oslo_service import periodic_task
 from oslo_utils import importutils
+from stevedore import driver
 
 import nova.conf
 from nova import exception
+from nova.i18n import _, _LW
 from nova import manager
 from nova import objects
 from nova import quota
 
 
+LOG = logging.getLogger(__name__)
+
 CONF = nova.conf.CONF
 
 QUOTAS = quota.QUOTAS
@@ -46,7 +51,28 @@ class SchedulerManager(manager.Manager):
     def __init__(self, scheduler_driver=None, *args, **kwargs):
         if not scheduler_driver:
             scheduler_driver = CONF.scheduler_driver
-        self.driver = importutils.import_object(scheduler_driver)
+        try:
+            self.driver = driver.DriverManager(
+                    "nova.scheduler.driver",
+                    scheduler_driver,
+                    invoke_on_load=True).driver
+        # TODO(Yingxin): Change to catch stevedore.exceptions.NoMatches after
+        # stevedore v1.9.0
+        except RuntimeError:
+            # NOTE(Yingxin): Loading full class path is deprecated and should
+            # be removed in the N release.
+            try:
+                self.driver = importutils.import_object(scheduler_driver)
+                LOG.warning(_LW("DEPRECATED: scheduler_driver uses "
+                                "classloader to load %(path)s. This legacy "
+                                "loading style will be removed in the "
+                                "N release."),
+                            {'path': scheduler_driver})
+            except (ImportError, ValueError):
+                raise RuntimeError(
+                        _("Cannot load scheduler driver from configuration "
+                          "%(conf)s."),
+                        {'conf': scheduler_driver})
         super(SchedulerManager, self).__init__(service_name='scheduler',
                                                *args, **kwargs)
 

nova/tests/functional/integrated_helpers.py

@@ -92,8 +92,7 @@ class _IntegratedTestBase(test.TestCase):
         return self.start_service('compute')
 
     def _setup_scheduler_service(self):
-        self.flags(scheduler_driver='nova.scheduler.'
+        self.flags(scheduler_driver='chance_scheduler')
-                    'chance.ChanceScheduler')
         return self.start_service('scheduler')
 
     def _setup_services(self):

nova/tests/functional/libvirt/test_numa_servers.py

@@ -79,8 +79,7 @@ class NUMAServersTest(ServersTestBase):
     def _setup_scheduler_service(self):
         self.flags(compute_driver='nova.virt.libvirt.LibvirtDriver')
 
-        self.flags(scheduler_driver='nova.scheduler.'
+        self.flags(scheduler_driver='filter_scheduler')
-                    'filter_scheduler.FilterScheduler')
         self.flags(scheduler_default_filters=CONF.scheduler_default_filters
                    + ['NUMATopologyFilter'])
         return self.start_service('scheduler')

nova/tests/unit/scheduler/test_scheduler.py

@@ -21,7 +21,10 @@ import mock
 
 from nova import context
 from nova import objects
+from nova.scheduler import caching_scheduler
+from nova.scheduler import chance
 from nova.scheduler import driver
+from nova.scheduler import filter_scheduler
 from nova.scheduler import host_manager
 from nova.scheduler import ironic_host_manager
 from nova.scheduler import manager
@@ -31,18 +34,74 @@ from nova.tests.unit import fake_server_actions
 from nova.tests.unit.scheduler import fakes
 
 
+class SchedulerManagerInitTestCase(test.NoDBTestCase):
+    """Test case for scheduler manager initiation."""
+    manager_cls = manager.SchedulerManager
+
+    @mock.patch.object(host_manager.HostManager, '_init_instance_info')
+    @mock.patch.object(host_manager.HostManager, '_init_aggregates')
+    def test_init_using_default_schedulerdriver(self,
+                                                mock_init_agg,
+                                                mock_init_inst):
+        driver = self.manager_cls().driver
+        self.assertIsInstance(driver, filter_scheduler.FilterScheduler)
+
+    @mock.patch.object(host_manager.HostManager, '_init_instance_info')
+    @mock.patch.object(host_manager.HostManager, '_init_aggregates')
+    def test_init_using_chance_schedulerdriver(self,
+                                               mock_init_agg,
+                                               mock_init_inst):
+        self.flags(scheduler_driver='chance_scheduler')
+        driver = self.manager_cls().driver
+        self.assertIsInstance(driver, chance.ChanceScheduler)
+
+    @mock.patch.object(host_manager.HostManager, '_init_instance_info')
+    @mock.patch.object(host_manager.HostManager, '_init_aggregates')
+    def test_init_using_caching_schedulerdriver(self,
+                                                mock_init_agg,
+                                                mock_init_inst):
+        self.flags(scheduler_driver='caching_scheduler')
+        driver = self.manager_cls().driver
+        self.assertIsInstance(driver, caching_scheduler.CachingScheduler)
+
+    @mock.patch.object(host_manager.HostManager, '_init_instance_info')
+    @mock.patch.object(host_manager.HostManager, '_init_aggregates')
+    def test_init_nonexist_schedulerdriver(self,
+                                           mock_init_agg,
+                                           mock_init_inst):
+        self.flags(scheduler_driver='nonexist_scheduler')
+        self.assertRaises(RuntimeError, self.manager_cls)
+
+    # NOTE(Yingxin): Loading full class path is deprecated and should be
+    # removed in the N release.
+    @mock.patch.object(manager.LOG, 'warning')
+    @mock.patch.object(host_manager.HostManager, '_init_instance_info')
+    @mock.patch.object(host_manager.HostManager, '_init_aggregates')
+    def test_init_using_classpath_to_schedulerdriver(self,
+                                                     mock_init_agg,
+                                                     mock_init_inst,
+                                                     mock_warning):
+        self.flags(
+            scheduler_driver=
+            'nova.scheduler.chance.ChanceScheduler')
+        driver = self.manager_cls().driver
+        self.assertIsInstance(driver, chance.ChanceScheduler)
+        warn_args, kwargs = mock_warning.call_args
+        self.assertIn("DEPRECATED", warn_args[0])
+
+
 class SchedulerManagerTestCase(test.NoDBTestCase):
     """Test case for scheduler manager."""
 
     manager_cls = manager.SchedulerManager
     driver_cls = fakes.FakeScheduler
-    driver_cls_name = 'nova.tests.unit.scheduler.fakes.FakeScheduler'
+    driver_plugin_name = 'fake_scheduler'
 
     @mock.patch.object(host_manager.HostManager, '_init_instance_info')
     @mock.patch.object(host_manager.HostManager, '_init_aggregates')
     def setUp(self, mock_init_agg, mock_init_inst):
         super(SchedulerManagerTestCase, self).setUp()
-        self.flags(scheduler_driver=self.driver_cls_name)
+        self.flags(scheduler_driver=self.driver_plugin_name)
         with mock.patch.object(host_manager.HostManager, '_init_aggregates'):
             self.manager = self.manager_cls()
         self.context = context.RequestContext('fake_user', 'fake_project')

releasenotes/notes/config_scheduler_driver-e751ae392bc1a1d0.yaml

@@ -0,0 +1,9 @@
+---
+upgrade:
+  - |
+    The option ``scheduler_driver`` is now changed to use entrypoint instead of
+    full class path. Set one of the entrypoints under the namespace
+    'nova.scheduler.driver' in 'setup.cfg'. Its default value is
+    'host_manager'. The full class path style is still supported in current
+    release. But it is not recommended because class path can be changed and
+    this support will be dropped in the next major release.

setup.cfg

@@ -191,6 +191,12 @@ nova.scheduler.host_manager =
     host_manager = nova.scheduler.host_manager:HostManager
     ironic_host_manager = nova.scheduler.ironic_host_manager:IronicHostManager
 
+nova.scheduler.driver =
+    filter_scheduler = nova.scheduler.filter_scheduler:FilterScheduler
+    caching_scheduler = nova.scheduler.caching_scheduler:CachingScheduler
+    chance_scheduler = nova.scheduler.chance:ChanceScheduler
+    fake_scheduler = nova.tests.unit.scheduler.fakes:FakeScheduler
+
 [build_sphinx]
 all_files = 1
 build-dir = doc/build