Share_server-pool mapping

Manila scheduler needs information about relationship between
pools and share_servers to make Manila scheduling decisions smarter,
as it would be based on pool capacity/capability.

Add new share driver method called 'get_share_server_pools'
which takes a share server and returns a list of pools.

Share manager calls that method for every share server on every stats update
and adds mapping to share_stats.

Add new function 'share_server_get_all_by_host' to DB API.

Change-Id: I41041cc247da5e25383c2f3f827d7d96b4347427

manila/db/api.py

@@ -640,6 +640,11 @@ def share_server_get_all(context):
     return IMPL.share_server_get_all(context)
 
 
+def share_server_get_all_by_host(context, host):
+    """Get all share servers related to particular host."""
+    return IMPL.share_server_get_all_by_host(context, host)
+
+
 def share_server_get_all_unused_deletable(context, host, updated_before):
     """Get all free share servers DB records."""
     return IMPL.share_server_get_all_unused_deletable(context, host,

manila/db/sqlalchemy/api.py

@@ -2028,6 +2028,11 @@ def share_server_get_all(context):
     return _server_get_query(context).all()
 
 
+@require_context
+def share_server_get_all_by_host(context, host):
+    return _server_get_query(context).filter_by(host=host).all()
+
+
 @require_context
 def share_server_get_all_unused_deletable(context, host, updated_before):
     valid_server_status = (

manila/share/driver.py

@@ -176,7 +176,7 @@ class ShareDriver(object):
         self.configuration = kwargs.get('configuration', None)
         self._stats = {}
 
-        self.pools = {}
+        self.pools = []
         if self.configuration:
             self.configuration.append_config_values(share_opts)
             network_config_group = (self.configuration.network_config_group or
@@ -437,7 +437,16 @@ class ShareDriver(object):
             total_capacity_gb='infinite',
             free_capacity_gb='infinite',
             reserved_percentage=0,
-            QoS_support=False)
+            QoS_support=False,
+            pools=self.pools or None,
+        )
         if isinstance(data, dict):
             common.update(data)
         self._stats = common
+
+    def get_share_server_pools(self, share_server):
+        """Return list of pools related to a particular share server.
+
+        :param share_server: ShareServer class instance.
+        """
+        return []

manila/share/manager.py

@@ -647,8 +647,22 @@ class ShareManager(manager.SchedulerDependentManager):
     def _report_driver_status(self, context):
         LOG.info(_LI('Updating share status'))
         share_stats = self.driver.get_share_stats(refresh=True)
-        if share_stats:
-            self.update_service_capabilities(share_stats)
+        if not share_stats:
+            return
+
+        if self.driver.driver_handles_share_servers:
+            share_stats['server_pools_mapping'] = (
+                self._get_servers_pool_mapping(context)
+            )
+
+        self.update_service_capabilities(share_stats)
+
+    def _get_servers_pool_mapping(self, context):
+        """Get info about relationships between pools and share_servers."""
+        share_servers = self.db.share_server_get_all_by_host(context,
+                                                             self.host)
+        return dict((server['id'], self.driver.get_share_server_pools(server))
+                    for server in share_servers)
 
     def publish_service_capabilities(self, context):
         """Collect driver status and then publish it."""

manila/tests/share/drivers/emc/test_driver.py

@@ -17,8 +17,8 @@ import mock
 from oslo_log import log
 from stevedore import extension
 
+from manila import network
 from manila.share import configuration as conf
-from manila.share import driver
 from manila.share.drivers.emc import driver as emcdriver
 from manila.share.drivers.emc.plugins import base
 from manila import test
@@ -113,7 +113,7 @@ class EMCShareFrameworkTestCase(test.TestCase):
         self.configuration.append_config_values = mock.Mock(return_value=0)
         self.configuration.share_backend_name = FAKE_BACKEND
         self.mock_object(self.configuration, 'safe_get', self._fake_safe_get)
-        self.mock_object(driver.ShareDriver, '__init__')
+        self.mock_object(network, 'API')
         self.driver = emcdriver.EMCShareDriver(
             configuration=self.configuration)
 
@@ -137,6 +137,7 @@ class EMCShareFrameworkTestCase(test.TestCase):
         data['free_capacity_gb'] = 'infinite'
         data['reserved_percentage'] = 0
         data['QoS_support'] = False
+        data['pools'] = None
         self.assertEqual(data, self.driver._stats)
 
     def _fake_safe_get(self, value):

manila/tests/share/drivers/hds/test_sop.py

@@ -467,6 +467,7 @@ class SopShareDriverTestCase(test.TestCase):
             'QoS_support': False,
             'total_capacity_gb': 1234,
             'free_capacity_gb': 2345,
+            'pools': None,
         }
         self.mock_object(self._driver, '_get_sop_filesystem_stats',
                          mock.Mock(return_value=(1234, 2345)))

manila/tests/share/drivers/hp/test_hp_3par_driver.py

@@ -441,6 +441,7 @@ class HP3ParDriverTestCase(test.TestCase):
             'storage_protocol': 'NFS_CIFS',
             'total_capacity_gb': expected_capacity,
             'vendor_name': 'HP',
+            'pools': None,
         }
 
         result = self.driver.get_share_stats(refresh=True)

manila/tests/share/drivers/test_glusterfs.py

@@ -533,6 +533,7 @@ class GlusterfsShareDriverTestCase(test.TestCase):
             'QoS_support': False,
             'total_capacity_gb': 2,
             'free_capacity_gb': 2,
+            'pools': None,
         }
         test_statvfs = mock.Mock(f_frsize=4096, f_blocks=524288,
                                  f_bavail=524288)

manila/tests/share/drivers/test_glusterfs_native.py

@@ -1062,6 +1062,7 @@ class GlusterfsNativeShareDriverTestCase(test.TestCase):
             'QoS_support': False,
             'total_capacity_gb': 'infinite',
             'free_capacity_gb': 'infinite',
+            'pools': None,
         }
 
         self._driver._update_share_stats()

manila/tests/share/test_driver.py

@@ -50,7 +50,7 @@ class ShareDriverTestCase(test.TestCase):
         self.mock_object(self.utils, 'execute', fake_execute_with_raise)
         self.time = time
         self.mock_object(self.time, 'sleep', fake_sleep)
-        self.mock_object(driver.CONF, 'driver_handles_share_servers', True)
+        driver.CONF.set_default('driver_handles_share_servers', True)
 
     def test__try_execute(self):
         execute_mixin = ShareDriverWithExecuteMixin(
@@ -199,3 +199,10 @@ class ShareDriverTestCase(test.TestCase):
             self.assertTrue(callable(getattr(obj, attr)))
 
         assert_is_callable(share_driver, method)
+
+    @ddt.data(True, False)
+    def test_get_share_server_pools(self, value):
+        driver.CONF.set_default('driver_handles_share_servers', value)
+        share_driver = driver.ShareDriver(value)
+        self.assertEqual([],
+                         share_driver.get_share_server_pools('fake_server'))

manila/tests/share/test_manager.py

@@ -1687,3 +1687,73 @@ class ShareManagerTestCase(test.TestCase):
         manager.db.share_update.assert_called_once_with(
             mock.ANY, share_id, shr_update
         )
+
+    def test_report_driver_status_driver_handles_ss_false(self):
+        fake_stats = {'field': 'val'}
+        fake_pool = {'name': 'pool1'}
+        self.share_manager.last_capabilities = {'field': 'old_val'}
+
+        self.mock_object(self.share_manager, 'driver', mock.Mock())
+        driver = self.share_manager.driver
+
+        driver.get_share_stats = mock.Mock(return_value=fake_stats)
+        self.mock_object(db, 'share_server_get_all_by_host', mock.Mock())
+        driver.driver_handles_share_servers = False
+        driver.get_share_server_pools = mock.Mock(return_value=fake_pool)
+
+        self.share_manager._report_driver_status(self.context)
+
+        driver.get_share_stats.assert_called_once_with(
+            refresh=True)
+        self.assertFalse(db.share_server_get_all_by_host.called)
+        self.assertFalse(driver.get_share_server_pools.called)
+        self.assertEqual(fake_stats, self.share_manager.last_capabilities)
+
+    def test_report_driver_status_driver_handles_ss(self):
+        fake_stats = {'field': 'val'}
+        fake_ss = {'id': '1234'}
+        fake_pool = {'name': 'pool1'}
+
+        self.mock_object(self.share_manager, 'driver', mock.Mock())
+        driver = self.share_manager.driver
+
+        driver.get_share_stats = mock.Mock(return_value=fake_stats)
+        self.mock_object(db, 'share_server_get_all_by_host', mock.Mock(
+            return_value=[fake_ss]))
+        driver.driver_handles_share_servers = True
+        driver.get_share_server_pools = mock.Mock(return_value=fake_pool)
+
+        self.share_manager._report_driver_status(self.context)
+
+        driver.get_share_stats.assert_called_once_with(refresh=True)
+        db.share_server_get_all_by_host.assert_called_once_with(
+            self.context,
+            self.share_manager.host)
+        driver.get_share_server_pools.assert_called_once_with(fake_ss)
+        expected_stats = {
+            'field': 'val',
+            'server_pools_mapping': {
+                '1234': fake_pool},
+        }
+        self.assertEqual(expected_stats, self.share_manager.last_capabilities)
+
+    def test_report_driver_status_empty_share_stats(self):
+        old_capabilities = {'field': 'old_val'}
+        fake_pool = {'name': 'pool1'}
+        self.share_manager.last_capabilities = old_capabilities
+
+        self.mock_object(self.share_manager, 'driver', mock.Mock())
+        driver = self.share_manager.driver
+
+        driver.get_share_stats = mock.Mock(return_value={})
+        self.mock_object(db, 'share_server_get_all_by_host', mock.Mock())
+        driver.driver_handles_share_servers = True
+        driver.get_share_server_pools = mock.Mock(return_value=fake_pool)
+
+        self.share_manager._report_driver_status(self.context)
+
+        driver.get_share_stats.assert_called_once_with(refresh=True)
+        self.assertFalse(db.share_server_get_all_by_host.called)
+        self.assertFalse(driver.get_share_server_pools.called)
+        self.assertEqual(old_capabilities,
+                         self.share_manager.last_capabilities)