Add share_type filter support to pool_list

Administrators intend to get the pool's information filtered
by share type(actually filtered by share_type's *extra_spec*)
more directly.
The blueprint is to add a filter key 'share_type' to cover
this situation.

APIImpact
Implements: blueprint pool-list-by-share-type
Change-Id: Ifd64bb84d03a02aa0a118cc42e1d1b373c439884

api-ref/source/parameters.yaml

@@ -351,6 +351,13 @@ share_type_id_2:
   in: query
   required: false
   type: string
+share_type_query:
+  description: |
+    The share type name or UUID. Allows filtering back end pools based
+    on the extra-specs in the share type.
+  in: query
+  required: false
+  type: string
 snapshot_id_share_response:
   description: |
     The UUID of the snapshot that was used to create

api-ref/source/scheduler-stats.inc

@@ -11,7 +11,7 @@ to the scheduler service.
 List back-end storage pools
 ===========================
 
-.. rest_method::  GET /v2/{tenant_id}/scheduler-stats/pools?pool={pool_name}&host={host_name}&backend={backend_name}&capabilities={capabilities}
+.. rest_method::  GET /v2/{tenant_id}/scheduler-stats/pools?pool={pool_name}&host={host_name}&backend={backend_name}&capabilities={capabilities}&share_type={share_type}
 
 Lists all back-end storage pools. If search options are provided, the pool
 list that is returned is filtered with these options.
@@ -29,6 +29,7 @@ Request
    - host: backend_host_query
    - backend: backend_query
    - capabilities: backend_capabilities_query
+   - share_type: share_type_query
 
 Response parameters
 -------------------
@@ -50,7 +51,7 @@ Response example
 List back-end storage pools with details
 ========================================
 
-.. rest_method::  GET /v2/{tenant_id}/scheduler-stats/pools/detail?pool={pool_name}&host={host_name}&backend={backend_name}&capabilities={capabilities}
+.. rest_method::  GET /v2/{tenant_id}/scheduler-stats/pools/detail?pool={pool_name}&host={host_name}&backend={backend_name}&capabilities={capabilities}&share_type={share_type}
 
 Lists all back-end storage pools with details. If search options are provided,
 the pool list that is returned is filtered with these options.
@@ -68,6 +69,7 @@ Request
    - host: backend_host_query
    - backend: backend_query
    - capabilities: backend_capabilities_query
+   - share_type: share_type_query
 
 Response parameters
 -------------------

manila/api/openstack/api_version_request.py

@@ -78,13 +78,14 @@ REST_API_VERSION_HISTORY = """
             'force_host_copy' to 'force_host_assisted_migration', removed
             'notify' parameter and removed previous migrate_share API support.
             Updated reset_task_state API to accept 'None' value.
+    * 2.23 - Added share_type to filter results of scheduler-stats/pools API.
 """
 
 # The minimum and maximum versions of the API supported
 # The default api version request is defined to be the
 # minimum version of the API supported.
 _MIN_API_VERSION = "2.0"
-_MAX_API_VERSION = "2.22"
+_MAX_API_VERSION = "2.23"
 DEFAULT_API_VERSION = _MIN_API_VERSION
 
 

manila/api/openstack/rest_api_version_history.rst

@@ -138,3 +138,7 @@ user documentation.
   'force_host_copy' to 'force_host_assisted_migration', removed 'notify'
   parameter and removed previous migrate_share API support. Updated
   reset_task_state API to accept 'None' value.
+
+2.23
+----
+  Added share_type to filter results of scheduler-stats/pools API.

manila/api/v1/scheduler_stats.py

@@ -11,10 +11,14 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+from webob import exc
 
 from manila.api.openstack import wsgi
 from manila.api.views import scheduler_stats as scheduler_stats_views
+from manila import exception
+from manila.i18n import _
 from manila.scheduler import rpcapi
+from manila.share import share_types
 
 
 class SchedulerStatsController(wsgi.Controller):
@@ -27,20 +31,46 @@ class SchedulerStatsController(wsgi.Controller):
         self._view_builder_class = scheduler_stats_views.ViewBuilder
         super(SchedulerStatsController, self).__init__()
 
+    @wsgi.Controller.api_version('1.0', '2.22')
     @wsgi.Controller.authorize('index')
     def pools_index(self, req):
         """Returns a list of storage pools known to the scheduler."""
         return self._pools(req, action='index')
 
+    @wsgi.Controller.api_version('2.23')   # noqa
+    @wsgi.Controller.authorize('index')
+    def pools_index(self, req):  # pylint: disable=E0102
+        return self._pools(req, action='index', enable_share_type=True)
+
+    @wsgi.Controller.api_version('1.0', '2.22')
     @wsgi.Controller.authorize('detail')
     def pools_detail(self, req):
         """Returns a detailed list of storage pools known to the scheduler."""
         return self._pools(req, action='detail')
 
-    def _pools(self, req, action='index'):
+    @wsgi.Controller.api_version('2.23')   # noqa
+    @wsgi.Controller.authorize('detail')
+    def pools_detail(self, req):  # pylint: disable=E0102
+        return self._pools(req, action='detail', enable_share_type=True)
+
+    def _pools(self, req, action='index', enable_share_type=False):
         context = req.environ['manila.context']
         search_opts = {}
         search_opts.update(req.GET)
+
+        if enable_share_type:
+            req_share_type = search_opts.pop('share_type', None)
+            if req_share_type:
+                try:
+                    share_type = share_types.get_share_type_by_name_or_id(
+                        context, req_share_type)
+
+                    search_opts['capabilities'] = share_type.get('extra_specs',
+                                                                 {})
+                except exception.ShareTypeNotFound:
+                    msg = _("Share type %s not found.") % req_share_type
+                    raise exc.HTTPBadRequest(explanation=msg)
+
         pools = self.scheduler_api.get_pools(context, filters=search_opts)
         detail = (action == 'detail')
         return self._view_builder.pools(pools, detail=detail)

manila/scheduler/filters/capabilities.py

@@ -16,7 +16,7 @@
 from oslo_log import log
 
 from manila.scheduler.filters import base_host
-from manila.scheduler.filters import extra_specs_ops
+from manila.scheduler import utils
 
 LOG = log.getLogger(__name__)
 
@@ -34,45 +34,7 @@ class CapabilitiesFilter(base_host.BaseHostFilter):
         if not extra_specs:
             return True
 
-        for key, req in extra_specs.items():
+        return utils.capabilities_satisfied(capabilities, extra_specs)
-
-            # Either not scoped format, or in capabilities scope
-            scope = key.split(':')
-
-            # Ignore scoped (such as vendor-specific) capabilities
-            if len(scope) > 1 and scope[0] != "capabilities":
-                continue
-            # Strip off prefix if spec started with 'capabilities:'
-            elif scope[0] == "capabilities":
-                del scope[0]
-
-            cap = capabilities
-            for index in range(len(scope)):
-                try:
-                    cap = cap.get(scope[index])
-                except AttributeError:
-                    cap = None
-                if cap is None:
-                    LOG.debug("Host doesn't provide capability '%(cap)s' "
-                              "listed in the extra specs",
-                              {'cap': scope[index]})
-                    return False
-
-            # Make all capability values a list so we can handle lists
-            cap_list = [cap] if not isinstance(cap, list) else cap
-
-            # Loop through capability values looking for any match
-            for cap_value in cap_list:
-                if extra_specs_ops.match(cap_value, req):
-                    break
-            else:
-                # Nothing matched, so bail out
-                LOG.debug('Share type extra spec requirement '
-                          '"%(key)s=%(req)s" does not match reported '
-                          'capability "%(cap)s"',
-                          {'key': key, 'req': req, 'cap': cap})
-                return False
-        return True
 
     def host_passes(self, host_state, filter_properties):
         """Return a list of hosts that can create resource_type."""

manila/scheduler/host_manager.py

@@ -35,10 +35,12 @@ from manila import db
 from manila import exception
 from manila.i18n import _LI, _LW
 from manila.scheduler.filters import base_host as base_host_filter
+from manila.scheduler import utils as scheduler_utils
 from manila.scheduler.weighers import base_host as base_host_weigher
 from manila.share import utils as share_utils
 from manila import utils
 
+
 host_manager_opts = [
     cfg.ListOpt('scheduler_default_filters',
                 default=[
@@ -584,7 +586,6 @@ class HostManager(object):
 
     def get_pools(self, context, filters=None):
         """Returns a dict of all pools on all hosts HostManager knows about."""
-
         self._update_host_state_map(context)
 
         all_pools = []
@@ -629,7 +630,11 @@ class HostManager(object):
         for filter_key, filter_value in filter_dict.items():
             if filter_key not in dict_to_check:
                 return False
-            if not re.match(filter_value, dict_to_check.get(filter_key)):
+            if filter_key == 'capabilities':
+                if not scheduler_utils.capabilities_satisfied(
+                        dict_to_check.get(filter_key), filter_value):
+                    return False
+            elif not re.match(filter_value, dict_to_check.get(filter_key)):
                 return False
 
         return True

manila/scheduler/utils.py

@@ -14,11 +14,13 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-
+from oslo_log import log
 from oslo_utils import strutils
 
 from manila.scheduler.filters import extra_specs_ops
 
+LOG = log.getLogger(__name__)
+
 
 def generate_stats(host_state, properties):
     """Generates statistics from host and share data."""
@@ -111,3 +113,45 @@ def thin_provisioning(host_state_thin_provisioning):
     thin_capability = [host_state_thin_provisioning] if not isinstance(
         host_state_thin_provisioning, list) else host_state_thin_provisioning
     return True in thin_capability
+
+
+def capabilities_satisfied(capabilities, extra_specs):
+
+    for key, req in extra_specs.items():
+        # Either not scoped format, or in capabilities scope
+        scope = key.split(':')
+
+        # Ignore scoped (such as vendor-specific) capabilities
+        if len(scope) > 1 and scope[0] != "capabilities":
+            continue
+        # Strip off prefix if spec started with 'capabilities:'
+        elif scope[0] == "capabilities":
+            del scope[0]
+
+        cap = capabilities
+        for index in range(len(scope)):
+            try:
+                cap = cap.get(scope[index])
+            except AttributeError:
+                cap = None
+            if cap is None:
+                LOG.debug("Host doesn't provide capability '%(cap)s' "
+                          "listed in the extra specs",
+                          {'cap': scope[index]})
+                return False
+
+        # Make all capability values a list so we can handle lists
+        cap_list = [cap] if not isinstance(cap, list) else cap
+
+        # Loop through capability values looking for any match
+        for cap_value in cap_list:
+            if extra_specs_ops.match(cap_value, req):
+                break
+        else:
+            # Nothing matched, so bail out
+            LOG.debug('Share type extra spec requirement '
+                      '"%(key)s=%(req)s" does not match reported '
+                      'capability "%(cap)s"',
+                      {'key': key, 'req': req, 'cap': cap})
+            return False
+    return True

manila/tests/api/v1/test_scheduler_stats.py

@@ -12,12 +12,17 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import ddt
 import mock
+from oslo_utils import uuidutils
+from webob import exc
 
+from manila.api.openstack import api_version_request as api_version
 from manila.api.v1 import scheduler_stats
 from manila import context
 from manila import policy
 from manila.scheduler import rpcapi
+from manila.share import share_types
 from manila import test
 from manila.tests.api import fakes
 
@@ -58,6 +63,7 @@ FAKE_POOLS = [
 ]
 
 
+@ddt.ddt
 class SchedulerStatsControllerTestCase(test.TestCase):
     def setUp(self):
         super(SchedulerStatsControllerTestCase, self).setUp()
@@ -99,17 +105,140 @@ class SchedulerStatsControllerTestCase(test.TestCase):
         self.mock_policy_check.assert_called_once_with(
             self.ctxt, self.resource_name, 'index')
 
-    def test_pools_index_with_filters(self):
+    @ddt.data(('index', False), ('detail', True))
+    @ddt.unpack
+    def test_pools_with_share_type_disabled(self, action, detail):
         mock_get_pools = self.mock_object(rpcapi.SchedulerAPI,
                                           'get_pools',
                                           mock.Mock(return_value=FAKE_POOLS))
 
-        url = '/v1/fake_project/scheduler-stats/pools/detail'
+        url = '/v1/fake_project/scheduler-stats/pools/%s' % action
-        url += '?backend=.%2A&host=host1&pool=pool%2A'
+        url += '?backend=back1&host=host1&pool=pool1'
 
         req = fakes.HTTPRequest.blank(url)
         req.environ['manila.context'] = self.ctxt
 
+        expected_filters = {
+            'host': 'host1',
+            'pool': 'pool1',
+            'backend': 'back1',
+        }
+
+        if detail:
+            expected_result = {"pools": FAKE_POOLS}
+        else:
+            expected_result = {
+                'pools': [
+                    {
+                        'name': 'host1@backend1#pool1',
+                        'host': 'host1',
+                        'backend': 'backend1',
+                        'pool': 'pool1',
+                    },
+                    {
+                        'name': 'host1@backend1#pool2',
+                        'host': 'host1',
+                        'backend': 'backend1',
+                        'pool': 'pool2',
+                    }
+                ]
+            }
+
+        result = self.controller._pools(req, action, False)
+
+        self.assertDictMatch(result, expected_result)
+        mock_get_pools.assert_called_once_with(self.ctxt,
+                                               filters=expected_filters)
+
+    @ddt.data(('index', False, True),
+              ('index', False, False),
+              ('detail', True, True),
+              ('detail', True, False))
+    @ddt.unpack
+    def test_pools_with_share_type_enable(self, action, detail, uuid):
+        mock_get_pools = self.mock_object(rpcapi.SchedulerAPI,
+                                          'get_pools',
+                                          mock.Mock(return_value=FAKE_POOLS))
+
+        if uuid:
+            share_type = uuidutils.generate_uuid()
+        else:
+            share_type = 'test_type'
+
+        self.mock_object(
+            share_types, 'get_share_type_by_name_or_id',
+            mock.Mock(return_value={'extra_specs':
+                                    {'snapshot_support': True}}))
+
+        url = '/v1/fake_project/scheduler-stats/pools/%s' % action
+        url += ('?backend=back1&host=host1&pool=pool1&share_type=%s'
+                % share_type)
+
+        req = fakes.HTTPRequest.blank(url)
+        req.environ['manila.context'] = self.ctxt
+
+        expected_filters = {
+            'host': 'host1',
+            'pool': 'pool1',
+            'backend': 'back1',
+            'capabilities': {
+                'snapshot_support': True
+            }
+        }
+
+        if detail:
+            expected_result = {"pools": FAKE_POOLS}
+        else:
+            expected_result = {
+                'pools': [
+                    {
+                        'name': 'host1@backend1#pool1',
+                        'host': 'host1',
+                        'backend': 'backend1',
+                        'pool': 'pool1',
+                    },
+                    {
+                        'name': 'host1@backend1#pool2',
+                        'host': 'host1',
+                        'backend': 'backend1',
+                        'pool': 'pool2',
+                    }
+                ]
+            }
+
+        result = self.controller._pools(req, action, True)
+
+        self.assertDictMatch(result, expected_result)
+        mock_get_pools.assert_called_once_with(self.ctxt,
+                                               filters=expected_filters)
+
+    @ddt.data('index', 'detail')
+    def test_pools_with_share_type_not_found(self, action):
+        url = '/v1/fake_project/scheduler-stats/pools/%s' % action
+        url += '?backend=.%2A&host=host1&pool=pool%2A&share_type=fake_name_1'
+
+        req = fakes.HTTPRequest.blank(url)
+
+        self.assertRaises(exc.HTTPBadRequest,
+                          self.controller._pools,
+                          req, action, True)
+
+    @ddt.data("1.0", "2.22", "2.23")
+    def test_pools_index_with_filters(self, microversion):
+        mock_get_pools = self.mock_object(rpcapi.SchedulerAPI,
+                                          'get_pools',
+                                          mock.Mock(return_value=FAKE_POOLS))
+        self.mock_object(
+            share_types, 'get_share_type_by_name',
+            mock.Mock(return_value={'extra_specs':
+                                    {'snapshot_support': True}}))
+
+        url = '/v1/fake_project/scheduler-stats/pools/detail'
+        url += '?backend=.%2A&host=host1&pool=pool%2A&share_type=test_type'
+
+        req = fakes.HTTPRequest.blank(url, version=microversion)
+        req.environ['manila.context'] = self.ctxt
+
         result = self.controller.pools_index(req)
 
         expected = {
@@ -128,7 +257,17 @@ class SchedulerStatsControllerTestCase(test.TestCase):
                 }
             ]
         }
-        expected_filters = {'host': 'host1', 'pool': 'pool*', 'backend': '.*'}
+        expected_filters = {
+            'host': 'host1',
+            'pool': 'pool*',
+            'backend': '.*',
+            'share_type': 'test_type',
+        }
+        if (api_version.APIVersionRequest(microversion) >=
+                api_version.APIVersionRequest('2.23')):
+            expected_filters.update(
+                {'capabilities': {'snapshot_support': True}})
+            expected_filters.pop('share_type', None)
 
         self.assertDictMatch(result, expected)
         mock_get_pools.assert_called_once_with(self.ctxt,

manila/tests/scheduler/test_host_manager.py

@@ -524,7 +524,8 @@ class HostManagerTestCase(test.TestCase):
 
             res = self.host_manager.get_pools(
                 context=fake_context,
-                filters={'host': 'host2', 'pool': 'pool*'})
+                filters={'host': 'host2', 'pool': 'pool*',
+                         'capabilities': {'dedupe': 'False'}})
 
             expected = [
                 {
@@ -562,22 +563,36 @@ class HostManagerTestCase(test.TestCase):
         None,
         {},
         {'key1': 'value1'},
+        {'capabilities': {'dedupe': 'False'}},
+        {'capabilities': {'dedupe': '<is> False'}},
         {'key1': 'value1', 'key2': 'value*'},
         {'key1': '.*', 'key2': '.*'},
     )
     def test_passes_filters_true(self, filter):
 
-        data = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
+        data = {
+            'key1': 'value1',
+            'key2': 'value2',
+            'key3': 'value3',
+            'capabilities': {'dedupe': False},
+        }
         self.assertTrue(self.host_manager._passes_filters(data, filter))
 
     @ddt.data(
         {'key1': 'value$'},
         {'key4': 'value'},
+        {'capabilities': {'dedupe': 'True'}},
+        {'capabilities': {'dedupe': '<is> True'}},
         {'key1': 'value1.+', 'key2': 'value*'},
     )
     def test_passes_filters_false(self, filter):
 
-        data = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
+        data = {
+            'key1': 'value1',
+            'key2': 'value2',
+            'key3': 'value3',
+            'capabilities': {'dedupe': False},
+        }
         self.assertFalse(self.host_manager._passes_filters(data, filter))
 
 

manila_tempest_tests/config.py

@@ -30,7 +30,7 @@ ShareGroup = [
                help="The minimum api microversion is configured to be the "
                     "value of the minimum microversion supported by Manila."),
     cfg.StrOpt("max_api_microversion",
-               default="2.22",
+               default="2.23",
                help="The maximum api microversion is configured to be the "
                     "value of the latest microversion supported by Manila."),
     cfg.StrOpt("region",

manila_tempest_tests/tests/api/admin/test_share_type_filter.py

@@ -0,0 +1,55 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import ddt
+from tempest.lib.common.utils import data_utils
+from testtools import testcase as tc
+
+from manila_tempest_tests.tests.api import base
+
+
+@ddt.ddt
+class ShareTypeFilterTest(base.BaseSharesAdminTest):
+
+    @classmethod
+    def _create_share_type(cls):
+        name = data_utils.rand_name("unique_st_name")
+        extra_specs = cls.add_required_extra_specs_to_dict()
+        return cls.create_share_type(
+            name, extra_specs=extra_specs,
+            client=cls.admin_client)
+
+    @classmethod
+    def resource_setup(cls):
+        super(ShareTypeFilterTest, cls).resource_setup()
+        cls.admin_client = cls.shares_v2_client
+        cls.st = cls._create_share_type()
+
+    @tc.attr(base.TAG_POSITIVE, base.TAG_API_WITH_BACKEND)
+    @base.skip_if_microversion_not_supported("2.23")
+    @ddt.data(True, False)
+    def test_get_pools_with_share_type_filter_with_detail(self, detail):
+        share_type = self.st["share_type"]["id"]
+        search_opts = {"share_type": share_type}
+        kwargs = {'search_opts': search_opts}
+
+        if detail:
+            kwargs.update({'detail': True})
+
+        pools = self.admin_client.list_pools(**kwargs)['pools']
+        for pool in pools:
+            pool_keys = pool.keys()
+            self.assertIn("name", pool_keys)
+            self.assertIn("host", pool_keys)
+            self.assertIn("backend", pool_keys)
+            self.assertIn("pool", pool_keys)
+            self.assertIs(detail, "capabilities" in pool_keys)

manila_tempest_tests/tests/api/admin/test_share_type_filter_negative.py

@@ -0,0 +1,54 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import ddt
+from oslo_utils import uuidutils
+from tempest import config
+from tempest.lib.common.utils import data_utils
+from testtools import testcase as tc
+
+from manila_tempest_tests.tests.api import base
+
+CONF = config.CONF
+
+
+@ddt.ddt
+class ShareTypeFilterNegativeTest(base.BaseSharesAdminTest):
+
+    @classmethod
+    def _create_share_type(cls):
+        name = data_utils.rand_name("unique_st_name")
+        extra_specs = {
+            'share_backend_name': uuidutils.generate_uuid(),
+        }
+        extra_specs = cls.add_required_extra_specs_to_dict(
+            extra_specs=extra_specs)
+        return cls.create_share_type(
+            name, extra_specs=extra_specs,
+            client=cls.admin_client)
+
+    @classmethod
+    def resource_setup(cls):
+        super(ShareTypeFilterNegativeTest, cls).resource_setup()
+        cls.admin_client = cls.shares_v2_client
+        cls.st = cls._create_share_type()
+
+    @tc.attr(base.TAG_NEGATIVE, base.TAG_API_WITH_BACKEND)
+    @base.skip_if_microversion_not_supported("2.23")
+    @ddt.data(True, False)
+    def test_get_pools_invalid_share_type_filter_with_detail(self, detail):
+        share_type = self.st["share_type"]["name"]
+        search_opts = {"share_type": share_type}
+        pools = self.admin_client.list_pools(
+            detail=detail, search_opts=search_opts)['pools']
+
+        self.assertEmpty(pools)

releasenotes/notes/add-share-type-filter-to-pool-list-api-267614b4d93j12de.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Added share_type to filter results of scheduler-stats/pools API.