openstack/manila
Code Issues Proposed changes

Add share_type filter support to pool_list

Browse Source 
Administrators intend to get the pool's information filtered
by share type(actually filtered by share_type's *extra_spec*)
more directly.
The blueprint is to add a filter key 'share_type' to cover
this situation.

APIImpact
Implements: blueprint pool-list-by-share-type
Change-Id: Ifd64bb84d03a02aa0a118cc42e1d1b373c439884
This commit is contained in:
zhongjun 2016-09-05 20:37:51 +08:00 committed by zhongjun2
parent 0d3151cac1
commit d5643c75f5
14 changed files with 376 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
api-ref/source/parameters.yaml
View File

@ -351,6 +351,13 @@ share_type_id_2:
in: query
required: false
type: string
share_type_query:
description: |
The share type name or UUID. Allows filtering back end pools based
on the extra-specs in the share type.
in: query
required: false
type: string
snapshot_id_share_response:
description: |
The UUID of the snapshot that was used to create

6
api-ref/source/scheduler-stats.inc
View File

@ -11,7 +11,7 @@ to the scheduler service.
List back-end storage pools
===========================
.. rest_method:: GET /v2/{tenant_id}/scheduler-stats/pools?pool={pool_name}&host={host_name}&backend={backend_name}&capabilities={capabilities}
.. rest_method:: GET /v2/{tenant_id}/scheduler-stats/pools?pool={pool_name}&host={host_name}&backend={backend_name}&capabilities={capabilities}&share_type={share_type}
Lists all back-end storage pools. If search options are provided, the pool
list that is returned is filtered with these options.
@ -29,6 +29,7 @@ Request
- host: backend_host_query
- backend: backend_query
- capabilities: backend_capabilities_query
- share_type: share_type_query
Response parameters
-------------------
@ -50,7 +51,7 @@ Response example
List back-end storage pools with details
========================================
.. rest_method:: GET /v2/{tenant_id}/scheduler-stats/pools/detail?pool={pool_name}&host={host_name}&backend={backend_name}&capabilities={capabilities}
.. rest_method:: GET /v2/{tenant_id}/scheduler-stats/pools/detail?pool={pool_name}&host={host_name}&backend={backend_name}&capabilities={capabilities}&share_type={share_type}
Lists all back-end storage pools with details. If search options are provided,
the pool list that is returned is filtered with these options.
@ -68,6 +69,7 @@ Request
- host: backend_host_query
- backend: backend_query
- capabilities: backend_capabilities_query
- share_type: share_type_query
Response parameters
-------------------

3
manila/api/openstack/api_version_request.py
View File

@ -78,13 +78,14 @@ REST_API_VERSION_HISTORY = """
'force_host_copy' to 'force_host_assisted_migration', removed
'notify' parameter and removed previous migrate_share API support.
Updated reset_task_state API to accept 'None' value.
* 2.23 - Added share_type to filter results of scheduler-stats/pools API.
"""
# The minimum and maximum versions of the API supported
# The default api version request is defined to be the
# minimum version of the API supported.
_MIN_API_VERSION = "2.0"
_MAX_API_VERSION = "2.22"
_MAX_API_VERSION = "2.23"
DEFAULT_API_VERSION = _MIN_API_VERSION

4
manila/api/openstack/rest_api_version_history.rst
View File

@ -138,3 +138,7 @@ user documentation.
'force_host_copy' to 'force_host_assisted_migration', removed 'notify'
parameter and removed previous migrate_share API support. Updated
reset_task_state API to accept 'None' value.
2.23
----
Added share_type to filter results of scheduler-stats/pools API.

32
manila/api/v1/scheduler_stats.py
View File

@ -11,10 +11,14 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from webob import exc
from manila.api.openstack import wsgi
from manila.api.views import scheduler_stats as scheduler_stats_views
from manila import exception
from manila.i18n import _
from manila.scheduler import rpcapi
from manila.share import share_types
class SchedulerStatsController(wsgi.Controller):
@ -27,20 +31,46 @@ class SchedulerStatsController(wsgi.Controller):
self._view_builder_class = scheduler_stats_views.ViewBuilder
super(SchedulerStatsController, self).__init__()
@wsgi.Controller.api_version('1.0', '2.22')
@wsgi.Controller.authorize('index')
def pools_index(self, req):
"""Returns a list of storage pools known to the scheduler."""
return self._pools(req, action='index')
@wsgi.Controller.api_version('2.23') # noqa
@wsgi.Controller.authorize('index')
def pools_index(self, req): # pylint: disable=E0102
return self._pools(req, action='index', enable_share_type=True)
@wsgi.Controller.api_version('1.0', '2.22')
@wsgi.Controller.authorize('detail')
def pools_detail(self, req):
"""Returns a detailed list of storage pools known to the scheduler."""
return self._pools(req, action='detail')
def _pools(self, req, action='index'):
@wsgi.Controller.api_version('2.23') # noqa
@wsgi.Controller.authorize('detail')
def pools_detail(self, req): # pylint: disable=E0102
return self._pools(req, action='detail', enable_share_type=True)
def _pools(self, req, action='index', enable_share_type=False):
context = req.environ['manila.context']
search_opts = {}
search_opts.update(req.GET)
if enable_share_type:
req_share_type = search_opts.pop('share_type', None)
if req_share_type:
try:
share_type = share_types.get_share_type_by_name_or_id(
context, req_share_type)
search_opts['capabilities'] = share_type.get('extra_specs',
{})
except exception.ShareTypeNotFound:
msg = _("Share type %s not found.") % req_share_type
raise exc.HTTPBadRequest(explanation=msg)
pools = self.scheduler_api.get_pools(context, filters=search_opts)
detail = (action == 'detail')
return self._view_builder.pools(pools, detail=detail)

42
manila/scheduler/filters/capabilities.py
View File

@ -16,7 +16,7 @@
from oslo_log import log
from manila.scheduler.filters import base_host
from manila.scheduler.filters import extra_specs_ops
from manila.scheduler import utils
LOG = log.getLogger(__name__)
@ -34,45 +34,7 @@ class CapabilitiesFilter(base_host.BaseHostFilter):
if not extra_specs:
return True
for key, req in extra_specs.items():
# Either not scoped format, or in capabilities scope
scope = key.split(':')
# Ignore scoped (such as vendor-specific) capabilities
if len(scope) > 1 and scope[0] != "capabilities":
continue
# Strip off prefix if spec started with 'capabilities:'
elif scope[0] == "capabilities":
del scope[0]
cap = capabilities
for index in range(len(scope)):
try:
cap = cap.get(scope[index])
except AttributeError:
cap = None
if cap is None:
LOG.debug("Host doesn't provide capability '%(cap)s' "
"listed in the extra specs",
{'cap': scope[index]})
return False
# Make all capability values a list so we can handle lists
cap_list = [cap] if not isinstance(cap, list) else cap
# Loop through capability values looking for any match
for cap_value in cap_list:
if extra_specs_ops.match(cap_value, req):
break
else:
# Nothing matched, so bail out
LOG.debug('Share type extra spec requirement '
'"%(key)s=%(req)s" does not match reported '
'capability "%(cap)s"',
{'key': key, 'req': req, 'cap': cap})
return False
return True
return utils.capabilities_satisfied(capabilities, extra_specs)
def host_passes(self, host_state, filter_properties):
"""Return a list of hosts that can create resource_type."""

9
manila/scheduler/host_manager.py
View File

@ -35,10 +35,12 @@ from manila import db
from manila import exception
from manila.i18n import _LI, _LW
from manila.scheduler.filters import base_host as base_host_filter
from manila.scheduler import utils as scheduler_utils
from manila.scheduler.weighers import base_host as base_host_weigher
from manila.share import utils as share_utils
from manila import utils
host_manager_opts = [
cfg.ListOpt('scheduler_default_filters',
default=[
@ -584,7 +586,6 @@ class HostManager(object):
def get_pools(self, context, filters=None):
"""Returns a dict of all pools on all hosts HostManager knows about."""
self._update_host_state_map(context)
all_pools = []
@ -629,7 +630,11 @@ class HostManager(object):
for filter_key, filter_value in filter_dict.items():
if filter_key not in dict_to_check:
return False
if not re.match(filter_value, dict_to_check.get(filter_key)):
if filter_key == 'capabilities':
if not scheduler_utils.capabilities_satisfied(
dict_to_check.get(filter_key), filter_value):
return False
elif not re.match(filter_value, dict_to_check.get(filter_key)):
return False
return True

46
manila/scheduler/utils.py
View File

@ -14,11 +14,13 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
from oslo_utils import strutils
from manila.scheduler.filters import extra_specs_ops
LOG = log.getLogger(__name__)
def generate_stats(host_state, properties):
"""Generates statistics from host and share data."""
@ -111,3 +113,45 @@ def thin_provisioning(host_state_thin_provisioning):
thin_capability = [host_state_thin_provisioning] if not isinstance(
host_state_thin_provisioning, list) else host_state_thin_provisioning
return True in thin_capability
def capabilities_satisfied(capabilities, extra_specs):
for key, req in extra_specs.items():
# Either not scoped format, or in capabilities scope
scope = key.split(':')
# Ignore scoped (such as vendor-specific) capabilities
if len(scope) > 1 and scope[0] != "capabilities":
continue
# Strip off prefix if spec started with 'capabilities:'
elif scope[0] == "capabilities":
del scope[0]
cap = capabilities
for index in range(len(scope)):
try:
cap = cap.get(scope[index])
except AttributeError:
cap = None
if cap is None:
LOG.debug("Host doesn't provide capability '%(cap)s' "
"listed in the extra specs",
{'cap': scope[index]})
return False
# Make all capability values a list so we can handle lists
cap_list = [cap] if not isinstance(cap, list) else cap
# Loop through capability values looking for any match
for cap_value in cap_list:
if extra_specs_ops.match(cap_value, req):
break
else:
# Nothing matched, so bail out
LOG.debug('Share type extra spec requirement '
'"%(key)s=%(req)s" does not match reported '
'capability "%(cap)s"',
{'key': key, 'req': req, 'cap': cap})
return False
return True

147
manila/tests/api/v1/test_scheduler_stats.py
View File

@ -12,12 +12,17 @@
# License for the specific language governing permissions and limitations
# under the License.
import ddt
import mock
from oslo_utils import uuidutils
from webob import exc
from manila.api.openstack import api_version_request as api_version
from manila.api.v1 import scheduler_stats
from manila import context
from manila import policy
from manila.scheduler import rpcapi
from manila.share import share_types
from manila import test
from manila.tests.api import fakes
@ -58,6 +63,7 @@ FAKE_POOLS = [
]
@ddt.ddt
class SchedulerStatsControllerTestCase(test.TestCase):
def setUp(self):
super(SchedulerStatsControllerTestCase, self).setUp()
@ -99,17 +105,140 @@ class SchedulerStatsControllerTestCase(test.TestCase):
self.mock_policy_check.assert_called_once_with(
self.ctxt, self.resource_name, 'index')
def test_pools_index_with_filters(self):
@ddt.data(('index', False), ('detail', True))
@ddt.unpack
def test_pools_with_share_type_disabled(self, action, detail):
mock_get_pools = self.mock_object(rpcapi.SchedulerAPI,
'get_pools',
mock.Mock(return_value=FAKE_POOLS))
url = '/v1/fake_project/scheduler-stats/pools/detail'
url += '?backend=.%2A&host=host1&pool=pool%2A'
url = '/v1/fake_project/scheduler-stats/pools/%s' % action
url += '?backend=back1&host=host1&pool=pool1'
req = fakes.HTTPRequest.blank(url)
req.environ['manila.context'] = self.ctxt
expected_filters = {
'host': 'host1',
'pool': 'pool1',
'backend': 'back1',
}
if detail:
expected_result = {"pools": FAKE_POOLS}
else:
expected_result = {
'pools': [
{
'name': 'host1@backend1#pool1',
'host': 'host1',
'backend': 'backend1',
'pool': 'pool1',
},
{
'name': 'host1@backend1#pool2',
'host': 'host1',
'backend': 'backend1',
'pool': 'pool2',
}
]
}
result = self.controller._pools(req, action, False)
self.assertDictMatch(result, expected_result)
mock_get_pools.assert_called_once_with(self.ctxt,
filters=expected_filters)
@ddt.data(('index', False, True),
('index', False, False),
('detail', True, True),
('detail', True, False))
@ddt.unpack
def test_pools_with_share_type_enable(self, action, detail, uuid):
mock_get_pools = self.mock_object(rpcapi.SchedulerAPI,
'get_pools',
mock.Mock(return_value=FAKE_POOLS))
if uuid:
share_type = uuidutils.generate_uuid()
else:
share_type = 'test_type'
self.mock_object(
share_types, 'get_share_type_by_name_or_id',
mock.Mock(return_value={'extra_specs':
{'snapshot_support': True}}))
url = '/v1/fake_project/scheduler-stats/pools/%s' % action
url += ('?backend=back1&host=host1&pool=pool1&share_type=%s'
% share_type)
req = fakes.HTTPRequest.blank(url)
req.environ['manila.context'] = self.ctxt
expected_filters = {
'host': 'host1',
'pool': 'pool1',
'backend': 'back1',
'capabilities': {
'snapshot_support': True
}
}
if detail:
expected_result = {"pools": FAKE_POOLS}
else:
expected_result = {
'pools': [
{
'name': 'host1@backend1#pool1',
'host': 'host1',
'backend': 'backend1',
'pool': 'pool1',
},
{
'name': 'host1@backend1#pool2',
'host': 'host1',
'backend': 'backend1',
'pool': 'pool2',
}
]
}
result = self.controller._pools(req, action, True)
self.assertDictMatch(result, expected_result)
mock_get_pools.assert_called_once_with(self.ctxt,
filters=expected_filters)
@ddt.data('index', 'detail')
def test_pools_with_share_type_not_found(self, action):
url = '/v1/fake_project/scheduler-stats/pools/%s' % action
url += '?backend=.%2A&host=host1&pool=pool%2A&share_type=fake_name_1'
req = fakes.HTTPRequest.blank(url)
self.assertRaises(exc.HTTPBadRequest,
self.controller._pools,
req, action, True)
@ddt.data("1.0", "2.22", "2.23")
def test_pools_index_with_filters(self, microversion):
mock_get_pools = self.mock_object(rpcapi.SchedulerAPI,
'get_pools',
mock.Mock(return_value=FAKE_POOLS))
self.mock_object(
share_types, 'get_share_type_by_name',
mock.Mock(return_value={'extra_specs':
{'snapshot_support': True}}))
url = '/v1/fake_project/scheduler-stats/pools/detail'
url += '?backend=.%2A&host=host1&pool=pool%2A&share_type=test_type'
req = fakes.HTTPRequest.blank(url, version=microversion)
req.environ['manila.context'] = self.ctxt
result = self.controller.pools_index(req)
expected = {
@ -128,7 +257,17 @@ class SchedulerStatsControllerTestCase(test.TestCase):
}
]
}
expected_filters = {'host': 'host1', 'pool': 'pool*', 'backend': '.*'}
expected_filters = {
'host': 'host1',
'pool': 'pool*',
'backend': '.*',
'share_type': 'test_type',
}
if (api_version.APIVersionRequest(microversion) >=
api_version.APIVersionRequest('2.23')):
expected_filters.update(
{'capabilities': {'snapshot_support': True}})
expected_filters.pop('share_type', None)
self.assertDictMatch(result, expected)
mock_get_pools.assert_called_once_with(self.ctxt,

21
manila/tests/scheduler/test_host_manager.py
View File

@ -524,7 +524,8 @@ class HostManagerTestCase(test.TestCase):
res = self.host_manager.get_pools(
context=fake_context,
filters={'host': 'host2', 'pool': 'pool*'})
filters={'host': 'host2', 'pool': 'pool*',
'capabilities': {'dedupe': 'False'}})
expected = [
{
@ -562,22 +563,36 @@ class HostManagerTestCase(test.TestCase):
None,
{},
{'key1': 'value1'},
{'capabilities': {'dedupe': 'False'}},
{'capabilities': {'dedupe': '<is> False'}},
{'key1': 'value1', 'key2': 'value*'},
{'key1': '.*', 'key2': '.*'},
)
def test_passes_filters_true(self, filter):
data = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
data = {
'key1': 'value1',
'key2': 'value2',
'key3': 'value3',
'capabilities': {'dedupe': False},
}
self.assertTrue(self.host_manager._passes_filters(data, filter))
@ddt.data(
{'key1': 'value$'},
{'key4': 'value'},
{'capabilities': {'dedupe': 'True'}},
{'capabilities': {'dedupe': '<is> True'}},
{'key1': 'value1.+', 'key2': 'value*'},
)
def test_passes_filters_false(self, filter):
data = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
data = {
'key1': 'value1',
'key2': 'value2',
'key3': 'value3',
'capabilities': {'dedupe': False},
}
self.assertFalse(self.host_manager._passes_filters(data, filter))

2
manila_tempest_tests/config.py
View File

@ -30,7 +30,7 @@ ShareGroup = [
help="The minimum api microversion is configured to be the "
"value of the minimum microversion supported by Manila."),
cfg.StrOpt("max_api_microversion",
default="2.22",
default="2.23",
help="The maximum api microversion is configured to be the "
"value of the latest microversion supported by Manila."),
cfg.StrOpt("region",

55
manila_tempest_tests/tests/api/admin/test_share_type_filter.py Normal file
View File

@ -0,0 +1,55 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ddt
from tempest.lib.common.utils import data_utils
from testtools import testcase as tc
from manila_tempest_tests.tests.api import base
@ddt.ddt
class ShareTypeFilterTest(base.BaseSharesAdminTest):
@classmethod
def _create_share_type(cls):
name = data_utils.rand_name("unique_st_name")
extra_specs = cls.add_required_extra_specs_to_dict()
return cls.create_share_type(
name, extra_specs=extra_specs,
client=cls.admin_client)
@classmethod
def resource_setup(cls):
super(ShareTypeFilterTest, cls).resource_setup()
cls.admin_client = cls.shares_v2_client
cls.st = cls._create_share_type()
@tc.attr(base.TAG_POSITIVE, base.TAG_API_WITH_BACKEND)
@base.skip_if_microversion_not_supported("2.23")
@ddt.data(True, False)
def test_get_pools_with_share_type_filter_with_detail(self, detail):
share_type = self.st["share_type"]["id"]
search_opts = {"share_type": share_type}
kwargs = {'search_opts': search_opts}
if detail:
kwargs.update({'detail': True})
pools = self.admin_client.list_pools(**kwargs)['pools']
for pool in pools:
pool_keys = pool.keys()
self.assertIn("name", pool_keys)
self.assertIn("host", pool_keys)
self.assertIn("backend", pool_keys)
self.assertIn("pool", pool_keys)
self.assertIs(detail, "capabilities" in pool_keys)

54
manila_tempest_tests/tests/api/admin/test_share_type_filter_negative.py Normal file
View File

@ -0,0 +1,54 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ddt
from oslo_utils import uuidutils
from tempest import config
from tempest.lib.common.utils import data_utils
from testtools import testcase as tc
from manila_tempest_tests.tests.api import base
CONF = config.CONF
@ddt.ddt
class ShareTypeFilterNegativeTest(base.BaseSharesAdminTest):
@classmethod
def _create_share_type(cls):
name = data_utils.rand_name("unique_st_name")
extra_specs = {
'share_backend_name': uuidutils.generate_uuid(),
}
extra_specs = cls.add_required_extra_specs_to_dict(
extra_specs=extra_specs)
return cls.create_share_type(
name, extra_specs=extra_specs,
client=cls.admin_client)
@classmethod
def resource_setup(cls):
super(ShareTypeFilterNegativeTest, cls).resource_setup()
cls.admin_client = cls.shares_v2_client
cls.st = cls._create_share_type()
@tc.attr(base.TAG_NEGATIVE, base.TAG_API_WITH_BACKEND)
@base.skip_if_microversion_not_supported("2.23")
@ddt.data(True, False)
def test_get_pools_invalid_share_type_filter_with_detail(self, detail):
share_type = self.st["share_type"]["name"]
search_opts = {"share_type": share_type}
pools = self.admin_client.list_pools(
detail=detail, search_opts=search_opts)['pools']
self.assertEmpty(pools)

3
releasenotes/notes/add-share-type-filter-to-pool-list-api-267614b4d93j12de.yaml Normal file
View File

@ -0,0 +1,3 @@
---
features:
- Added share_type to filter results of scheduler-stats/pools API.