openstack
/
deb-cinder
Code Issues Proposed changes

Merge "Support 'LIKE' operator to filter resource"

This commit is contained in:
Jenkins 2017-05-23 19:11:13 +00:00 committed by Gerrit Code Review
parent d9b8104153 6df8415411
commit 39d35e15bd
17 changed files with 248 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cinder/api/common.py
View File

@ -67,6 +67,7 @@ CONF.register_opts(api_common_opts)
LOG = logging.getLogger(__name__)
_FILTERS_COLLECTION = None
FILTERING_VERSION = '3.31'
LIKE_FILTER_VERSION = '3.34'
METADATA_TYPES = enum.Enum('METADATA_TYPES', 'user image')
@ -446,7 +447,8 @@ def get_enabled_resource_filters(resource=None):
return {}
def reject_invalid_filters(context, filters, resource):
def reject_invalid_filters(context, filters, resource,
enable_like_filter=False):
if context.is_admin:
# Allow all options
return
@ -458,8 +460,14 @@ def reject_invalid_filters(context, filters, resource):
configured_filters = []
invalid_filters = []
for key in filters.copy().keys():
if key not in configured_filters:
invalid_filters.append(key)
if not enable_like_filter:
if key not in configured_filters:
invalid_filters.append(key)
else:
# If 'key~' is configured, both 'key' and 'key~' is valid.
if (key not in configured_filters or
"%s~" % key not in configured_filters):
invalid_filters.append(key)
if invalid_filters:
raise webob.exc.HTTPBadRequest(
explanation=_('Invalid filters %s are found in query '
@ -473,7 +481,11 @@ def process_general_filtering(resource):
filters = kwargs.get('filters')
context = kwargs.get('context')
if req_version.matches(FILTERING_VERSION):
reject_invalid_filters(context, filters, resource)
support_like = False
if req_version.matches(LIKE_FILTER_VERSION):
support_like = True
reject_invalid_filters(context, filters,
resource, support_like)
else:
process_non_general_filtering(*args, **kwargs)
return _decorator

6
cinder/api/openstack/api_version_request.py
View File

@ -86,6 +86,10 @@ REST_API_VERSION_HISTORY = """
* 3.33 - Add ``resource_filters`` API to retrieve configured
resource filters.
* 3.34 - Add like filter support in ``volume``, ``backup``, ``snapshot``,
``message``, ``attachment``, ``group`` and ``group-snapshot``
list APIs.
"""
# The minimum and maximum versions of the API supported
@ -93,7 +97,7 @@ REST_API_VERSION_HISTORY = """
# minimum version of the API supported.
# Explicitly using /v1 or /v2 endpoints will still work
_MIN_API_VERSION = "3.0"
_MAX_API_VERSION = "3.33"
_MAX_API_VERSION = "3.34"
_LEGACY_API_VERSION1 = "1.0"
_LEGACY_API_VERSION2 = "2.0"

5
cinder/api/openstack/rest_api_version_history.rst
View File

@ -308,3 +308,8 @@ user documentation.
3.33
----
Add ``resource_filters`` API to retrieve configured resource filters.
3.34
----
Add like filter support in ``volume``, ``backup``, ``snapshot``, ``message``,
``attachment``, ``group`` and ``group-snapshot`` list APIs.

7
cinder/api/v3/group_snapshots.py
View File

@ -114,8 +114,11 @@ class GroupSnapshotsController(wsgi.Controller):
marker, limit, offset = common.get_pagination_params(filters)
sort_keys, sort_dirs = common.get_sort_params(filters)
if req.api_version_request.matches(common.FILTERING_VERSION):
common.reject_invalid_filters(context, filters, 'group_snapshot')
if req_version.matches(common.FILTERING_VERSION):
support_like = (True if req_version.matches(
common.LIKE_FILTER_VERSION) else False)
common.reject_invalid_filters(context, filters, 'group_snapshot',
support_like)
group_snapshots = self.group_snapshot_api.get_all_group_snapshots(
context, filters=filters, marker=marker, limit=limit,

8
cinder/api/v3/groups.py
View File

@ -163,12 +163,16 @@ class GroupsController(wsgi.Controller):
"""Returns a list of groups through view builder."""
context = req.environ['cinder.context']
filters = req.params.copy()
api_version = req.api_version_request
marker, limit, offset = common.get_pagination_params(filters)
sort_keys, sort_dirs = common.get_sort_params(filters)
filters.pop('list_volume', None)
if req.api_version_request.matches(common.FILTERING_VERSION):
common.reject_invalid_filters(context, filters, 'group')
if api_version.matches(common.FILTERING_VERSION):
support_like = (True if api_version.matches(
common.LIKE_FILTER_VERSION) else False)
common.reject_invalid_filters(context, filters, 'group',
support_like)
groups = self.group_api.get_all(
context, filters=filters, marker=marker, limit=limit,

10
cinder/api/v3/messages.py
View File

@ -80,6 +80,7 @@ class MessagesController(wsgi.Controller):
def index(self, req):
"""Returns a list of messages, transformed through view builder."""
context = req.environ['cinder.context']
api_version = req.api_version_request
check_policy(context, 'get_all')
filters = None
marker = None
@ -88,13 +89,16 @@ class MessagesController(wsgi.Controller):
sort_keys = None
sort_dirs = None
if (req.api_version_request.matches("3.5")):
if api_version.matches("3.5"):
filters = req.params.copy()
marker, limit, offset = common.get_pagination_params(filters)
sort_keys, sort_dirs = common.get_sort_params(filters)
if req.api_version_request.matches(common.FILTERING_VERSION):
common.reject_invalid_filters(context, filters, 'message')
if api_version.matches(common.FILTERING_VERSION):
support_like = (True if api_version.matches(
common.LIKE_FILTER_VERSION) else False)
common.reject_invalid_filters(context, filters, 'message',
support_like)
messages = self.message_api.get_all(context, filters=filters,
marker=marker, limit=limit,

47
cinder/db/sqlalchemy/api.py
View File

@ -1725,6 +1725,45 @@ def volume_detached(context, volume_id, attachment_id):
return (volume_updates, attachment_updates)
def _process_model_like_filter(model, query, filters):
"""Applies regex expression filtering to a query.
:param model: model to apply filters to
:param query: query to apply filters to
:param filters: dictionary of filters with regex values
:returns: the updated query.
"""
if query is None:
return query
for key in filters:
column_attr = getattr(model, key)
if 'property' == type(column_attr).__name__:
continue
value = filters[key]
if not isinstance(value, six.string_types):
continue
query = query.filter(column_attr.op('LIKE')(u'%' + value + u'%'))
return query
def apply_like_filters(model):
def decorator_filters(process_exact_filters):
def _decorator(query, filters):
exact_filters = filters.copy()
regex_filters = {}
for key, value in filters.items():
# NOTE(tommylikehu): For inexact match, the filter keys
# are in the format of 'key~=value'
if key.endswith('~'):
exact_filters.pop(key)
regex_filters[key.rstrip('~')] = value
query = process_exact_filters(query, exact_filters)
return _process_model_like_filter(model, query, regex_filters)
return _decorator
return decorator_filters
@require_context
def _volume_get_query(context, session=None, project_only=False,
joined_load=True):
@ -1815,6 +1854,7 @@ def _attachment_get_query(context, session=None, project_only=False):
project_only=project_only).options(joinedload('volume'))
@apply_like_filters(model=models.VolumeAttachment)
def _process_attachment_filters(query, filters):
if filters:
project_id = filters.pop('project_id', None)
@ -2227,6 +2267,7 @@ def _generate_paginate_query(context, session, marker, limit, sort_keys,
offset=offset)
@apply_like_filters(model=models.Volume)
def _process_volume_filters(query, filters):
"""Common filter processing for Volume queries.
@ -2900,6 +2941,7 @@ def _snaps_get_query(context, session=None, project_only=False):
options(joinedload('snapshot_metadata'))
@apply_like_filters(model=models.Snapshot)
def _process_snaps_filters(query, filters):
if filters:
filters = filters.copy()
@ -4914,6 +4956,7 @@ def _backups_get_query(context, session=None, project_only=False):
project_only=project_only)
@apply_like_filters(model=models.Backup)
def _process_backups_filters(query, filters):
if filters:
# Ensure that filters' keys exist on the model
@ -5499,6 +5542,7 @@ def _group_snapshot_get_query(context, session=None, project_only=False):
project_only=project_only)
@apply_like_filters(model=models.Group)
def _process_groups_filters(query, filters):
if filters:
# Ensure that filters' keys exist on the model
@ -5508,6 +5552,7 @@ def _process_groups_filters(query, filters):
return query
@apply_like_filters(model=models.GroupSnapshot)
def _process_group_snapshot_filters(query, filters):
if filters:
# Ensure that filters' keys exist on the model
@ -5784,6 +5829,7 @@ def is_valid_model_filters(model, filters, exclude_list=None):
if exclude_list and key in exclude_list:
continue
try:
key = key.rstrip('~')
getattr(model, key)
except AttributeError:
LOG.debug("'%s' filter key is not valid.", key)
@ -6183,6 +6229,7 @@ def message_get_all(context, filters=None, marker=None, limit=None,
return _translate_messages(results)
@apply_like_filters(model=models.Message)
def _process_messages_filters(query, filters):
if filters:
# Ensure that filters' keys exist on the model

6
cinder/tests/unit/api/v3/test_attachments.py
View File

@ -139,7 +139,7 @@ class AttachmentsAPITestCase(test.TestCase):
self.controller.delete, req,
self.attachment1.id)
@ddt.data('3.30', '3.31')
@ddt.data('3.30', '3.31', '3.34')
@mock.patch('cinder.api.common.reject_invalid_filters')
def test_attachment_list_with_general_filter(self, version, mock_update):
url = '/v3/%s/attachments' % fake.PROJECT_ID
@ -149,8 +149,10 @@ class AttachmentsAPITestCase(test.TestCase):
self.controller.index(req)
if version != '3.30':
support_like = True if version == '3.34' else False
mock_update.assert_called_once_with(req.environ['cinder.context'],
mock.ANY, 'attachment')
mock.ANY, 'attachment',
support_like)
@ddt.data('reserved', 'attached')
@mock.patch.object(volume_rpcapi.VolumeAPI, 'attachment_delete')

6
cinder/tests/unit/api/v3/test_backups.py
View File

@ -85,7 +85,7 @@ class BackupsControllerAPITestCase(test.TestCase):
self.controller.update,
req, fake.BACKUP_ID, body)
@ddt.data('3.30', '3.31')
@ddt.data('3.30', '3.31', '3.34')
@mock.patch('cinder.api.common.reject_invalid_filters')
def test_backup_list_with_general_filter(self, version, mock_update):
url = '/v3/%s/backups' % fake.PROJECT_ID
@ -95,8 +95,10 @@ class BackupsControllerAPITestCase(test.TestCase):
self.controller.index(req)
if version != '3.30':
support_like = True if version == '3.34' else False
mock_update.assert_called_once_with(req.environ['cinder.context'],
mock.ANY, 'backup')
mock.ANY, 'backup',
support_like)
def test_backup_update(self):
backup = test_utils.create_backup(

6
cinder/tests/unit/api/v3/test_group_snapshots.py
View File

@ -183,7 +183,7 @@ class GroupSnapshotsAPITestCase(test.TestCase):
res_dict['group_snapshots'][0].keys())
group_snapshot.destroy()
@ddt.data('3.30', '3.31')
@ddt.data('3.30', '3.31', '3.34')
@mock.patch('cinder.api.common.reject_invalid_filters')
def test_group_snapshot_list_with_general_filter(self,
version, mock_update):
@ -194,8 +194,10 @@ class GroupSnapshotsAPITestCase(test.TestCase):
self.controller.index(req)
if version != '3.30':
support_like = True if version == '3.34' else False
mock_update.assert_called_once_with(req.environ['cinder.context'],
mock.ANY, 'group_snapshot')
mock.ANY, 'group_snapshot',
support_like)
@ddt.data(False, True)
def test_list_group_snapshot_with_filter(self, is_detail):

6
cinder/tests/unit/api/v3/test_groups.py
View File

@ -239,7 +239,7 @@ class GroupsAPITestCase(test.TestCase):
self.assertRaises(exception.GroupNotFound, self.controller.show,
req, fake.WILL_NOT_BE_FOUND_ID)
@ddt.data('3.30', '3.31')
@ddt.data('3.30', '3.31', '3.34')
@mock.patch('cinder.api.common.reject_invalid_filters')
def test_group_list_with_general_filter(self, version, mock_update):
url = '/v3/%s/groups' % fake.PROJECT_ID
@ -249,8 +249,10 @@ class GroupsAPITestCase(test.TestCase):
self.controller.index(req)
if version != '3.30':
support_like = True if version == '3.34' else False
mock_update.assert_called_once_with(req.environ['cinder.context'],
mock.ANY, 'group')
mock.ANY, 'group',
support_like)
def test_list_groups_json(self):
self.group2.group_type_id = fake.GROUP_TYPE2_ID

6
cinder/tests/unit/api/v3/test_messages.py
View File

@ -123,7 +123,7 @@ class MessageApiTest(test.TestCase):
self.assertRaises(exception.MessageNotFound, self.controller.delete,
req, fakes.FAKE_UUID)
@ddt.data('3.30', '3.31')
@ddt.data('3.30', '3.31', '3.34')
@mock.patch('cinder.api.common.reject_invalid_filters')
def test_message_list_with_general_filter(self, version, mock_update):
url = '/v3/%s/messages' % fakes.FAKE_UUID
@ -133,8 +133,10 @@ class MessageApiTest(test.TestCase):
self.controller.index(req)
if version != '3.30':
support_like = True if version == '3.34' else False
mock_update.assert_called_once_with(req.environ['cinder.context'],
mock.ANY, 'message')
mock.ANY, 'message',
support_like)
def test_index(self):
self.mock_object(message_api.API, 'get_all',

6
cinder/tests/unit/api/v3/test_snapshots.py
View File

@ -195,7 +195,7 @@ class SnapshotApiTest(test.TestCase):
self.assertDictEqual({"key1": "val1", "key11": "val11"}, res_dict[
'snapshots'][0]['metadata'])
@ddt.data('3.30', '3.31')
@ddt.data('3.30', '3.31', '3.34')
@mock.patch('cinder.api.common.reject_invalid_filters')
def test_snapshot_list_with_general_filter(self, version, mock_update):
url = '/v3/%s/snapshots' % fake.PROJECT_ID
@ -205,8 +205,10 @@ class SnapshotApiTest(test.TestCase):
self.controller.index(req)
if version != '3.30':
support_like = True if version == '3.34' else False
mock_update.assert_called_once_with(req.environ['cinder.context'],
mock.ANY, 'snapshot')
mock.ANY, 'snapshot',
support_like)
def test_snapshot_list_with_metadata_unsupported_microversion(self):
# Create snapshot with metadata key1: value1

6
cinder/tests/unit/api/v3/test_volumes.py
View File

@ -389,7 +389,7 @@ class VolumeApiTest(test.TestCase):
self.assertRaises(webob.exc.HTTPBadRequest, self.controller.create,
req, body)
@ddt.data('3.30', '3.31')
@ddt.data('3.30', '3.31', '3.34')
@mock.patch.object(volume_api.API, 'check_volume_filters', mock.Mock())
@mock.patch.object(utils, 'add_visible_admin_metadata', mock.Mock())
@mock.patch('cinder.api.common.reject_invalid_filters')
@ -397,8 +397,10 @@ class VolumeApiTest(test.TestCase):
req = fakes.HTTPRequest.blank('/v3/volumes', version=version)
self.controller.index(req)
if version != '3.30':
support_like = True if version == '3.34' else False
mock_update.assert_called_once_with(req.environ['cinder.context'],
mock.ANY, 'volume')
mock.ANY, 'volume',
support_like)
@ddt.data({'admin': True, 'version': '3.21'},
{'admin': False, 'version': '3.21'},

77
cinder/tests/unit/test_db_api.py
View File

@ -19,14 +19,17 @@ import datetime
import ddt
import enum
import mock
from mock import call
from oslo_config import cfg
from oslo_utils import timeutils
from oslo_utils import uuidutils
from sqlalchemy.sql import operators
from cinder.api import common
from cinder import context
from cinder import db
from cinder.db.sqlalchemy import api as sqlalchemy_api
from cinder.db.sqlalchemy import models
from cinder import exception
from cinder import objects
from cinder.objects import fields
@ -77,6 +80,80 @@ class BaseTest(test.TestCase, test.ModelsObjectComparatorMixin):
self.ctxt = context.get_admin_context()
@ddt.ddt
class DBCommonFilterTestCase(BaseTest):
def setUp(self):
super(DBCommonFilterTestCase, self).setUp()
self.fake_volume = db.volume_create(self.ctxt,
{'display_name': 'fake_name'})
self.fake_group = utils.create_group(
self.ctxt,
group_type_id=fake.GROUP_TYPE_ID,
volume_type_ids=[fake.VOLUME_TYPE_ID])
@mock.patch('sqlalchemy.orm.query.Query.filter')
def test__process_model_like_filter(self, mock_filter):
filters = {'display_name': 'fake_name',
'display_description': 'fake_description',
'status': []}
session = sqlalchemy_api.get_session()
query = session.query(models.Volume)
mock_filter.return_value = query
with mock.patch.object(operators.Operators, 'op') as mock_op:
def fake_operator(value):
return value
mock_op.return_value = fake_operator
sqlalchemy_api._process_model_like_filter(models.Volume,
query, filters)
calls = [call('%fake_name%'), call('%fake_description%')]
mock_filter.assert_has_calls(calls)
@ddt.data({'handler': [db.volume_create, db.volume_get_all],
'column': 'display_name',
'resource': 'volume'},
{'handler': [db.snapshot_create, db.snapshot_get_all],
'column': 'display_name',
'resource': 'snapshot'},
{'handler': [db.message_create, db.message_get_all],
'column': 'message_level',
'resource': 'message'},
{'handler': [db.backup_create, db.backup_get_all],
'column': 'display_name',
'resource': 'backup'},
{'handler': [db.group_create, db.group_get_all],
'column': 'name',
'resource': 'group'},
{'handler': [utils.create_group_snapshot,
db.group_snapshot_get_all],
'column': 'name',
'resource': 'group_snapshot'})
@ddt.unpack
def test_resource_get_all_like_filter(self, handler, column, resource):
for index in ['001', '002']:
option = {column: "fake_%s_%s" % (column, index)}
if resource in ['snapshot', 'backup']:
option['volume_id'] = self.fake_volume.id
if resource in ['message']:
option['project_id'] = fake.PROJECT_ID
option['event_id'] = fake.UUID1
if resource in ['group_snapshot']:
handler[0](self.ctxt, self.fake_group.id,
name="fake_%s_%s" % (column, index))
else:
handler[0](self.ctxt, option)
# test exact match
exact_filter = {column: 'fake_%s' % column}
resources = handler[1](self.ctxt, filters=exact_filter)
self.assertEqual(0, len(resources))
# test inexact match
inexact_filter = {"%s~" % column: 'fake_%s' % column}
resources = handler[1](self.ctxt, filters=inexact_filter)
self.assertEqual(2, len(resources))
@ddt.ddt
class DBAPIServiceTestCase(BaseTest):

67
doc/source/man/generalized_filters.rst
View File

@ -35,31 +35,44 @@ Which filter keys are supported?
--------------------------------
Not all the attributes are supported at present, so we add this table below to
indicate which filter keys are valid and can be used in the configuration:
indicate which filter keys are valid and can be used in the configuration.
+----------------+-------------------------------------------------------------------------+
| API | Valid filter keys |
+================+=========================================================================+
| | id, group_id, name, status, bootable, migration_status, metadata, host, |
| list volume | image_metadata, availability_zone, user_id, volume_type_id, project_id, |
| | size, description, replication_status, multiattach |
+----------------+-------------------------------------------------------------------------+
| | id, volume_id, user_id, project_id, status, volume_size, name, |
| list snapshot | description, volume_type_id, group_snapshot_id, metadata |
+----------------+-------------------------------------------------------------------------+
| | id, name, status, container, availability_zone, description, |
| list backup | volume_id, is_incremental, size, host, parent_id |
+----------------+-------------------------------------------------------------------------+
| | id, user_id, status, availability_zone, group_type, name, description, |
| list group | host |
+----------------+-------------------------------------------------------------------------+
| list g-snapshot| id, name, description, group_id, group_type_id, status |
+----------------+-------------------------------------------------------------------------+
| | id, volume_id, instance_id, attach_status, attach_mode, |
| list attachment| connection_info, mountpoint, attached_host |
+----------------+-------------------------------------------------------------------------+
| | id, event_id, resource_uuid, resource_type, request_id, message_level, |
| list message | project_id |
+----------------+-------------------------------------------------------------------------+
| get pools | name |
+----------------+-------------------------------------------------------------------------+
Since v3.34 we could use '~' to indicate supporting querying resource by inexact match,
for example, if we have a configuration file as below:
.. code-block:: json
{
"volume": ["name~"]
}
User can query volume both by ``name=volume`` and ``name~=volume``, and the volumes
named ``volume123`` and ``a_volume123`` are both valid for second input while neither are
valid for first. The supported APIs are marked with "*" below in the table.
+-----------------+-------------------------------------------------------------------------+
| API | Valid filter keys |
+=================+=========================================================================+
| | id, group_id, name, status, bootable, migration_status, metadata, host, |
| list volume* | image_metadata, availability_zone, user_id, volume_type_id, project_id, |
| | size, description, replication_status, multiattach |
+-----------------+-------------------------------------------------------------------------+
| | id, volume_id, user_id, project_id, status, volume_size, name, |
| list snapshot* | description, volume_type_id, group_snapshot_id, metadata |
+-----------------+-------------------------------------------------------------------------+
| | id, name, status, container, availability_zone, description, |
| list backup* | volume_id, is_incremental, size, host, parent_id |
+-----------------+-------------------------------------------------------------------------+
| | id, user_id, status, availability_zone, group_type, name, description, |
| list group* | host |
+-----------------+-------------------------------------------------------------------------+
| list g-snapshot*| id, name, description, group_id, group_type_id, status |
+-----------------+-------------------------------------------------------------------------+
| | id, volume_id, instance_id, attach_status, attach_mode, |
| list attachment*| connection_info, mountpoint, attached_host |
+-----------------+-------------------------------------------------------------------------+
| | id, event_id, resource_uuid, resource_type, request_id, message_level, |
| list message* | project_id |
+-----------------+-------------------------------------------------------------------------+
| get pools | name |
+-----------------+-------------------------------------------------------------------------+

12
releasenotes/notes/add-like-filter-support-7d4r78d6de3984dv.yaml Normal file
View File

@ -0,0 +1,12 @@
---
features:
- |
Added like operator support to filters for the following resources::
- volume
- snapshot
- backup
- group
- group-snapshot
- attachment
- message