Support 'LIKE' operator to filter resource

Added like operator support to filters
for the following resources.

1. volume
2. snapshot
3. backup
4. group
5. group-snapshot
6. attachment
7. message

Depends-On: ff3d41b15abb2915de87830980147be51e5da971

APIImpact
DocImpact
Partial-Implements: blueprint support-regexp-based-query

Change-Id: I6c2ea07b0bfc5852b28e44989406cc10eb972e26

cinder/api/common.py

@@ -64,6 +64,7 @@ CONF.register_opts(api_common_opts)
 LOG = logging.getLogger(__name__)
 _FILTERS_COLLECTION = None
 FILTERING_VERSION = '3.31'
+LIKE_FILTER_VERSION = '3.34'
 
 
 METADATA_TYPES = enum.Enum('METADATA_TYPES', 'user image')
@@ -443,7 +444,8 @@ def get_enabled_resource_filters(resource=None):
         return {}
 
 
-def reject_invalid_filters(context, filters, resource):
+def reject_invalid_filters(context, filters, resource,
+                           enable_like_filter=False):
     if context.is_admin:
         # Allow all options
         return
@@ -455,8 +457,14 @@ def reject_invalid_filters(context, filters, resource):
         configured_filters = []
     invalid_filters = []
     for key in filters.copy().keys():
+        if not enable_like_filter:
             if key not in configured_filters:
                 invalid_filters.append(key)
+        else:
+            # If 'key~' is configured, both 'key' and 'key~' is valid.
+            if (key not in configured_filters or
+                    "%s~" % key not in configured_filters):
+                invalid_filters.append(key)
     if invalid_filters:
         raise webob.exc.HTTPBadRequest(
             explanation=_('Invalid filters %s are found in query '
@@ -470,7 +478,11 @@ def process_general_filtering(resource):
             filters = kwargs.get('filters')
             context = kwargs.get('context')
             if req_version.matches(FILTERING_VERSION):
-                reject_invalid_filters(context, filters, resource)
+                support_like = False
+                if req_version.matches(LIKE_FILTER_VERSION):
+                    support_like = True
+                reject_invalid_filters(context, filters,
+                                       resource, support_like)
             else:
                 process_non_general_filtering(*args, **kwargs)
         return _decorator

cinder/api/openstack/api_version_request.py

@@ -86,6 +86,10 @@ REST_API_VERSION_HISTORY = """
     * 3.33 - Add ``resource_filters`` API to retrieve configured
              resource filters.
 
+    * 3.34 - Add like filter support in ``volume``, ``backup``, ``snapshot``,
+             ``message``, ``attachment``, ``group`` and ``group-snapshot``
+             list APIs.
+
 """
 
 # The minimum and maximum versions of the API supported
@@ -93,7 +97,7 @@ REST_API_VERSION_HISTORY = """
 # minimum version of the API supported.
 # Explicitly using /v1 or /v2 endpoints will still work
 _MIN_API_VERSION = "3.0"
-_MAX_API_VERSION = "3.33"
+_MAX_API_VERSION = "3.34"
 _LEGACY_API_VERSION1 = "1.0"
 _LEGACY_API_VERSION2 = "2.0"
 

cinder/api/openstack/rest_api_version_history.rst

@@ -308,3 +308,8 @@ user documentation.
 3.33
 ----
   Add ``resource_filters`` API to retrieve configured resource filters.
+
+3.34
+----
+  Add like filter support in ``volume``, ``backup``, ``snapshot``, ``message``,
+  ``attachment``, ``group`` and ``group-snapshot`` list APIs.

cinder/api/v3/group_snapshots.py

@@ -114,8 +114,11 @@ class GroupSnapshotsController(wsgi.Controller):
             marker, limit, offset = common.get_pagination_params(filters)
             sort_keys, sort_dirs = common.get_sort_params(filters)
 
-        if req.api_version_request.matches(common.FILTERING_VERSION):
+        if req_version.matches(common.FILTERING_VERSION):
-            common.reject_invalid_filters(context, filters, 'group_snapshot')
+            support_like = (True if req_version.matches(
+                common.LIKE_FILTER_VERSION) else False)
+            common.reject_invalid_filters(context, filters, 'group_snapshot',
+                                          support_like)
 
         group_snapshots = self.group_snapshot_api.get_all_group_snapshots(
             context, filters=filters, marker=marker, limit=limit,

cinder/api/v3/groups.py

@@ -163,12 +163,16 @@ class GroupsController(wsgi.Controller):
         """Returns a list of groups through view builder."""
         context = req.environ['cinder.context']
         filters = req.params.copy()
+        api_version = req.api_version_request
         marker, limit, offset = common.get_pagination_params(filters)
         sort_keys, sort_dirs = common.get_sort_params(filters)
 
         filters.pop('list_volume', None)
-        if req.api_version_request.matches(common.FILTERING_VERSION):
+        if api_version.matches(common.FILTERING_VERSION):
-            common.reject_invalid_filters(context, filters, 'group')
+            support_like = (True if api_version.matches(
+                common.LIKE_FILTER_VERSION) else False)
+            common.reject_invalid_filters(context, filters, 'group',
+                                          support_like)
 
         groups = self.group_api.get_all(
             context, filters=filters, marker=marker, limit=limit,

cinder/api/v3/messages.py

@@ -80,6 +80,7 @@ class MessagesController(wsgi.Controller):
     def index(self, req):
         """Returns a list of messages, transformed through view builder."""
         context = req.environ['cinder.context']
+        api_version = req.api_version_request
         check_policy(context, 'get_all')
         filters = None
         marker = None
@@ -88,13 +89,16 @@ class MessagesController(wsgi.Controller):
         sort_keys = None
         sort_dirs = None
 
-        if (req.api_version_request.matches("3.5")):
+        if api_version.matches("3.5"):
             filters = req.params.copy()
             marker, limit, offset = common.get_pagination_params(filters)
             sort_keys, sort_dirs = common.get_sort_params(filters)
 
-        if req.api_version_request.matches(common.FILTERING_VERSION):
+        if api_version.matches(common.FILTERING_VERSION):
-            common.reject_invalid_filters(context, filters, 'message')
+            support_like = (True if api_version.matches(
+                common.LIKE_FILTER_VERSION) else False)
+            common.reject_invalid_filters(context, filters, 'message',
+                                          support_like)
 
         messages = self.message_api.get_all(context, filters=filters,
                                             marker=marker, limit=limit,

cinder/db/sqlalchemy/api.py

@@ -1725,6 +1725,45 @@ def volume_detached(context, volume_id, attachment_id):
         return (volume_updates, attachment_updates)
 
 
+def _process_model_like_filter(model, query, filters):
+    """Applies regex expression filtering to a query.
+
+    :param model: model to apply filters to
+    :param query: query to apply filters to
+    :param filters: dictionary of filters with regex values
+    :returns: the updated query.
+    """
+    if query is None:
+        return query
+
+    for key in filters:
+        column_attr = getattr(model, key)
+        if 'property' == type(column_attr).__name__:
+            continue
+        value = filters[key]
+        if not isinstance(value, six.string_types):
+            continue
+        query = query.filter(column_attr.op('LIKE')(u'%' + value + u'%'))
+    return query
+
+
+def apply_like_filters(model):
+    def decorator_filters(process_exact_filters):
+        def _decorator(query, filters):
+            exact_filters = filters.copy()
+            regex_filters = {}
+            for key, value in filters.items():
+                # NOTE(tommylikehu): For inexact match, the filter keys
+                # are in the format of 'key~=value'
+                if key.endswith('~'):
+                    exact_filters.pop(key)
+                    regex_filters[key.rstrip('~')] = value
+            query = process_exact_filters(query, exact_filters)
+            return _process_model_like_filter(model, query, regex_filters)
+        return _decorator
+    return decorator_filters
+
+
 @require_context
 def _volume_get_query(context, session=None, project_only=False,
                       joined_load=True):
@@ -1815,6 +1854,7 @@ def _attachment_get_query(context, session=None, project_only=False):
                        project_only=project_only).options(joinedload('volume'))
 
 
+@apply_like_filters(model=models.VolumeAttachment)
 def _process_attachment_filters(query, filters):
     if filters:
         project_id = filters.pop('project_id', None)
@@ -2227,6 +2267,7 @@ def _generate_paginate_query(context, session, marker, limit, sort_keys,
                                           offset=offset)
 
 
+@apply_like_filters(model=models.Volume)
 def _process_volume_filters(query, filters):
     """Common filter processing for Volume queries.
 
@@ -2900,6 +2941,7 @@ def _snaps_get_query(context, session=None, project_only=False):
         options(joinedload('snapshot_metadata'))
 
 
+@apply_like_filters(model=models.Snapshot)
 def _process_snaps_filters(query, filters):
     if filters:
         filters = filters.copy()
@@ -4914,6 +4956,7 @@ def _backups_get_query(context, session=None, project_only=False):
                        project_only=project_only)
 
 
+@apply_like_filters(model=models.Backup)
 def _process_backups_filters(query, filters):
     if filters:
         # Ensure that filters' keys exist on the model
@@ -5499,6 +5542,7 @@ def _group_snapshot_get_query(context, session=None, project_only=False):
                        project_only=project_only)
 
 
+@apply_like_filters(model=models.Group)
 def _process_groups_filters(query, filters):
     if filters:
         # Ensure that filters' keys exist on the model
@@ -5508,6 +5552,7 @@ def _process_groups_filters(query, filters):
     return query
 
 
+@apply_like_filters(model=models.GroupSnapshot)
 def _process_group_snapshot_filters(query, filters):
     if filters:
         # Ensure that filters' keys exist on the model
@@ -5786,6 +5831,7 @@ def is_valid_model_filters(model, filters, exclude_list=None):
         if exclude_list and key in exclude_list:
             continue
         try:
+            key = key.rstrip('~')
             getattr(model, key)
         except AttributeError:
             LOG.debug("'%s' filter key is not valid.", key)
@@ -6185,6 +6231,7 @@ def message_get_all(context, filters=None, marker=None, limit=None,
         return _translate_messages(results)
 
 
+@apply_like_filters(model=models.Message)
 def _process_messages_filters(query, filters):
     if filters:
         # Ensure that filters' keys exist on the model

cinder/tests/unit/api/v3/test_attachments.py

@@ -139,7 +139,7 @@ class AttachmentsAPITestCase(test.TestCase):
                           self.controller.delete, req,
                           self.attachment1.id)
 
-    @ddt.data('3.30', '3.31')
+    @ddt.data('3.30', '3.31', '3.34')
     @mock.patch('cinder.api.common.reject_invalid_filters')
     def test_attachment_list_with_general_filter(self, version, mock_update):
         url = '/v3/%s/attachments' % fake.PROJECT_ID
@@ -149,8 +149,10 @@ class AttachmentsAPITestCase(test.TestCase):
         self.controller.index(req)
 
         if version != '3.30':
+            support_like = True if version == '3.34' else False
             mock_update.assert_called_once_with(req.environ['cinder.context'],
-                                                mock.ANY, 'attachment')
+                                                mock.ANY, 'attachment',
+                                                support_like)
 
     @ddt.data('reserved', 'attached')
     @mock.patch.object(volume_rpcapi.VolumeAPI, 'attachment_delete')

cinder/tests/unit/api/v3/test_backups.py

@@ -85,7 +85,7 @@ class BackupsControllerAPITestCase(test.TestCase):
                           self.controller.update,
                           req, fake.BACKUP_ID, body)
 
-    @ddt.data('3.30', '3.31')
+    @ddt.data('3.30', '3.31', '3.34')
     @mock.patch('cinder.api.common.reject_invalid_filters')
     def test_backup_list_with_general_filter(self, version, mock_update):
         url = '/v3/%s/backups' % fake.PROJECT_ID
@@ -95,8 +95,10 @@ class BackupsControllerAPITestCase(test.TestCase):
         self.controller.index(req)
 
         if version != '3.30':
+            support_like = True if version == '3.34' else False
             mock_update.assert_called_once_with(req.environ['cinder.context'],
-                                                mock.ANY, 'backup')
+                                                mock.ANY, 'backup',
+                                                support_like)
 
     def test_backup_update(self):
         backup = test_utils.create_backup(

cinder/tests/unit/api/v3/test_group_snapshots.py

@@ -183,7 +183,7 @@ class GroupSnapshotsAPITestCase(test.TestCase):
                              res_dict['group_snapshots'][0].keys())
         group_snapshot.destroy()
 
-    @ddt.data('3.30', '3.31')
+    @ddt.data('3.30', '3.31', '3.34')
     @mock.patch('cinder.api.common.reject_invalid_filters')
     def test_group_snapshot_list_with_general_filter(self,
                                                      version, mock_update):
@@ -194,8 +194,10 @@ class GroupSnapshotsAPITestCase(test.TestCase):
         self.controller.index(req)
 
         if version != '3.30':
+            support_like = True if version == '3.34' else False
             mock_update.assert_called_once_with(req.environ['cinder.context'],
-                                                mock.ANY, 'group_snapshot')
+                                                mock.ANY, 'group_snapshot',
+                                                support_like)
 
     @ddt.data(False, True)
     def test_list_group_snapshot_with_filter(self, is_detail):

cinder/tests/unit/api/v3/test_groups.py

@@ -239,7 +239,7 @@ class GroupsAPITestCase(test.TestCase):
         self.assertRaises(exception.GroupNotFound, self.controller.show,
                           req, fake.WILL_NOT_BE_FOUND_ID)
 
-    @ddt.data('3.30', '3.31')
+    @ddt.data('3.30', '3.31', '3.34')
     @mock.patch('cinder.api.common.reject_invalid_filters')
     def test_group_list_with_general_filter(self, version, mock_update):
         url = '/v3/%s/groups' % fake.PROJECT_ID
@@ -249,8 +249,10 @@ class GroupsAPITestCase(test.TestCase):
         self.controller.index(req)
 
         if version != '3.30':
+            support_like = True if version == '3.34' else False
             mock_update.assert_called_once_with(req.environ['cinder.context'],
-                                                mock.ANY, 'group')
+                                                mock.ANY, 'group',
+                                                support_like)
 
     def test_list_groups_json(self):
         self.group2.group_type_id = fake.GROUP_TYPE2_ID

cinder/tests/unit/api/v3/test_messages.py

@@ -123,7 +123,7 @@ class MessageApiTest(test.TestCase):
         self.assertRaises(exception.MessageNotFound, self.controller.delete,
                           req, fakes.FAKE_UUID)
 
-    @ddt.data('3.30', '3.31')
+    @ddt.data('3.30', '3.31', '3.34')
     @mock.patch('cinder.api.common.reject_invalid_filters')
     def test_message_list_with_general_filter(self, version, mock_update):
         url = '/v3/%s/messages' % fakes.FAKE_UUID
@@ -133,8 +133,10 @@ class MessageApiTest(test.TestCase):
         self.controller.index(req)
 
         if version != '3.30':
+            support_like = True if version == '3.34' else False
             mock_update.assert_called_once_with(req.environ['cinder.context'],
-                                                mock.ANY, 'message')
+                                                mock.ANY, 'message',
+                                                support_like)
 
     def test_index(self):
         self.mock_object(message_api.API, 'get_all',

cinder/tests/unit/api/v3/test_snapshots.py

@@ -195,7 +195,7 @@ class SnapshotApiTest(test.TestCase):
         self.assertDictEqual({"key1": "val1", "key11": "val11"}, res_dict[
             'snapshots'][0]['metadata'])
 
-    @ddt.data('3.30', '3.31')
+    @ddt.data('3.30', '3.31', '3.34')
     @mock.patch('cinder.api.common.reject_invalid_filters')
     def test_snapshot_list_with_general_filter(self, version, mock_update):
         url = '/v3/%s/snapshots' % fake.PROJECT_ID
@@ -205,8 +205,10 @@ class SnapshotApiTest(test.TestCase):
         self.controller.index(req)
 
         if version != '3.30':
+            support_like = True if version == '3.34' else False
             mock_update.assert_called_once_with(req.environ['cinder.context'],
-                                                mock.ANY, 'snapshot')
+                                                mock.ANY, 'snapshot',
+                                                support_like)
 
     def test_snapshot_list_with_metadata_unsupported_microversion(self):
         # Create snapshot with metadata key1: value1

cinder/tests/unit/api/v3/test_volumes.py

@@ -389,7 +389,7 @@ class VolumeApiTest(test.TestCase):
         self.assertRaises(webob.exc.HTTPBadRequest, self.controller.create,
                           req, body)
 
-    @ddt.data('3.30', '3.31')
+    @ddt.data('3.30', '3.31', '3.34')
     @mock.patch.object(volume_api.API, 'check_volume_filters', mock.Mock())
     @mock.patch.object(utils, 'add_visible_admin_metadata', mock.Mock())
     @mock.patch('cinder.api.common.reject_invalid_filters')
@@ -397,8 +397,10 @@ class VolumeApiTest(test.TestCase):
         req = fakes.HTTPRequest.blank('/v3/volumes', version=version)
         self.controller.index(req)
         if version != '3.30':
+            support_like = True if version == '3.34' else False
             mock_update.assert_called_once_with(req.environ['cinder.context'],
-                                                mock.ANY, 'volume')
+                                                mock.ANY, 'volume',
+                                                support_like)
 
     @ddt.data({'admin': True, 'version': '3.21'},
               {'admin': False, 'version': '3.21'},

cinder/tests/unit/test_db_api.py

@@ -19,14 +19,17 @@ import datetime
 import ddt
 import enum
 import mock
+from mock import call
 from oslo_config import cfg
 from oslo_utils import timeutils
 from oslo_utils import uuidutils
+from sqlalchemy.sql import operators
 
 from cinder.api import common
 from cinder import context
 from cinder import db
 from cinder.db.sqlalchemy import api as sqlalchemy_api
+from cinder.db.sqlalchemy import models
 from cinder import exception
 from cinder import objects
 from cinder.objects import fields
@@ -77,6 +80,80 @@ class BaseTest(test.TestCase, test.ModelsObjectComparatorMixin):
         self.ctxt = context.get_admin_context()
 
 
+@ddt.ddt
+class DBCommonFilterTestCase(BaseTest):
+
+    def setUp(self):
+        super(DBCommonFilterTestCase, self).setUp()
+        self.fake_volume = db.volume_create(self.ctxt,
+                                            {'display_name': 'fake_name'})
+        self.fake_group = utils.create_group(
+            self.ctxt,
+            group_type_id=fake.GROUP_TYPE_ID,
+            volume_type_ids=[fake.VOLUME_TYPE_ID])
+
+    @mock.patch('sqlalchemy.orm.query.Query.filter')
+    def test__process_model_like_filter(self, mock_filter):
+        filters = {'display_name': 'fake_name',
+                   'display_description': 'fake_description',
+                   'status': []}
+        session = sqlalchemy_api.get_session()
+        query = session.query(models.Volume)
+        mock_filter.return_value = query
+        with mock.patch.object(operators.Operators, 'op') as mock_op:
+            def fake_operator(value):
+                return value
+            mock_op.return_value = fake_operator
+            sqlalchemy_api._process_model_like_filter(models.Volume,
+                                                      query, filters)
+            calls = [call('%fake_name%'), call('%fake_description%')]
+            mock_filter.assert_has_calls(calls)
+
+    @ddt.data({'handler': [db.volume_create, db.volume_get_all],
+               'column': 'display_name',
+               'resource': 'volume'},
+              {'handler': [db.snapshot_create, db.snapshot_get_all],
+               'column': 'display_name',
+               'resource': 'snapshot'},
+              {'handler': [db.message_create, db.message_get_all],
+               'column': 'message_level',
+               'resource': 'message'},
+              {'handler': [db.backup_create, db.backup_get_all],
+               'column': 'display_name',
+               'resource': 'backup'},
+              {'handler': [db.group_create, db.group_get_all],
+               'column': 'name',
+               'resource': 'group'},
+              {'handler': [utils.create_group_snapshot,
+                           db.group_snapshot_get_all],
+               'column': 'name',
+               'resource': 'group_snapshot'})
+    @ddt.unpack
+    def test_resource_get_all_like_filter(self, handler, column, resource):
+        for index in ['001', '002']:
+            option = {column: "fake_%s_%s" % (column, index)}
+            if resource in ['snapshot', 'backup']:
+                option['volume_id'] = self.fake_volume.id
+            if resource in ['message']:
+                option['project_id'] = fake.PROJECT_ID
+                option['event_id'] = fake.UUID1
+            if resource in ['group_snapshot']:
+                handler[0](self.ctxt, self.fake_group.id,
+                           name="fake_%s_%s" % (column, index))
+            else:
+                handler[0](self.ctxt, option)
+
+        # test exact match
+        exact_filter = {column: 'fake_%s' % column}
+        resources = handler[1](self.ctxt, filters=exact_filter)
+        self.assertEqual(0, len(resources))
+
+        # test inexact match
+        inexact_filter = {"%s~" % column: 'fake_%s' % column}
+        resources = handler[1](self.ctxt, filters=inexact_filter)
+        self.assertEqual(2, len(resources))
+
+
 @ddt.ddt
 class DBAPIServiceTestCase(BaseTest):
 

doc/source/man/generalized_filters.rst

@@ -35,31 +35,44 @@ Which filter keys are supported?
 --------------------------------
 
 Not all the attributes are supported at present, so we add this table below to
-indicate which filter keys are valid and can be used in the configuration:
+indicate which filter keys are valid and can be used in the configuration.
 
-+----------------+-------------------------------------------------------------------------+
+Since v3.34 we could use '~' to indicate supporting querying resource by inexact match,
+for example, if we have a configuration file as below:
+
+.. code-block:: json
+
+    {
+       "volume": ["name~"]
+    }
+
+User can query volume both by ``name=volume`` and ``name~=volume``, and the volumes
+named ``volume123`` and ``a_volume123`` are both valid for second input while neither are
+valid for first. The supported APIs are marked with "*" below in the table.
+
++-----------------+-------------------------------------------------------------------------+
 |    API          | Valid filter keys                                                       |
-+================+=========================================================================+
++=================+=========================================================================+
 |                 | id, group_id, name, status, bootable, migration_status, metadata, host, |
-| list volume    | image_metadata, availability_zone, user_id, volume_type_id, project_id, |
+| list volume*    | image_metadata, availability_zone, user_id, volume_type_id, project_id, |
 |                 | size, description, replication_status, multiattach                      |
-+----------------+-------------------------------------------------------------------------+
++-----------------+-------------------------------------------------------------------------+
 |                 | id, volume_id, user_id, project_id, status, volume_size, name,          |
-| list snapshot  | description, volume_type_id, group_snapshot_id, metadata                |
+| list snapshot*  | description, volume_type_id, group_snapshot_id, metadata                |
-+----------------+-------------------------------------------------------------------------+
++-----------------+-------------------------------------------------------------------------+
 |                 | id, name, status, container, availability_zone, description,            |
-| list backup    | volume_id, is_incremental, size, host, parent_id                        |
+| list backup*    | volume_id, is_incremental, size, host, parent_id                        |
-+----------------+-------------------------------------------------------------------------+
++-----------------+-------------------------------------------------------------------------+
 |                 | id, user_id, status, availability_zone, group_type, name, description,  |
-| list group     | host                                                                    |
+| list group*     | host                                                                    |
-+----------------+-------------------------------------------------------------------------+
++-----------------+-------------------------------------------------------------------------+
-| list g-snapshot| id, name, description, group_id, group_type_id, status                  |
+| list g-snapshot*| id, name, description, group_id, group_type_id, status                  |
-+----------------+-------------------------------------------------------------------------+
++-----------------+-------------------------------------------------------------------------+
 |                 | id, volume_id, instance_id, attach_status, attach_mode,                 |
-| list attachment| connection_info, mountpoint, attached_host                              |
+| list attachment*| connection_info, mountpoint, attached_host                              |
-+----------------+-------------------------------------------------------------------------+
++-----------------+-------------------------------------------------------------------------+
 |                 | id, event_id, resource_uuid, resource_type, request_id, message_level,  |
-| list message   | project_id                                                              |
+| list message*   | project_id                                                              |
-+----------------+-------------------------------------------------------------------------+
++-----------------+-------------------------------------------------------------------------+
 | get pools       | name                                                                    |
-+----------------+-------------------------------------------------------------------------+
++-----------------+-------------------------------------------------------------------------+

releasenotes/notes/add-like-filter-support-7d4r78d6de3984dv.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Added like operator support to filters for the following resources::
+
+    - volume
+    - snapshot
+    - backup
+    - group
+    - group-snapshot
+    - attachment
+    - message