Merge "Add filter based on 'in' operator"

doc/source/specification/murano-repository.rst

@@ -39,6 +39,11 @@ Methods for application package management
 - ``enabled``: determines whether the package is browsed in the Application Catalog
 - ``owner_id``: id of a project that owns the package
 
+.. note::
+
+    It is possible to use ``in`` operator for properties ``id``, ``category`` and ``tag``.
+    For example to get packages with ``id1, id2, id3`` use ``id=in:id1,id2,id3``.
+
 List packages
 -------------
 

murano/api/v1/__init__.py

@@ -22,6 +22,7 @@ SUPPORTED_PARAMS = {'id', 'order_by', 'category', 'marker', 'tag',
                     'search', 'include_disabled', 'sort_dir', 'name'}
 LIST_PARAMS = {'id', 'category', 'tag', 'class', 'order_by'}
 ORDER_VALUES = {'fqn', 'name', 'created'}
+OPERATOR_VALUES = {'id', 'category', 'tag'}
 PKG_PARAMS_MAP = {'display_name': 'name',
                   'full_name': 'fully_qualified_name',
                   'ui': 'ui_definition',

murano/api/v1/catalog.py

@@ -31,6 +31,7 @@ import murano.api.v1
 from murano.api.v1 import schemas
 from murano.common import exceptions
 from murano.common import policy
+import murano.common.utils as murano_utils
 from murano.common import wsgi
 from murano.db.catalog import api as db_api
 from murano.common.i18n import _, _LW
@@ -46,6 +47,7 @@ SUPPORTED_PARAMS = murano.api.v1.SUPPORTED_PARAMS
 LIST_PARAMS = murano.api.v1.LIST_PARAMS
 ORDER_VALUES = murano.api.v1.ORDER_VALUES
 PKG_PARAMS_MAP = murano.api.v1.PKG_PARAMS_MAP
+OPERATOR_VALUES = murano.api.v1.OPERATOR_VALUES
 
 
 def _check_content_type(req, content_type):
@@ -67,6 +69,16 @@ def _get_filters(query_params):
             continue
 
         if k in LIST_PARAMS:
+            if v.startswith('in:') and k in OPERATOR_VALUES:
+                in_value = v[len('in:'):]
+                try:
+                    filters[k] = murano_utils.split_for_quotes(in_value)
+                except ValueError as err:
+                    LOG.warning(_LW("Search by parameter '{name}' "
+                                    "caused an {message} error."
+                                    "Skipping it.").format(name=k,
+                                                           message=err))
+            else:
                 filters.setdefault(k, []).append(v)
         else:
             filters[k] = v

murano/common/utils.py

@@ -14,6 +14,7 @@
 
 import collections
 import functools as func
+import re
 
 import jsonschema
 from oslo_log import log as logging
@@ -244,3 +245,57 @@ def validate_body(schema):
                 return f(*args, **kwargs)
         return f_validate_body
     return deco_validate_body
+
+
+def validate_quotes(value):
+    """Validate filter values
+
+    Validation opening/closing quotes in the expression.
+    """
+    open_quotes = True
+    count_backslash_in_row = 0
+    for i in range(len(value)):
+        if value[i] == '"':
+            if count_backslash_in_row % 2:
+                continue
+            if open_quotes:
+                if i and value[i - 1] != ',':
+                    msg = _("Invalid filter value %s. There is no comma "
+                            "before opening quotation mark.") % value
+                    raise ValueError(msg)
+            else:
+                if i + 1 != len(value) and value[i + 1] != ",":
+                    msg = _("Invalid filter value %s. There is no comma "
+                            "after opening quotation mark.") % value
+                    raise ValueError(msg)
+            open_quotes = not open_quotes
+        elif value[i] == '\\':
+            count_backslash_in_row += 1
+        else:
+            count_backslash_in_row = 0
+    if not open_quotes:
+        msg = _("Invalid filter value %s. The quote is not closed.") % value
+        raise ValueError(msg)
+    return True
+
+
+def split_for_quotes(value):
+    """Split filter values
+
+    Split values by commas and quotes for 'in' operator, according api-wg.
+    """
+    validate_quotes(value)
+    tmp = re.compile(r'''
+        "(                 # if found a double-quote
+           [^\"\\]*        # take characters either non-quotes or backslashes
+           (?:\\.          # take backslashes and character after it
+            [^\"\\]*)*     # take characters either non-quotes or backslashes
+         )                 # before double-quote
+        ",?                # a double-quote with comma maybe
+        | ([^,]+),?        # if not found double-quote take any non-comma
+                           # characters with comma maybe
+        | ,                # if we have only comma take empty string
+        ''', re.VERBOSE)
+    val_split = [val[0] or val[1] for val in re.findall(tmp, value)]
+    replaced_inner_quotes = [s.replace(r'\"', '"') for s in val_split]
+    return replaced_inner_quotes

murano/tests/unit/api/v1/test_catalog.py

@@ -203,6 +203,175 @@ class TestCatalogApi(test_base.ControllerTest, test_base.MuranoApiTestCase):
 
         self.assertEqual(expected_package.id, found_package['id'])
 
+    def test_packages_filter_by_in_category(self):
+        """Test that packages are filtered by in:cat1,cat2,cat3
+
+        GET /catalog/packages with parameter "category=in:cat1,cat2,cat3"
+        returns packages filtered by category.
+        """
+        names = ['cat1', 'cat2', 'cat3', 'cat4']
+        for name in names:
+            db_catalog_api.category_add(name)
+        self._set_policy_rules(
+            {'get_package': '',
+             'manage_public_package': ''}
+        )
+        _, package1_data = self._test_package()
+        _, package2_data = self._test_package()
+        _, package3_data = self._test_package()
+
+        package1_data['fully_qualified_name'] += '_1'
+        package1_data['name'] += '_1'
+        package1_data['class_definitions'] = (u'test.mpl.v1.app.Thing1',)
+        package1_data['categories'] = ('cat1', 'cat2')
+
+        package2_data['fully_qualified_name'] += '_2'
+        package2_data['name'] += '_2'
+        package2_data['class_definitions'] = (u'test.mpl.v1.app.Thing2',)
+        package2_data['categories'] = ('cat2', 'cat3')
+
+        package3_data['fully_qualified_name'] += '_3'
+        package3_data['name'] += '_3'
+        package3_data['class_definitions'] = (u'test.mpl.v1.app.Thing3',)
+        package3_data['categories'] = ('cat2', 'cat4')
+
+        expected_packages = [db_catalog_api.package_upload(package1_data, ''),
+                             db_catalog_api.package_upload(package2_data, '')]
+        db_catalog_api.package_upload(package3_data, '')
+
+        categories_in = "in:cat1,cat3"
+
+        req = self._get('/catalog/packages',
+                        params={'category': categories_in})
+        self.expect_policy_check('get_package')
+        self.expect_policy_check('manage_public_package')
+
+        res = req.get_response(self.api)
+        self.assertEqual(200, res.status_code)
+        self.assertEqual(2, len(res.json['packages']))
+
+        found_packages = res.json['packages']
+
+        self.assertEqual([pack.id for pack in expected_packages],
+                         [pack['id'] for pack in found_packages])
+
+    def test_packages_filter_by_in_tag(self):
+        """Test that packages are filtered by in:tag1,tag2,tag3
+
+        GET /catalog/packages with parameter "tag=in:tag1,tag2,tag3"
+        returns packages filtered by category.
+        """
+        self._set_policy_rules(
+            {'get_package': '',
+             'manage_public_package': ''}
+        )
+        _, package1_data = self._test_package()
+        _, package2_data = self._test_package()
+        _, package3_data = self._test_package()
+
+        package1_data['fully_qualified_name'] += '_1'
+        package1_data['name'] += '_1'
+        package1_data['class_definitions'] = (u'test.mpl.v1.app.Thing1',)
+        package1_data['tags'] = ('tag1', 'tag2')
+
+        package2_data['fully_qualified_name'] += '_2'
+        package2_data['name'] += '_2'
+        package2_data['class_definitions'] = (u'test.mpl.v1.app.Thing2',)
+        package2_data['tags'] = ('tag2', 'tag3')
+
+        package3_data['fully_qualified_name'] += '_3'
+        package3_data['name'] += '_3'
+        package3_data['class_definitions'] = (u'test.mpl.v1.app.Thing3',)
+        package3_data['tags'] = ('tag2', 'tag4')
+
+        expected_packages = [db_catalog_api.package_upload(package1_data, ''),
+                             db_catalog_api.package_upload(package2_data, '')]
+        db_catalog_api.package_upload(package3_data, '')
+
+        tag_in = "in:tag1,tag3"
+
+        req = self._get('/catalog/packages',
+                        params={'tag': tag_in})
+        self.expect_policy_check('get_package')
+        self.expect_policy_check('manage_public_package')
+
+        res = req.get_response(self.api)
+        self.assertEqual(200, res.status_code)
+        self.assertEqual(2, len(res.json['packages']))
+
+        found_packages = res.json['packages']
+
+        self.assertEqual([pack.id for pack in expected_packages],
+                         [pack['id'] for pack in found_packages])
+
+    def test_packages_filter_by_in_id(self):
+        """Test that packages are filtered by in:id1,id2,id3
+
+        GET /catalog/packages with parameter "id=in:id1,id2" returns packages
+        filtered by id.
+        """
+        self._set_policy_rules(
+            {'get_package': '',
+             'manage_public_package': ''}
+        )
+        _, package1_data = self._test_package()
+        _, package2_data = self._test_package()
+        _, package3_data = self._test_package()
+
+        package1_data['fully_qualified_name'] += '_1'
+        package1_data['name'] += '_1'
+        package1_data['class_definitions'] = (u'test.mpl.v1.app.Thing1',)
+        package2_data['fully_qualified_name'] += '_2'
+        package2_data['name'] += '_2'
+        package2_data['class_definitions'] = (u'test.mpl.v1.app.Thing2',)
+        package3_data['fully_qualified_name'] += '_3'
+        package3_data['name'] += '_3'
+        package3_data['class_definitions'] = (u'test.mpl.v1.app.Thing3',)
+
+        expected_packages = [db_catalog_api.package_upload(package1_data, ''),
+                             db_catalog_api.package_upload(package2_data, '')]
+        db_catalog_api.package_upload(package3_data, '')
+
+        id_in = "in:" + ",".join(pack.id for pack in expected_packages)
+
+        req = self._get('/catalog/packages',
+                        params={'id': id_in})
+        self.expect_policy_check('get_package')
+        self.expect_policy_check('manage_public_package')
+
+        res = req.get_response(self.api)
+        self.assertEqual(200, res.status_code)
+
+        self.assertEqual(2, len(res.json['packages']))
+
+        found_packages = res.json['packages']
+
+        self.assertEqual([pack.id for pack in expected_packages],
+                         [pack['id'] for pack in found_packages])
+
+    def test_packages_filter_by_in_id_empty(self):
+        """Test that packages are filtered by "id=in:"
+
+        GET /catalog/packages with parameter "id=in:" returns packages
+        filtered by id, in this case no packages should be returned.
+        """
+        self._set_policy_rules(
+            {'get_package': '',
+             'manage_public_package': ''}
+        )
+        _, package1_data = self._test_package()
+
+        db_catalog_api.package_upload(package1_data, '')
+
+        req = self._get('/catalog/packages', params={'id': "in:"})
+        self.expect_policy_check('get_package')
+        self.expect_policy_check('manage_public_package')
+
+        res = req.get_response(self.api)
+        self.assertEqual(200, res.status_code)
+
+        self.assertEqual(0, len(res.json['packages']))
+
     def test_packages_filter_by_name(self):
         """Test that packages are filtered by name
 

murano/tests/unit/common/test_utils.py

@@ -0,0 +1,35 @@
+#    Copyright (c) 2013 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from murano.common import utils
+from murano.tests.unit import base
+
+
+class UtilsTests(base.MuranoTestCase):
+    def test_validate_quotes(self):
+        self.assertEqual(True, utils.validate_quotes('"ab"'))
+
+    def test_validate_quotes_not_closed_quotes(self):
+        self.assertRaises(ValueError, utils.validate_quotes, '"ab","b""')
+
+    def test_validate_quotes_no_coma_before_opening_quotes(self):
+        self.assertRaises(ValueError, utils.validate_quotes, '"ab""b"')
+
+    def test_split_for_quotes(self):
+        self.assertEqual(["a,b", "ac"], utils.split_for_quotes('"a,b","ac"'))
+
+    def test_split_for_quotes_with_backslash(self):
+        self.assertEqual(['a"bc', 'de', 'fg,h', r'klm\\', '"nop'],
+                         utils.split_for_quotes(r'"a\"bc","de",'
+                                                r'"fg,h","klm\\","\"nop"'))

releasenotes/notes/add_api_in_operator-371e3a1d2aec6421.yaml

@@ -0,0 +1,9 @@
+---
+
+features:
+  - Added an ability to filter operator values 'id', 'category', 'tag',
+    using the 'in' operator among the provided values.
+    An example of using the 'in' operator for 'id' is
+    'id=in:id1,id2,id3'.
+    This filter is added using syntax that conforms to the latest
+    guidelines from the OpenStack API-WG.