Introduce validation utils

This adds a "validation_utils" module to cloudkitty, which aims at
centralizing common voluptuous helpers.

Work items:

* Create the validation_utils module, and move the "get_string_type" helper
  function to it. API-specific helpers have not been moved.

* Update the module of the "get_string_type" helper function in the
  documentation.

* Add two validators to the "validation_utils" module, allowing to check
  the types of a dict's values and keys.

Change-Id: I938f78b9987fcc4752b84cff8881ceecb5e20caf
Story: 2005890
Task: 35657

cloudkitty/api/v2/scope/state.py

@@ -22,6 +22,7 @@ from cloudkitty.common import policy
 from cloudkitty import messaging
 from cloudkitty import storage_state
 from cloudkitty import tzutils
+from cloudkitty import validation_utils as vutils
 
 
 class ScopeState(base.BaseResource):
@@ -42,11 +43,11 @@ class ScopeState(base.BaseResource):
             api_utils.MultiQueryParam(str),
     })
     @api_utils.add_output_schema({'results': [{
-        voluptuous.Required('scope_id'): api_utils.get_string_type(),
+        voluptuous.Required('scope_id'): vutils.get_string_type(),
-        voluptuous.Required('scope_key'): api_utils.get_string_type(),
+        voluptuous.Required('scope_key'): vutils.get_string_type(),
-        voluptuous.Required('fetcher'): api_utils.get_string_type(),
+        voluptuous.Required('fetcher'): vutils.get_string_type(),
-        voluptuous.Required('collector'): api_utils.get_string_type(),
+        voluptuous.Required('collector'): vutils.get_string_type(),
-        voluptuous.Required('state'): api_utils.get_string_type(),
+        voluptuous.Required('state'): vutils.get_string_type(),
     }]})
     def get(self,
             offset=0,

cloudkitty/api/v2/utils.py

@@ -17,7 +17,6 @@ import itertools
 
 import flask
 import flask_restful
-import six
 import voluptuous
 from werkzeug import exceptions
 
@@ -33,7 +32,8 @@ class SingleQueryParam(object):
     containing a single element.
 
     Note that this validator uses ``voluptuous.Coerce`` internally and thus
-    should not be used together with ``api_utils.get_string_type`` in python2.
+    should not be used together with
+    ``cloudkitty.validation_utils.get_string_type`` in python2.
 
     :param param_type: Type of the query parameter
     """
@@ -57,7 +57,8 @@ class MultiQueryParam(object):
     containing a single element.
 
     Note that this validator uses ``voluptuous.Coerce`` internally and thus
-    should not be used together with ``api_utils.get_string_type`` in python2.
+    should not be used together with
+    ``cloudkitty.validation_utils.get_string_type`` in python2.
 
     :param param_type: Type of the query parameter
     """
@@ -224,7 +225,7 @@ def paginated(func):
                voluptuous.Required(
                    'message',
                    default='This is an example endpoint',
-               ): api_utils.get_string_type(),
+               ): validation_utils.get_string_type(),
            })
            def get(self, offset=0, limit=100):
                # [...]
@@ -248,7 +249,7 @@ def add_output_schema(schema):
                voluptuous.Required(
                    'message',
                    default='This is an example endpoint',
-               ): api_utils.get_string_type(),
+               ): validation_utils.get_string_type(),
            })
            def get(self):
                return {}
@@ -330,8 +331,3 @@ def do_init(app, blueprint_name, resources):
     if not blueprint_name.startswith('/'):
         blueprint_name = '/' + blueprint_name
     app.register_blueprint(blueprint, url_prefix=blueprint_name)
-
-
-def get_string_type():
-    """Returns ``basestring`` in python2 and ``str`` in python3."""
-    return six.string_types[0]

cloudkitty/tests/test_validation_utils.py

@@ -0,0 +1,92 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import unittest
+
+import voluptuous.error
+
+from cloudkitty import validation_utils
+
+
+class DictTypeValidatorTest(unittest.TestCase):
+
+    def test_dictvalidator_valid_dict_with_cast(self):
+        validator = validation_utils.DictTypeValidator(str, str)
+        self.assertEqual(validator({'a': '1', 'b': 2}), {'a': '1', 'b': '2'})
+
+    def test_dictvalidator_valid_dict_without_cast(self):
+        validator = validation_utils.DictTypeValidator(str, str, cast=False)
+        self.assertEqual(validator({'a': '1', 'b': '2'}), {'a': '1', 'b': '2'})
+
+    def test_dictvalidator_invalid_dict_without_cast(self):
+        validator = validation_utils.DictTypeValidator(str, str, cast=False)
+        self.assertRaises(
+            voluptuous.error.Invalid, validator, {'a': '1', 'b': 2})
+
+    def test_dictvalidator_invalid_dict_with_cast(self):
+        validator = validation_utils.DictTypeValidator(str, int)
+        self.assertRaises(
+            voluptuous.error.Invalid, validator, {'a': '1', 'b': 'aa'})
+
+    def test_dictvalidator_invalid_type_tuple(self):
+        validator = validation_utils.DictTypeValidator(str, int)
+        self.assertRaises(
+            voluptuous.error.Invalid, validator, ('a', '1'))
+
+    def test_dictvalidator_invalid_type_str(self):
+        validator = validation_utils.DictTypeValidator(str, int)
+        self.assertRaises(
+            voluptuous.error.Invalid, validator, 'aaaa')
+
+
+class IterableValuesDictTest(unittest.TestCase):
+
+    def test_iterablevaluesdict_valid_list_and_tuple_with_cast(self):
+        validator = validation_utils.IterableValuesDict(str, str)
+        self.assertEqual(
+            validator({'a': [1, '2'], 'b': ('3', 4)}),
+            {'a': ['1', '2'], 'b': ('3', '4')},
+        )
+
+    def test_iterablevaluesdict_valid_list_and_tuple_without_cast(self):
+        validator = validation_utils.IterableValuesDict(str, str)
+        self.assertEqual(
+            validator({'a': ['1', '2'], 'b': ('3', '4')}),
+            {'a': ['1', '2'], 'b': ('3', '4')},
+        )
+
+    def test_iterablevaluesdict_invalid_dict_iterable_without_cast(self):
+        validator = validation_utils.IterableValuesDict(str, str, cast=False)
+        self.assertRaises(
+            voluptuous.error.Invalid, validator, {'a': ['1'], 'b': (2, )})
+
+    def test_iterablevaluesdict_invalid_dict_iterable_with_cast(self):
+        validator = validation_utils.IterableValuesDict(str, int, cast=False)
+        self.assertRaises(
+            voluptuous.error.Invalid, validator, {'a': ['1'], 'b': ('aa', )})
+
+    def test_iterablevaluesdict_invalid_iterable_with_cast(self):
+        validator = validation_utils.IterableValuesDict(str, int)
+        self.assertRaises(
+            voluptuous.error.Invalid, validator, {'a': ['1'], 'b': 42, })
+
+    def test_iterablevaluesdict_invalid_type_tuple(self):
+        validator = validation_utils.IterableValuesDict(str, int)
+        self.assertRaises(
+            voluptuous.error.Invalid, validator, ('a', '1'))
+
+    def test_iterablevaluesdict_invalid_type_str(self):
+        validator = validation_utils.IterableValuesDict(str, int)
+        self.assertRaises(
+            voluptuous.error.Invalid, validator, 'aaaa')

cloudkitty/validation_utils.py

@@ -0,0 +1,98 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+"""Common utils for voluptuous schema validation"""
+try:
+    from collections.abc import Iterable
+except ImportError:
+    from collections import Iterable
+import functools
+
+import six
+
+import voluptuous
+
+
+class DictTypeValidator(object):
+    """Voluptuous helper validating dict key and value types.
+
+    When possible, keys and values will be converted to the required type.
+    This behaviour can be disabled through the `cast` param.
+
+    :param key_type: type of the dict keys
+    :param value_type: type of the dict values
+    :param cast: Set to False if you do not want to cast elements to the
+                 required type.
+    :type cast: bool
+    :rtype: dict
+    """
+
+    def __init__(self, key_type, value_type, cast=True):
+        if cast:
+            self._kval = voluptuous.Coerce(key_type)
+            self._vval = voluptuous.Coerce(value_type)
+        else:
+            def __type_validator(type_, elem):
+                if not isinstance(elem, type_):
+                    raise voluptuous.Invalid(
+                        "{e} is not of type {t}".format(e=elem, t=type_))
+                return elem
+
+            self._kval = functools.partial(__type_validator, key_type)
+            self._vval = functools.partial(__type_validator, value_type)
+
+    def __call__(self, item):
+        try:
+            return {self._kval(k): self._vval(v)
+                    for k, v in dict(item).items()}
+        except (TypeError, ValueError):
+            raise voluptuous.Invalid(
+                "{} can't be converted to dict".format(item))
+
+
+class IterableValuesDict(DictTypeValidator):
+    """Voluptuous helper validating dicts with iterable values.
+
+    When possible, keys and elements of values will be converted to the
+    required type. This behaviour can be disabled through the `cast`
+    param.
+
+    :param key_type: type of the dict keys
+    :param value_type: type of the dict values
+    :param cast: Set to False if you do not want to convert elements to the
+                 required type.
+    :type cast: bool
+    :rtype: dict
+    """
+    def __init__(self, key_type, value_type, cast=True):
+        super(IterableValuesDict, self).__init__(key_type, value_type, cast)
+        # NOTE(peschk_l): Using type(it) to return an iterable of the same
+        # type as the passed argument.
+        self.__vval = lambda it: type(it)(self._vval(i) for i in it)
+
+    def __call__(self, item):
+        try:
+            for v in dict(item).values():
+                if not isinstance(v, Iterable):
+                    raise voluptuous.Invalid("{} is not iterable".format(v))
+
+            return {self._kval(k): self.__vval(v) for k, v in item.items()}
+        except (TypeError, ValueError) as e:
+            raise voluptuous.Invalid(
+                "{} can't be converted to a dict: {}".format(item, e))
+
+
+def get_string_type():
+    """Returns ``basestring`` in python2 and ``str`` in python3."""
+    return six.string_types[0]

doc/source/developer/api/tutorial.rst

@@ -80,7 +80,7 @@ Let's update our ``get`` method in order to use this decorator:
    import voluptuous
 
    from cloudkitty.api.v2 import base
-   from cloudkitty.api.v2 import utils as api_utils
+   from cloudkitty import validation_utils
 
 
    class Example(base.BaseResource):
@@ -89,7 +89,7 @@ Let's update our ``get`` method in order to use this decorator:
            voluptuous.Required(
                'message',
                default='This is an example endpoint',
-           ): api_utils.get_string_type(),
+           ): validation_utils.get_string_type(),
        })
        def get(self):
            return {}
@@ -116,7 +116,7 @@ exceptions for HTTP return codes.
 .. code-block:: python
 
    @api_utils.add_input_schema('body', {
-       voluptuous.Required('fruit'): api_utils.get_string_type(),
+       voluptuous.Required('fruit'): validation_utils.get_string_type(),
    })
    def post(self, fruit=None):
        policy.authorize(flask.request.context, 'example:submit_fruit', {})
@@ -159,8 +159,8 @@ parameter is provided only once, and returns it.
 .. autoclass:: cloudkitty.api.v2.utils.SingleQueryParam
 
 .. warning:: ``SingleQueryParam`` uses ``voluptuous.Coerce`` internally for
-             type checking. Thus, ``api_utils.get_string_type`` cannot be used
+             type checking. Thus, ``validation_utils.get_string_type`` cannot
-             as ``basestring`` can't be instantiated.
+             be used as ``basestring`` can't be instantiated.
 
 
 Authorising methods

doc/source/developer/api/utils.rst

@@ -10,4 +10,4 @@
    page
 
 .. automodule:: cloudkitty.api.v2.utils
-   :members: SingleQueryParam, add_input_schema, paginated, add_output_schema, do_init, get_string_type
+   :members: SingleQueryParam, add_input_schema, paginated, add_output_schema, do_init