API attribute processing: allow to populate dict attribute default values

Currently, the neutron-lib API code fills defaults values for the
attributes defined in a resource map, but does not fill defaults inside
the attributes that are dictionaries.

This change introduces a 'dict_populate_defaults' flag that allows the
defaults to be filled inside dictionary attributes (attributes of any
dictionary type).

This is useful to incorporate networking-sfc APIs and avoid using
the specific normalize_* converter methods [1] (see follow-up changes).

[1] https://github.com/openstack/networking-sfc/blob/master/networking_sfc/extensions/sfc.py#L226-L239

Change-Id: I0d7ff1981aa92d17811233e29a6ca7264a9ddb6c

doc/source/contributor/api_attributes.rst

@@ -91,6 +91,7 @@ The following are the defined keys for attribute maps:
 ``enforce_policy``          the attribute is actively part of the policy enforcing mechanism, ie: there might be rules which refer to this attribute
 ``primary_key``             Mark the attribute as a unique key.
 ``default_overrides_none``  if set, if the value passed is None, it will be replaced by the ``default`` value
+``dict_populate_defaults``  if set, the ``default`` values of keys inside dict attributes, will be filled if not specified
 ==========================  ======
 
 When extending existing sub-resources, the sub-attribute map must define all

neutron_lib/api/attributes.py

@@ -70,6 +70,72 @@ def _fill_default(res_dict, attr_name, attr_spec):
                                        attr_spec.get('default'))
 
 
+def _dict_populate_defaults(attr_value, attr_spec):
+    # attr_value: dict
+    # attr_spec: an attribute specification dict e.g.
+    # {
+    #      ...
+    #      'dict_populate_defaults': True,
+    #      'default_overrides_none': ,
+    #      'validate': {
+    #                 'type:dict': {
+    #                     'foo': {
+    #                         'default': FOO_DEFAULT,
+    #                         'type:values': [42, 43]
+    #                     },
+    #                     'bar': {
+    #                         'default': BAR_DEFAULT,
+    #                         'convert_to': converters.convert_to_boolean
+    #                     }
+    #                     'baz': {
+    #                         'dict_populate_defaults': True
+    #                         'default_overrides_none': True,
+    #                         'type:dict': {
+    #                             'baz_bar: {
+    #                                 'default': 77,
+    #                                 'type:boolean'
+    #                             },
+    #                             'baz_foo': {
+    #                                 'default': 88,
+    #                                 'type:boolean'
+    #                             }
+    #                         }
+    #                     }
+    #             }
+    #      }
+    # }
+    if not attr_spec.get(constants.DICT_POPULATE_DEFAULTS):
+        return attr_value
+
+    if attr_value is None or attr_value is constants.ATTR_NOT_SPECIFIED:
+        attr_value = {}
+
+    for rule_type, rule_content in attr_spec['validate'].items():
+        # we only recursively apply defaults for dict rules
+        if 'dict' not in rule_type:
+            continue
+        for key, key_validator in rule_content.items():
+            validator_name, _dummy, validator_params = (
+                validators._extract_validator(key_validator))
+
+            # recurse if required:
+            if 'dict' in validator_name:
+                value = _dict_populate_defaults(
+                    attr_value.get(key),
+                    {
+                        constants.DICT_POPULATE_DEFAULTS: key_validator.get(
+                            constants.DICT_POPULATE_DEFAULTS),
+                        'validate': {validator_name: validator_params}
+                    }
+                )
+                if value is not None:
+                    attr_value[key] = value
+
+            _fill_default(attr_value, key, key_validator)
+
+    return attr_value
+
+
 class AttributeInfo(object):
     """Provides operations on a resource's attribute map.
 
@@ -122,10 +188,17 @@ class AttributeInfo(object):
         """
         for attr, attr_vals in self.attributes.items():
             if attr_vals['allow_post']:
+                value = _dict_populate_defaults(
+                    res_dict.get(attr, constants.ATTR_NOT_SPECIFIED),
+                    attr_vals)
+                if value is not constants.ATTR_NOT_SPECIFIED:
+                    res_dict[attr] = value
+
                 if 'default' not in attr_vals and attr not in res_dict:
                     msg = _("Failed to parse request. Required "
                             "attribute '%s' not specified") % attr
                     raise exc_cls(msg)
+
                 _fill_default(res_dict, attr, attr_vals)
             elif check_allow_post:
                 if attr in res_dict:
@@ -155,8 +228,8 @@ class AttributeInfo(object):
                 continue
             for rule in attr_vals['validate']:
                 validator = validators.get_validator(rule)
-                res = validator(res_dict[attr], attr_vals['validate'][rule])
+                res = validator(res_dict[attr],
-
+                                attr_vals['validate'][rule])
                 if res:
                     msg_dict = dict(attr=attr, reason=res)
                     msg = _("Invalid input for %(attr)s. "

neutron_lib/api/definitions/base.py

@@ -153,4 +153,5 @@ KNOWN_KEYWORDS = (
     'required_by_policy',
     'validate',
     'default_overrides_none',
+    'dict_populate_defaults',
 )

neutron_lib/api/validators/__init__.py

@@ -805,29 +805,39 @@ def validate_uuid_list(data, valid_values=None):
     return _validate_uuid_list(data, valid_values)
 
 
+def _extract_validator(key_validator):
+    # Find validator function in key validation spec
+    #
+    # TODO(salv-orlando): Structure of dict attributes should be improved
+    # to avoid iterating over items
+    for (k, v) in key_validator.items():
+        if k.startswith('type:'):
+            (validator_name, validator_params) = (k, v)
+            try:
+                return (validator_name,
+                        validators[validator_name],
+                        validator_params)
+            except KeyError:
+                raise UndefinedValidator(validator_name)
+    return None, None, None
+
+
 def _validate_dict_item(key, key_validator, data):
     # Find conversion function, if any, and apply it
     conv_func = key_validator.get('convert_to')
     if conv_func:
         data[key] = conv_func(data.get(key))
-    # Find validator function
+    try:
-    # TODO(salv-orlando): Structure of dict attributes should be improved
+        dummy_, val_func, val_params = _extract_validator(key_validator)
-    # to avoid iterating over items
+        if val_func:
-    val_func = val_params = None
+            return val_func(data.get(key), val_params)
-    for (k, v) in key_validator.items():
+        # NOTE(tmorin): here we silently omit to validate a key for which
-        if k.startswith('type:'):
+        # no type validator has been defined
-            # ask forgiveness, not permission
+    except UndefinedValidator as e:
-            try:
+        # NOTE(tmorin): Should we really return an API error on such
-                val_func = validators[k]
+        # an issue. Wouldn't InternalServer error be more natural ?
-            except KeyError:
+        LOG.debug(e.message)
-                msg = _("Validator '%s' does not exist") % k
+        return e.message
-                LOG.debug(msg)
-                return msg
-            val_params = v
-            break
-    # Process validation
-    if val_func:
-        return val_func(data.get(key), val_params)
 
 
 def validate_dict(data, key_specs=None):
@@ -1090,6 +1100,13 @@ validators = {'type:dict': validate_dict,
               }
 
 
+class UndefinedValidator(Exception):
+
+    def __init__(self, validator_name):
+        self.validator_name = validator_name
+        self.message = _("Validator '%s' does not exist") % self.validator_name
+
+
 def _to_validation_type(validation_type):
     return (validation_type
             if validation_type.startswith('type:')

neutron_lib/constants.py

@@ -345,6 +345,8 @@ class Sentinel(object):
 
 ATTR_NOT_SPECIFIED = Sentinel()
 
+DICT_POPULATE_DEFAULTS = 'dict_populate_defaults'
+
 HEX_ELEM = '[0-9A-Fa-f]'
 UUID_PATTERN = '-'.join([HEX_ELEM + '{8}', HEX_ELEM + '{4}',
                          HEX_ELEM + '{4}', HEX_ELEM + '{4}',

neutron_lib/tests/unit/api/definitions/base.py

@@ -28,7 +28,8 @@ def assert_bool(tester, attribute, attribute_dict, keyword, value):
 
 def assert_converter(tester, attribute, attribute_dict, keyword, value):
     if ('default' not in attribute_dict or
-            attribute_dict['default'] is constants.ATTR_NOT_SPECIFIED):
+            attribute_dict['default'] is constants.ATTR_NOT_SPECIFIED or
+            attribute_dict.get(constants.DICT_POPULATE_DEFAULTS)):
         return
     try:
         attribute_dict['convert_to'](attribute_dict['default'])
@@ -65,6 +66,7 @@ ASSERT_FUNCTIONS = {
     'required_by_policy': assert_bool,
     'validate': assert_validator,
     'default_overrides_none': assert_bool,
+    'dict_populate_defaults': assert_bool,
 }
 
 

releasenotes/notes/populate-dict-defaults-3f205c414f21bf54.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    A new ``dict_populate_defaults`` flag can be used in API definition for
+    a dictionary attribute, which will results in default values for the keys
+    to be filled in. This can also be used on values of a dictionary attribute
+    if they are dictionaries as well.