Allow null values to be returned from Macros

This change allows a Macro returns a 'null' value (as opposed to None)
that makes it effectively cease to exist when that makes sense. If it
appears as a list item, that list item is dropped. If it appears as a
value in a dict, the corresponding key is removed from the dict. If it
appears as a property value, the property is treated as if the user had
not mentioned it in the template.

In all other circumstances (as an argument to a Function, as the top
level of some other field, like an output value), the result will still
appear as None.

A null value is represented internally by the Ellipsis singleton (to
distinguish it from None, which is a value that may appear in the
template from a user or be returned from a Function).

Change-Id: Iaed0982e0db902f6eaf8f986c12b4885bd77e8b9
Story: 2007388
Task: 38976

heat/engine/function.py

@@ -195,7 +195,7 @@ class Macro(Function, metaclass=abc.ABCMeta):
 
     def result(self):
         """Return the resolved result of the macro contents."""
-        return resolve(self.parsed)
+        return resolve(self.parsed, nullable=True)
 
     def dependencies(self, path):
         return dependencies(self.parsed, '.'.join([path, self.fn_name]))
@@ -250,15 +250,30 @@ class Macro(Function, metaclass=abc.ABCMeta):
         return repr(self.parsed)
 
 
-def resolve(snippet):
+def _non_null_item(i):
+    k, v = i
+    return v is not Ellipsis
+
+
+def _non_null_value(v):
+    return v is not Ellipsis
+
+
+def resolve(snippet, nullable=False):
     if isinstance(snippet, Function):
-        return snippet.result()
+        result = snippet.result()
+        if not (nullable or _non_null_value(result)):
+            result = None
+        return result
 
     if isinstance(snippet, collections.Mapping):
-        return dict((k, resolve(v)) for k, v in snippet.items())
+        return dict(filter(_non_null_item,
+                           ((k, resolve(v, nullable=True))
+                            for k, v in snippet.items())))
     elif (not isinstance(snippet, str) and
           isinstance(snippet, collections.Iterable)):
-        return [resolve(v) for v in snippet]
+        return list(filter(_non_null_value,
+                           (resolve(v, nullable=True) for v in snippet)))
 
     return snippet
 

heat/engine/properties.py

@@ -372,9 +372,14 @@ class Property(object):
         return _value
 
 
+def _default_resolver(d, nullable=False):
+    return d
+
+
 class Properties(collections.Mapping):
 
-    def __init__(self, schema, data, resolver=lambda d: d, parent_name=None,
+    def __init__(self, schema, data, resolver=_default_resolver,
+                 parent_name=None,
                  context=None, section=None, translation=None,
                  rsrc_description=None):
         self.props = dict((k, Property(s, k, context, path=parent_name))
@@ -464,6 +469,11 @@ class Properties(collections.Mapping):
     def _resolve_user_value(self, key, prop, validate):
         """Return the user-supplied value (or None), and whether it was found.
 
+        This allows us to distinguish between, on the one hand, either a
+        Function that returns None or an explicit null value passed and, on the
+        other hand, either no value passed or a Macro that returns Ellipsis,
+        meaning that the result should be treated the same as if no value were
+        passed.
         """
         if key not in self.data:
             return None, False
@@ -478,7 +488,10 @@ class Properties(collections.Mapping):
                 if self._find_deps_any_in_init(unresolved_value):
                     validate = False
 
-            value = self.resolve(unresolved_value)
+            value = self.resolve(unresolved_value, nullable=True)
+            if value is Ellipsis:
+                # Treat as if the property value were not specified at all
+                return None, False
 
             if self.translation.has_translation(prop.path):
                 value = self.translation.translate(prop.path,

heat/engine/resources/openstack/heat/resource_group.py

@@ -600,19 +600,25 @@ class ResourceGroup(stack_resource.StackResource):
         # At this stage, we don't mind if all of the parameters have values
         # assigned. Pass in a custom resolver to the properties to not
         # error when a parameter does not have a user entered value.
-        def ignore_param_resolve(snippet):
+        def ignore_param_resolve(snippet, nullable=False):
             if isinstance(snippet, function.Function):
                 try:
-                    return snippet.result()
+                    result = snippet.result()
                 except exception.UserParameterMissing:
                     return None
+                if not (nullable or function._non_null_value(result)):
+                    result = None
+                return result
 
             if isinstance(snippet, collections.Mapping):
-                return dict((k, ignore_param_resolve(v))
-                            for k, v in snippet.items())
+                return dict(filter(function._non_null_item,
+                                   ((k, ignore_param_resolve(v, nullable=True))
+                                    for k, v in snippet.items())))
             elif (not isinstance(snippet, str) and
                   isinstance(snippet, collections.Iterable)):
-                return [ignore_param_resolve(v) for v in snippet]
+                return list(filter(function._non_null_value,
+                                   (ignore_param_resolve(v, nullable=True)
+                                    for v in snippet)))
 
             return snippet
 

heat/tests/test_function.py

@@ -41,6 +41,11 @@ class TestFunction(function.Function):
         return 'wibble'
 
 
+class NullFunction(function.Function):
+    def result(self):
+        return Ellipsis
+
+
 class TestFunctionKeyError(function.Function):
     def result(self):
         raise TypeError
@@ -169,6 +174,28 @@ class ResolveTest(common.HeatTestCase):
                          result)
         self.assertIsNot(result, snippet)
 
+    def test_resolve_func_with_null(self):
+        func = NullFunction(None, 'foo', ['bar', 'baz'])
+
+        self.assertIsNone(function.resolve(func))
+        self.assertIs(Ellipsis, function.resolve(func, nullable=True))
+
+    def test_resolve_dict_with_null(self):
+        func = NullFunction(None, 'foo', ['bar', 'baz'])
+        snippet = {'foo': 'bar', 'baz': func, 'blarg': 'wibble'}
+
+        result = function.resolve(snippet)
+
+        self.assertEqual({'foo': 'bar', 'blarg': 'wibble'}, result)
+
+    def test_resolve_list_with_null(self):
+        func = NullFunction(None, 'foo', ['bar', 'baz'])
+        snippet = ['foo', func, 'bar']
+
+        result = function.resolve(snippet)
+
+        self.assertEqual(['foo', 'bar'], result)
+
 
 class ValidateTest(common.HeatTestCase):
     def setUp(self):

heat/tests/test_properties.py

@@ -16,6 +16,7 @@ from oslo_serialization import jsonutils
 
 from heat.common import exception
 from heat.engine import constraints
+from heat.engine import function
 from heat.engine.hot import functions as hot_funcs
 from heat.engine.hot import parameters as hot_param
 from heat.engine import parameters
@@ -1072,7 +1073,7 @@ class PropertiesTest(common.HeatTestCase):
             'default_override': 21,
         }
 
-        def double(d):
+        def double(d, nullable=False):
             return d * 2
 
         self.props = properties.Properties(schema, data, double, 'wibble')
@@ -1207,7 +1208,7 @@ class PropertiesTest(common.HeatTestCase):
     def test_resolve_returns_none(self):
         schema = {'foo': {'Type': 'String', "MinLength": "5"}}
 
-        def test_resolver(prop):
+        def test_resolver(prop, nullable=False):
             return None
 
         self.patchobject(properties.Properties,
@@ -1244,7 +1245,7 @@ class PropertiesTest(common.HeatTestCase):
         }
 
         # define parameters for function
-        def test_resolver(prop):
+        def test_resolver(prop, nullable=False):
             return 'None'
 
         class rsrc(object):
@@ -1675,6 +1676,26 @@ class PropertiesTest(common.HeatTestCase):
         bar_props = bar_rsrc.properties(schema)
         self.assertEqual('bar', bar_props['description'])
 
+    def test_null_property_value(self):
+        class NullFunction(function.Function):
+            def result(self):
+                return Ellipsis
+
+        schema = {
+            'Foo': properties.Schema('String', required=False),
+            'Bar': properties.Schema('String', required=False),
+            'Baz': properties.Schema('String', required=False),
+        }
+        user_props = {'Foo': NullFunction(None, 'null', []), 'Baz': None}
+        props = properties.Properties(schema, user_props, function.resolve)
+
+        self.assertEqual(None, props['Foo'])
+        self.assertEqual(None, props.get_user_value('Foo'))
+        self.assertEqual(None, props['Bar'])
+        self.assertEqual(None, props.get_user_value('Bar'))
+        self.assertEqual('', props['Baz'])
+        self.assertEqual('', props.get_user_value('Baz'))
+
 
 class PropertiesValidationTest(common.HeatTestCase):
     def test_required(self):

heat/tests/test_rsrc_defn.py

@@ -15,6 +15,7 @@
 from heat.common import exception
 from heat.common import template_format
 from heat.engine.cfn import functions as cfn_funcs
+from heat.engine import function
 from heat.engine.hot import functions as hot_funcs
 from heat.engine import properties
 from heat.engine import rsrc_defn
@@ -169,6 +170,39 @@ class ResourceDefinitionTest(common.HeatTestCase):
         self.assertEqual('bar', frozen._properties['Foo'])
         self.assertEqual('wibble', frozen._metadata['Baz'])
 
+    def test_freeze_nullable_top_level(self):
+        class NullFunction(function.Function):
+            def result(self):
+                return Ellipsis
+
+        null_func = NullFunction(None, 'null', [])
+
+        rd = rsrc_defn.ResourceDefinition(
+            'rsrc', 'SomeType',
+            properties=null_func,
+            metadata=null_func,
+            update_policy=null_func)
+
+        frozen = rd.freeze()
+        self.assertIsNone(frozen._properties)
+        self.assertIsNone(frozen._metadata)
+        self.assertIsNone(frozen._update_policy)
+
+        rd2 = rsrc_defn.ResourceDefinition(
+            'rsrc', 'SomeType',
+            properties={'Foo': null_func,
+                        'Blarg': 'wibble'},
+            metadata={'Bar': null_func,
+                      'Baz': 'quux'},
+            update_policy={'some_policy': null_func})
+
+        frozen2 = rd2.freeze()
+        self.assertNotIn('Foo', frozen2._properties)
+        self.assertEqual('wibble', frozen2._properties['Blarg'])
+        self.assertNotIn('Bar', frozen2._metadata)
+        self.assertEqual('quux', frozen2._metadata['Baz'])
+        self.assertEqual({}, frozen2._update_policy)
+
     def test_render_hot(self):
         rd = self.make_me_one_with_everything()