Add support for a resource level external_id

This adds support for the following to the template:
 heat_template_version: 2016-10-14
 resources:
   ...
   res_a:
     type: OS::Nova::Server
     external_id: the-new-server-id
     properties:
     ...

Co-Authored-By: Rico Lin <rico.l@inwinstack.com>

blueprint external-resources
Change-Id: I8fda1380504d1d8b1e96649bf20b86d6309fdeca

doc/source/template_guide/hot_spec.rst

@@ -634,6 +634,7 @@ the following syntax
        depends_on: <resource ID or list of ID>
        update_policy: <update policy>
        deletion_policy: <deletion policy>
+       external_id: <external resource ID>
 
 resource ID
     A resource ID which must be unique within the ``resources`` section of the
@@ -671,6 +672,16 @@ deletion_policy
     This attribute is optional; the default policy is to delete the physical
     resource when deleting a resource from the stack.
 
+external_id
+   Allows for specifying the resource_id for an existing external
+   (to the stack) resource. External resources can not depend on other
+   resources, but we allow other resources depend on external resource.
+   This attribute is optional.
+   Note: when this is specified, properties will not be used for building the
+   resource and the resource is not managed by Heat. This is not possible to
+   update that attribute. Also resource won't be deleted by heat when stack
+   is deleted.
+
 Depending on the type of resource, the resource block might include more
 resource specific data.
 

heat/common/exception.py

@@ -153,6 +153,11 @@ class InvalidTemplateReference(HeatException):
                 ' is incorrect.')
 
 
+class InvalidExternalResourceDependency(HeatException):
+    msg_fmt = _("Invalid dependency with external %(resource_type)s "
+                "resource: %(external_id)s")
+
+
 class EntityNotFound(HeatException):
     msg_fmt = _("The %(entity)s (%(name)s) could not be found.")
 

heat/engine/hot/template.py

@@ -10,6 +10,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import collections
 import six
 
 from heat.common import exception
@@ -70,6 +71,7 @@ class HOTemplate20130523(template_common.CommonTemplate):
     _HOT_TO_CFN_ATTRS.update(
         {OUTPUT_VALUE: cfn_template.CfnTemplate.OUTPUT_VALUE})
 
+    extra_rsrc_defn = ()
     functions = {
         'Fn::GetAZs': cfn_funcs.GetAZs,
         'get_param': hot_funcs.GetParam,
@@ -181,7 +183,8 @@ class HOTemplate20130523(template_common.CommonTemplate):
 
             for attr, attr_value in six.iteritems(attrs):
                 cfn_attr = mapping[attr]
-                cfn_object[cfn_attr] = attr_value
+                if cfn_attr is not None:
+                    cfn_object[cfn_attr] = attr_value
 
             cfn_objects[name] = cfn_object
 
@@ -216,6 +219,50 @@ class HOTemplate20130523(template_common.CommonTemplate):
                                         user_params=user_params,
                                         param_defaults=param_defaults)
 
+    def validate_resource_definitions(self, stack):
+        resources = self.t.get(self.RESOURCES) or {}
+        allowed_keys = set(self._RESOURCE_KEYS)
+
+        try:
+            for name, snippet in resources.items():
+                path = '.'.join([self.RESOURCES, name])
+                data = self.parse(stack, snippet, path)
+
+                if not self.validate_resource_key_type(self.RES_TYPE,
+                                                       six.string_types,
+                                                       'string',
+                                                       allowed_keys,
+                                                       name, data):
+                    args = {'name': name, 'type_key': self.RES_TYPE}
+                    msg = _('Resource %(name)s is missing '
+                            '"%(type_key)s"') % args
+                    raise KeyError(msg)
+                self._validate_resource_key_types(allowed_keys, name, data)
+        except (TypeError, ValueError) as ex:
+            raise exception.StackValidationFailed(message=six.text_type(ex))
+
+    def _validate_resource_key_types(self, allowed_keys, name, data):
+                self.validate_resource_key_type(
+                    self.RES_PROPERTIES,
+                    (collections.Mapping, function.Function),
+                    'object', allowed_keys, name, data)
+                self.validate_resource_key_type(
+                    self.RES_METADATA,
+                    (collections.Mapping, function.Function),
+                    'object', allowed_keys, name, data)
+                self.validate_resource_key_type(
+                    self.RES_DEPENDS_ON,
+                    collections.Sequence,
+                    'list or string', allowed_keys, name, data)
+                self.validate_resource_key_type(
+                    self.RES_DELETION_POLICY,
+                    (six.string_types, function.Function),
+                    'string', allowed_keys, name, data)
+                self.validate_resource_key_type(
+                    self.RES_UPDATE_POLICY,
+                    (collections.Mapping, function.Function),
+                    'object', allowed_keys, name, data)
+
     def resource_definitions(self, stack):
         resources = self.t.get(self.RESOURCES) or {}
         parsed_resources = self.parse(stack, resources)
@@ -245,7 +292,8 @@ class HOTemplate20130523(template_common.CommonTemplate):
             'update_policy': data.get(cls.RES_UPDATE_POLICY),
             'description': None
         }
-
+        for key in cls.extra_rsrc_defn:
+            kwargs[key] = data.get(key)
         return rsrc_defn.ResourceDefinition(name, **kwargs)
 
     def add_resource(self, definition, name=None):
@@ -377,7 +425,6 @@ class HOTemplate20160408(HOTemplate20151015):
 
 
 class HOTemplate20161014(HOTemplate20160408):
-
     CONDITIONS = 'conditions'
 
     SECTIONS = HOTemplate20160408.SECTIONS + (CONDITIONS,)
@@ -385,6 +432,20 @@ class HOTemplate20161014(HOTemplate20160408):
     _CFN_TO_HOT_SECTIONS = HOTemplate20160408._CFN_TO_HOT_SECTIONS
     _CFN_TO_HOT_SECTIONS.update({
         cfn_template.CfnTemplate.CONDITIONS: CONDITIONS})
+    _RESOURCE_KEYS = HOTemplate20160408._RESOURCE_KEYS
+    _EXT_KEY = (RES_EXTERNAL_ID,) = ('external_id',)
+    _RESOURCE_KEYS += _EXT_KEY
+    _RESOURCE_HOT_TO_CFN_ATTRS = HOTemplate20160408._RESOURCE_HOT_TO_CFN_ATTRS
+    _RESOURCE_HOT_TO_CFN_ATTRS.update({RES_EXTERNAL_ID: None})
+    extra_rsrc_defn = HOTemplate20160408.extra_rsrc_defn + (RES_EXTERNAL_ID,)
+
+    def _validate_resource_key_types(self, allowed_keys, name, data):
+        super(HOTemplate20161014, self)._validate_resource_key_types(
+            allowed_keys, name, data)
+        self.validate_resource_key_type(
+            self.RES_EXTERNAL_ID,
+            (six.string_types, function.Function),
+            'string', allowed_keys, name, data)
 
     deletion_policies = {
         'Delete': rsrc_defn.ResourceDefinition.DELETE,

heat/engine/resource.py

@@ -573,10 +573,10 @@ class Resource(object):
                                     if k in immutable_set]
 
         if update_replace_forbidden:
-            mesg = _("Update to properties %(props)s of %(name)s (%(res)s)"
-                     ) % {'props': ", ".join(sorted(update_replace_forbidden)),
-                          'res': self.type(), 'name': self.name}
-            raise exception.NotSupported(feature=mesg)
+            msg = _("Update to properties %(props)s of %(name)s (%(res)s)"
+                    ) % {'props': ", ".join(sorted(update_replace_forbidden)),
+                         'res': self.type(), 'name': self.name}
+            raise exception.NotSupported(feature=msg)
 
         if changed_properties_set and self.needs_replace_with_prop_diff(
                 changed_properties_set,
@@ -865,6 +865,13 @@ class Resource(object):
         Subclasses should provide a handle_create() method to customise
         creation.
         """
+        external = self.t.external_id()
+        if external is not None:
+            yield self._do_action(self.ADOPT,
+                                  resource_data={'resource_id': external})
+            self.check()
+            return
+
         action = self.CREATE
         if (self.action, self.status) != (self.INIT, self.COMPLETE):
             exc = exception.Error(_('State %s invalid for create')
@@ -1198,8 +1205,18 @@ class Resource(object):
         if before is None:
             before = self.frozen_definition()
 
-        after_props, before_props = self._prepare_update_props(
-            after, before)
+        external = after.external_id()
+        if before.external_id() != external:
+            msg = _("Update to property %(prop)s of %(name)s (%(res)s)"
+                    ) % {'prop': hot_tmpl.HOTemplate20161014.RES_EXTERNAL_ID,
+                         'res': self.type(), 'name': self.name}
+            exc = exception.NotSupported(feature=msg)
+            raise exception.ResourceFailure(exc, self, action)
+        elif external is not None:
+            LOG.debug("Skip update on external resource.")
+            return
+
+        after_props, before_props = self._prepare_update_props(after, before)
 
         yield self._break_if_required(
             self.UPDATE, environment.HOOK_PRE_UPDATE)

heat/engine/rsrc_defn.py

@@ -70,7 +70,7 @@ class ResourceDefinitionCore(object):
 
     def __init__(self, name, resource_type, properties=None, metadata=None,
                  depends=None, deletion_policy=None, update_policy=None,
-                 description=None):
+                 description=None, external_id=None):
         """Initialise with the parsed definition of a resource.
 
         Any intrinsic functions present in any of the sections should have been
@@ -84,6 +84,7 @@ class ResourceDefinitionCore(object):
         :param deletion_policy: The deletion policy for the resource
         :param update_policy: A dictionary of supplied update policies
         :param description: A string describing the resource
+        :param external_id: A uuid of an external resource
         """
         self.name = name
         self.resource_type = resource_type
@@ -93,6 +94,7 @@ class ResourceDefinitionCore(object):
         self._depends = depends
         self._deletion_policy = deletion_policy
         self._update_policy = update_policy
+        self._external_id = external_id
 
         self._hash = hash(self.resource_type)
         self._rendering = None
@@ -124,6 +126,12 @@ class ResourceDefinitionCore(object):
                                               function.Function))
             self._hash ^= _hash_data(update_policy)
 
+        if external_id is not None:
+            assert isinstance(external_id, (six.string_types,
+                                            function.Function))
+            self._hash ^= _hash_data(external_id)
+            self._deletion_policy = self.RETAIN
+
     def freeze(self, **overrides):
         """Return a frozen resource definition, with all functions resolved.
 
@@ -147,7 +155,7 @@ class ResourceDefinitionCore(object):
 
         args = ('name', 'resource_type', '_properties', '_metadata',
                 '_depends', '_deletion_policy', '_update_policy',
-                'description')
+                'description', '_external_id')
 
         defn = type(self)(**dict(arg_item(a) for a in args))
         defn._frozen = True
@@ -171,7 +179,8 @@ class ResourceDefinitionCore(object):
             metadata=reparse_snippet(self._metadata),
             depends=reparse_snippet(self._depends),
             deletion_policy=reparse_snippet(self._deletion_policy),
-            update_policy=reparse_snippet(self._update_policy))
+            update_policy=reparse_snippet(self._update_policy),
+            external_id=reparse_snippet(self._external_id))
 
     def dep_attrs(self, resource_name):
         """Iterate over attributes of a given resource that this references.
@@ -196,16 +205,26 @@ class ResourceDefinitionCore(object):
             return stack[res_name]
 
         def strict_func_deps(data, datapath):
-            return six.moves.filter(lambda r: getattr(r, 'strict_dependency',
-                                                      True),
-                                    function.dependencies(data, datapath))
+            return six.moves.filter(
+                lambda r: getattr(r, 'strict_dependency', True),
+                function.dependencies(data, datapath))
 
         explicit_depends = [] if self._depends is None else self._depends
+        prop_deps = strict_func_deps(self._properties, path(PROPERTIES))
+        metadata_deps = strict_func_deps(self._metadata, path(METADATA))
+
+        # (ricolin) External resource should not depend on any other resources.
+        # This operation is not allowed for now.
+        if self.external_id():
+            if explicit_depends:
+                raise exception.InvalidExternalResourceDependency(
+                    external_id=self.external_id(),
+                    resource_type=self.resource_type
+                )
+            return itertools.chain()
+
         return itertools.chain((get_resource(dep) for dep in explicit_depends),
-                               strict_func_deps(self._properties,
-                                                path(PROPERTIES)),
-                               strict_func_deps(self._metadata,
-                                                path(METADATA)))
+                               prop_deps, metadata_deps)
 
     def properties(self, schema, context=None):
         """Return a Properties object representing the resource properties.
@@ -238,6 +257,10 @@ class ResourceDefinitionCore(object):
         """Return the resource metadata."""
         return function.resolve(self._metadata) or {}
 
+    def external_id(self):
+        """Return the external resource id."""
+        return function.resolve(self._external_id)
+
     def render_hot(self):
         """Return a HOT snippet for the resource definition."""
         if self._rendering is None:
@@ -248,6 +271,7 @@ class ResourceDefinitionCore(object):
                 'deletion_policy': '_deletion_policy',
                 'update_policy': '_update_policy',
                 'depends_on': '_depends',
+                'external_id': '_external_id',
             }
 
             def rawattrs():

heat/tests/test_resource.py

@@ -335,6 +335,30 @@ class ResourceTest(common.HeatTestCase):
         actual = res.prepare_abandon()
         self.assertEqual(expected, actual)
 
+    def test_create_from_external(self):
+        tmpl = rsrc_defn.ResourceDefinition(
+            'test_resource', 'GenericResourceType',
+            external_id='f00d')
+        res = generic_rsrc.GenericResource('test_resource', tmpl, self.stack)
+        scheduler.TaskRunner(res.create)()
+        self.assertEqual((res.CHECK, res.COMPLETE), res.state)
+        self.assertEqual('f00d', res.resource_id)
+
+    def test_updated_from_external(self):
+        tmpl = rsrc_defn.ResourceDefinition('test_resource',
+                                            'GenericResourceType')
+        utmpl = rsrc_defn.ResourceDefinition(
+            'test_resource', 'GenericResourceType',
+            external_id='f00d')
+        res = generic_rsrc.GenericResource('test_resource', tmpl, self.stack)
+        expected_err_msg = ('NotSupported: resources.test_resource: Update '
+                            'to property external_id of test_resource '
+                            '(GenericResourceType) is not supported.')
+        err = self.assertRaises(exception.ResourceFailure,
+                                scheduler.TaskRunner(res.update, utmpl)
+                                )
+        self.assertEqual(expected_err_msg, six.text_type(err))
+
     def test_state_set_invalid(self):
         tmpl = rsrc_defn.ResourceDefinition('test_resource', 'Foo')
         res = generic_rsrc.GenericResource('test_resource', tmpl, self.stack)

heat/tests/test_rsrc_defn.py

@@ -15,11 +15,25 @@ import six
 import warnings
 
 from heat.common import exception
+from heat.common import template_format
 from heat.engine.cfn import functions as cfn_funcs
 from heat.engine.hot import functions as hot_funcs
 from heat.engine import properties
 from heat.engine import rsrc_defn
 from heat.tests import common
+from heat.tests import utils
+
+TEMPLATE_WITH_EX_REF_IMPLICIT_DEPEND = '''
+heat_template_version: 2016-10-14
+resources:
+  test1:
+    type: OS::Heat::TestResource
+    external_id: foobar
+    properties:
+        value: {get_resource: test2}
+  test2:
+    type: OS::Heat::TestResource
+'''
 
 
 class ResourceDefinitionTest(common.HeatTestCase):
@@ -76,6 +90,20 @@ class ResourceDefinitionTest(common.HeatTestCase):
         stack = {'foo': 'FOO', 'bar': 'BAR'}
         self.assertEqual(['FOO'], list(rd.dependencies(stack)))
 
+    def test_dependencies_explicit_ext(self):
+        rd = rsrc_defn.ResourceDefinition('rsrc', 'SomeType', depends=['foo'],
+                                          external_id='abc')
+        stack = {'foo': 'FOO', 'bar': 'BAR'}
+        self.assertRaises(
+            exception.InvalidExternalResourceDependency,
+            rd.dependencies, stack)
+
+    def test_dependencies_implicit_ext(self):
+        t = template_format.parse(TEMPLATE_WITH_EX_REF_IMPLICIT_DEPEND)
+        stack = utils.parse_stack(t)
+        rsrc = stack['test1']
+        self.assertEqual([], list(rsrc.t.dependencies(stack)))
+
     def test_dependencies_explicit_invalid(self):
         rd = rsrc_defn.ResourceDefinition('rsrc', 'SomeType', depends=['baz'])
         stack = {'foo': 'FOO', 'bar': 'BAR'}

heat_integrationtests/functional/test_external_ref.py

@@ -0,0 +1,83 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from heat_integrationtests.functional import functional_base
+
+
+class ExternalReferencesTest(functional_base.FunctionalTestsBase):
+
+    TEMPLATE = '''
+heat_template_version: 2016-10-14
+resources:
+  test1:
+    type: OS::Heat::TestResource
+'''
+    TEMPLATE_WITH_EX_REF = '''
+heat_template_version: 2016-10-14
+resources:
+  test1:
+    type: OS::Heat::TestResource
+    external_id: foobar
+outputs:
+  str:
+    value: {get_resource: test1}
+'''
+
+    def test_create_with_external_ref(self):
+        stack_name = self._stack_rand_name()
+        stack_identifier = self.stack_create(
+            stack_name=stack_name,
+            template=self.TEMPLATE_WITH_EX_REF,
+            files={},
+            disable_rollback=True,
+            parameters={},
+            environment={}
+        )
+
+        stack = self.client.stacks.get(stack_identifier)
+
+        self._wait_for_stack_status(stack_identifier, 'CREATE_COMPLETE')
+        expected_resources = {'test1': 'OS::Heat::TestResource'}
+        self.assertEqual(expected_resources,
+                         self.list_resources(stack_identifier))
+        stack = self.client.stacks.get(stack_identifier)
+        self.assertEqual(
+            [{'description': 'No description given',
+              'output_key': 'str',
+              'output_value': 'foobar'}], stack.outputs)
+
+    def test_update_with_external_ref(self):
+        stack_name = self._stack_rand_name()
+        stack_identifier = self.stack_create(
+            stack_name=stack_name,
+            template=self.TEMPLATE,
+            files={},
+            disable_rollback=True,
+            parameters={},
+            environment={}
+        )
+        stack = self.client.stacks.get(stack_identifier)
+
+        self._wait_for_stack_status(stack_identifier, 'CREATE_COMPLETE')
+        expected_resources = {'test1': 'OS::Heat::TestResource'}
+        self.assertEqual(expected_resources,
+                         self.list_resources(stack_identifier))
+        stack = self.client.stacks.get(stack_identifier)
+        self.assertEqual([], stack.outputs)
+
+        stack_name = stack_identifier.split('/')[0]
+        kwargs = {'stack_id': stack_identifier, 'stack_name': stack_name,
+                  'template': self.TEMPLATE_WITH_EX_REF, 'files': {},
+                  'disable_rollback': True, 'parameters': {}, 'environment': {}
+                  }
+        self.client.stacks.update(**kwargs)
+        self._wait_for_stack_status(stack_identifier, 'UPDATE_FAILED')

releasenotes/notes/external-resources-965d01d690d32bd2.yaml

@@ -0,0 +1,10 @@
+---
+prelude: >
+    Support external resource reference in template.
+features:
+  - Add `external_id` attribute for resource to reference
+    on an exists external resource. The resource (with
+    `external_id` attribute) will not able to be updated.
+    This will keep management rights stay externally.
+  - This feature only supports templates with version over
+    `2016-10-14`.