Implement caching of resource attributes

The default caching mode cache_local will return the previously
resolved value if it is available.

Whenever any resource changes state, all resource attribute caches
are cleared just in case the state change has side-effects in
other resources.

The caching mode cache_none performs no caching, and is chosen
for attributes which are one of the following:
* Derived from any resource's metadata, resource data or resource_id
* An API call which returns a secret

Caching currently only exists for the duration of the parser.Stack
object, but there is future potential for a caching mode which
caches attributes spanning multiple requests.

Closes-Bug: #1321970

Change-Id: I01bf2983b726f0e81a2b8d5be94627353bdeb406

contrib/marconi/marconi/resources/queue.py

@@ -57,7 +57,8 @@ class MarconiQueue(resource.Resource):
 
     attributes_schema = {
         QUEUE_ID: attributes.Schema(
-            _("ID of the queue.")
+            _("ID of the queue."),
+            cache_mode=attributes.Schema.CACHE_NONE
         ),
         HREF: attributes.Schema(
             _("The resource href of the queue.")

contrib/rackspace/rackspace/resources/auto_scale.py

@@ -513,10 +513,12 @@ class WebHook(resource.Resource):
 
     attributes_schema = {
         EXECUTE_URL: attributes.Schema(
-            _("The url for executing the webhook (requires auth).")
+            _("The url for executing the webhook (requires auth)."),
+            cache_mode=attributes.Schema.CACHE_NONE
         ),
         CAPABILITY_URL: attributes.Schema(
-            _("The url for executing the webhook (doesn't require auth).")
+            _("The url for executing the webhook (doesn't require auth)."),
+            cache_mode=attributes.Schema.CACHE_NONE
         ),
     }
 

contrib/rackspace/rackspace/resources/cloud_server.py

@@ -79,7 +79,8 @@ class CloudServer(server.Server):
                 _('The private IPv4 address of the server.')
             ),
             ADMIN_PASS_ATTR: attributes.Schema(
-                _('The administrator password for the server.')
+                _('The administrator password for the server.'),
+                cache_mode=attributes.Schema.CACHE_NONE
             ),
         }
     )

heat/engine/attributes.py

@@ -32,10 +32,20 @@ class Schema(constr.Schema):
         'description',
     )
 
+    CACHE_MODES = (
+        CACHE_LOCAL,
+        CACHE_NONE
+    ) = (
+        'cache_local',
+        'cache_none'
+    )
+
     def __init__(self, description=None,
-                 support_status=support.SupportStatus()):
+                 support_status=support.SupportStatus(),
+                 cache_mode=CACHE_LOCAL):
         self.description = description
         self.support_status = support_status
+        self.cache_mode = cache_mode
 
     def __getitem__(self, key):
         if key == self.DESCRIPTION:
@@ -104,6 +114,10 @@ class Attributes(collections.Mapping):
         self._resource_name = res_name
         self._resolver = resolver
         self._attributes = Attributes._make_attributes(schema)
+        self.reset_resolved_values()
+
+    def reset_resolved_values(self):
+        self._resolved_values = {}
 
     @staticmethod
     def _make_attributes(schema):
@@ -133,7 +147,20 @@ class Attributes(collections.Mapping):
         if key not in self:
             raise KeyError(_('%(resource)s: Invalid attribute %(key)s') %
                            dict(resource=self._resource_name, key=key))
-        return self._resolver(key)
+
+        attrib = self._attributes.get(key)
+        if attrib.schema.cache_mode == Schema.CACHE_NONE:
+            return self._resolver(key)
+
+        if key in self._resolved_values:
+            return self._resolved_values[key]
+
+        value = self._resolver(key)
+        if value is not None:
+            # only store if not None, it may resolve to an actual value
+            # on subsequent calls
+            self._resolved_values[key] = value
+        return value
 
     def __len__(self):
         return len(self._attributes)

heat/engine/parser.py

@@ -862,3 +862,12 @@ class Stack(collections.Mapping):
                       'Use heat.engine.function.resolve() instead',
                       DeprecationWarning)
         return function.resolve(snippet)
+
+    def reset_resource_attributes(self):
+        # nothing is cached if no resources exist
+        if not self._resources:
+            return
+        # a change in some resource may have side-effects in the attributes
+        # of other resources, so ensure that attributes are re-calculated
+        for res in self.resources.itervalues():
+            res.attributes.reset_resolved_values()

heat/engine/resource.py

@@ -796,6 +796,8 @@ class Resource(object):
         if new_state != old_state:
             self._add_event(action, status, reason)
 
+        self.stack.reset_resource_attributes()
+
     @property
     def state(self):
         '''Returns state, tuple of action, status.'''

heat/engine/resources/nova_keypair.py

@@ -73,7 +73,8 @@ class KeyPair(resource.Resource):
             _('The public key.')
         ),
         PRIVATE_KEY: attributes.Schema(
-            _('The private key if it has been saved.')
+            _('The private key if it has been saved.'),
+            cache_mode=attributes.Schema.CACHE_NONE
         ),
     }
 

heat/engine/resources/random_string.py

@@ -71,7 +71,8 @@ class RandomString(resource.Resource):
     attributes_schema = {
         VALUE: attributes.Schema(
             _('The random string generated by this resource. This value is '
-              'also available by referencing the resource.')
+              'also available by referencing the resource.'),
+            cache_mode=attributes.Schema.CACHE_NONE
         ),
     }
 

heat/engine/resources/user.py

@@ -160,10 +160,12 @@ class AccessKey(resource.Resource):
 
     attributes_schema = {
         USER_NAME: attributes.Schema(
-            _('Username associated with the AccessKey.')
+            _('Username associated with the AccessKey.'),
+            cache_mode=attributes.Schema.CACHE_NONE
         ),
         SECRET_ACCESS_KEY: attributes.Schema(
-            _('Keypair secret key.')
+            _('Keypair secret key.'),
+            cache_mode=attributes.Schema.CACHE_NONE
         ),
     }
 

heat/engine/resources/wait_condition.py

@@ -187,7 +187,8 @@ class WaitCondition(resource.Resource):
     attributes_schema = {
         DATA: attributes.Schema(
             _('JSON serialized dict containing data associated with wait '
-              'condition signals sent to the handle.')
+              'condition signals sent to the handle.'),
+            cache_mode=attributes.Schema.CACHE_NONE
         ),
     }
 

heat/tests/test_attributes.py

@@ -86,7 +86,9 @@ class AttributesTest(common.HeatTestCase):
     attributes_schema = {
         "test1": attributes.Schema("Test attrib 1"),
         "test2": attributes.Schema("Test attrib 2"),
-        "test3": attributes.Schema("Test attrib 3"),
+        "test3": attributes.Schema(
+            "Test attrib 3",
+            cache_mode=attributes.Schema.CACHE_NONE)
     }
 
     def setUp(self):
@@ -147,3 +149,28 @@ class AttributesTest(common.HeatTestCase):
             expected,
             attributes.Attributes.as_outputs("test_resource",
                                              MyTestResourceClass))
+
+    def test_caching_local(self):
+        value = 'value1'
+        test_resolver = lambda x: value
+        self.m.ReplayAll()
+        attribs = attributes.Attributes('test resource',
+                                        self.attributes_schema,
+                                        test_resolver)
+        self.assertEqual("value1", attribs['test1'])
+        value = 'value1 changed'
+        self.assertEqual("value1", attribs['test1'])
+
+        attribs.reset_resolved_values()
+        self.assertEqual("value1 changed", attribs['test1'])
+
+    def test_caching_none(self):
+        value = 'value3'
+        test_resolver = lambda x: value
+        self.m.ReplayAll()
+        attribs = attributes.Attributes('test resource',
+                                        self.attributes_schema,
+                                        test_resolver)
+        self.assertEqual("value3", attribs['test3'])
+        value = 'value3 changed'
+        self.assertEqual("value3 changed", attribs['test3'])