De-duplicate raw_template.files

Save space in the db by allowing templates (especially nested
templates) to reference the same template files dictionary objects,
now stored in the table raw_template_files. Also, cache template files
per heat-engine process as a RAM/performance optimization.

Also, begin to allow for future rolling upgrades with
oslo.versionedobjects.

Change-Id: I1842ea6ba6773a135b1007b793a5f0884417747d
Closes-Bug: 1570983

heat/db/api.py

@@ -58,6 +58,14 @@ def raw_template_delete(context, template_id):
     return IMPL.raw_template_delete(context, template_id)
 
 
+def raw_template_files_create(context, values):
+    return IMPL.raw_template_files_create(context, values)
+
+
+def raw_template_files_get(context, tmpl_files_id):
+    return IMPL.raw_template_files_get(context, tmpl_files_id)
+
+
 def resource_data_get_all(context, resource_id, data=None):
     return IMPL.resource_data_get_all(context, resource_id, data)
 

heat/db/sqlalchemy/api.py

@@ -133,7 +133,33 @@ def raw_template_update(context, template_id, values):
 
 def raw_template_delete(context, template_id):
     raw_template = raw_template_get(context, template_id)
+    raw_tmpl_files_id = raw_template.files_id
     raw_template.delete()
+    if raw_tmpl_files_id is None:
+        return
+    # If no other raw_template is referencing the same raw_template_files,
+    # delete that too
+    if _session(context).query(models.RawTemplate).filter_by(
+            files_id=raw_tmpl_files_id).first() is None:
+        raw_template_files_get(context, raw_tmpl_files_id).delete()
+
+
+def raw_template_files_create(context, values):
+    session = _session(context)
+    raw_templ_files_ref = models.RawTemplateFiles()
+    raw_templ_files_ref.update(values)
+    with session.begin():
+        raw_templ_files_ref.save(session)
+    return raw_templ_files_ref
+
+
+def raw_template_files_get(context, files_id):
+    result = model_query(context, models.RawTemplateFiles).get(files_id)
+    if not result:
+        raise exception.NotFound(
+            _("raw_template_files with files_id %d not found") %
+            files_id)
+    return result
 
 
 def resource_get(context, resource_id):
@@ -1106,6 +1132,8 @@ def purge_deleted(age, granularity='days'):
     resource_data = sqlalchemy.Table('resource_data', meta, autoload=True)
     event = sqlalchemy.Table('event', meta, autoload=True)
     raw_template = sqlalchemy.Table('raw_template', meta, autoload=True)
+    raw_template_files = sqlalchemy.Table('raw_template_files', meta,
+                                          autoload=True)
     user_creds = sqlalchemy.Table('user_creds', meta, autoload=True)
     service = sqlalchemy.Table('service', meta, autoload=True)
     syncpoint = sqlalchemy.Table('sync_point', meta, autoload=True)
@@ -1159,9 +1187,26 @@ def purge_deleted(age, granularity='days'):
                 stack.c.prev_raw_template_id.in_(raw_template_ids))
             raw_tmpl = [i[0] for i in engine.execute(raw_tmpl_sel)]
             raw_template_ids = raw_template_ids - set(raw_tmpl)
+            raw_tmpl_file_sel = sqlalchemy.select(
+                [raw_template.c.files_id]).where(
+                    raw_template.c.id.in_(raw_template_ids))
+            raw_tmpl_file_ids = [i[0] for i in engine.execute(
+                raw_tmpl_file_sel)]
             raw_templ_del = raw_template.delete().where(
                 raw_template.c.id.in_(raw_template_ids))
             engine.execute(raw_templ_del)
+            # purge any raw_template_files that are no longer referenced
+            if raw_tmpl_file_ids:
+                raw_tmpl_file_sel = sqlalchemy.select(
+                    [raw_template.c.files_id]).where(
+                        raw_template.c.files_id.in_(raw_tmpl_file_ids))
+                raw_tmpl_files = [i[0] for i in engine.execute(
+                    raw_tmpl_file_sel)]
+                raw_tmpl_file_ids = set(raw_tmpl_file_ids) \
+                    - set(raw_tmpl_files)
+                raw_tmpl_file_del = raw_template_files.delete().where(
+                    raw_template_files.c.id.in_(raw_tmpl_file_ids))
+                engine.execute(raw_tmpl_file_del)
         # purge any user creds that are no longer referenced
         user_creds_ids = [i[3] for i in stacks if i[3] is not None]
         if user_creds_ids:

heat/db/sqlalchemy/migrate_repo/versions/072_raw_template_files.py

@@ -0,0 +1,40 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import sqlalchemy
+
+from heat.db.sqlalchemy import types
+
+
+def upgrade(migrate_engine):
+    meta = sqlalchemy.MetaData(bind=migrate_engine)
+    raw_template_files = sqlalchemy.Table(
+        'raw_template_files', meta,
+        sqlalchemy.Column('id', sqlalchemy.Integer,
+                          primary_key=True,
+                          nullable=False),
+        sqlalchemy.Column('files', types.Json),
+        sqlalchemy.Column('created_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'
+
+    )
+    raw_template_files.create()
+
+    raw_template = sqlalchemy.Table('raw_template', meta, autoload=True)
+    files_id = sqlalchemy.Column(
+        'files_id', sqlalchemy.Integer(),
+        sqlalchemy.ForeignKey('raw_template_files.id',
+                              name='raw_tmpl_files_fkey_ref'))
+    files_id.create(raw_template)

heat/db/sqlalchemy/models.py

@@ -96,10 +96,22 @@ class RawTemplate(BASE, HeatBase):
     __tablename__ = 'raw_template'
     id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
     template = sqlalchemy.Column(types.Json)
+    # legacy column
     files = sqlalchemy.Column(types.Json)
+    # modern column, reference to raw_template_files
+    files_id = sqlalchemy.Column(
+        sqlalchemy.Integer(),
+        sqlalchemy.ForeignKey('raw_template_files.id'))
     environment = sqlalchemy.Column('environment', types.Json)
 
 
+class RawTemplateFiles(BASE, HeatBase):
+    """Where template files json dicts are stored."""
+    __tablename__ = 'raw_template_files'
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    files = sqlalchemy.Column(types.Json)
+
+
 class StackTag(BASE, HeatBase):
     """Key/value store of arbitrary stack tags."""
 

heat/engine/service.py

@@ -856,7 +856,7 @@ class EngineService(service.Service):
             new_env = environment.Environment(existing_env)
             new_env.load(params)
 
-            new_files = current_stack.t.files.copy()
+            new_files = current_stack.t.files
             new_files.update(files or {})
 
             assert template_id is None, \

heat/engine/template.py

@@ -23,6 +23,7 @@ from stevedore import extension
 from heat.common import exception
 from heat.common.i18n import _
 from heat.engine import environment
+from heat.engine import template_files
 from heat.objects import raw_template as template_object
 
 __all__ = ['Template']
@@ -127,13 +128,17 @@ class Template(collections.Mapping):
         if t is None:
             t = template_object.RawTemplate.get_by_id(context, template_id)
         env = environment.Environment(t.environment)
-        return cls(t.template, template_id=template_id, files=t.files, env=env)
+        # support loading the legacy t.files, but modern templates will
+        # have a t.files_id
+        t_files = t.files or t.files_id
+        return cls(t.template, template_id=template_id, env=env,
+                   files=t_files)
 
     def store(self, context=None):
         """Store the Template in the database and return its ID."""
         rt = {
             'template': self.t,
-            'files': self.files,
+            'files_id': self.files.store(),
             'environment': self.env.user_env_as_dict()
         }
         if self.id is None:
@@ -143,6 +148,14 @@ class Template(collections.Mapping):
             template_object.RawTemplate.update_by_id(context, self.id, rt)
         return self.id
 
+    @property
+    def files(self):
+        return self._template_files
+
+    @files.setter
+    def files(self, files):
+        self._template_files = template_files.TemplateFiles(files)
+
     def __iter__(self):
         """Return an iterator over the section names."""
         return (s for s in self.SECTIONS

heat/engine/template_files.py

@@ -0,0 +1,134 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import collections
+import six
+import weakref
+
+from heat.common.i18n import _
+from heat.db import api as db_api
+from heat.objects import raw_template_files
+
+_d = weakref.WeakValueDictionary()
+
+
+class ReadOnlyDict(dict):
+    def __setitem__(self, key):
+        raise ValueError("Attempted to write to internal TemplateFiles cache")
+
+
+class TemplateFiles(collections.Mapping):
+
+    def __init__(self, files):
+        self.files = None
+        self.files_id = None
+        if files is None:
+            return
+        if isinstance(files, TemplateFiles):
+            self.files_id = files.files_id
+            self.files = files.files
+            return
+        if isinstance(files, six.integer_types):
+            self.files_id = files
+            if self.files_id in _d:
+                self.files = _d[self.files_id]
+            return
+        if not isinstance(files, dict):
+            raise ValueError(_('Expected dict, got %(cname)s for files, '
+                               '(value is %(val)s)') %
+                             {'cname': files.__class__,
+                              'val': str(files)})
+        # the dict has not been persisted as a raw_template_files db obj
+        # yet, so no self.files_id
+        self.files = ReadOnlyDict(files)
+
+    def __getitem__(self, key):
+        self._refresh_if_needed()
+        if self.files is None:
+            raise KeyError
+        return self.files[key]
+
+    def __setitem__(self, key, value):
+        self.update({key: value})
+
+    def __len__(self):
+        self._refresh_if_needed()
+        if not self.files:
+            return 0
+        return len(self.files)
+
+    def __contains__(self, key):
+        self._refresh_if_needed()
+        if not self.files:
+            return False
+        return key in self.files
+
+    def __iter__(self):
+        self._refresh_if_needed()
+        if self.files_id is None:
+            return iter(ReadOnlyDict({}))
+        return iter(self.files)
+
+    def _refresh_if_needed(self):
+        # retrieve files from db if needed
+        if self.files_id is None:
+            return
+        if self.files_id in _d:
+            self.files = _d[self.files_id]
+            return
+        self._refresh()
+
+    def _refresh(self):
+        rtf_obj = db_api.raw_template_files_get(None, self.files_id)
+        _files_dict = ReadOnlyDict(rtf_obj.files)
+        self.files = _files_dict
+        _d[self.files_id] = _files_dict
+
+    def store(self, ctxt=None):
+        if not self.files or self.files_id is not None:
+            # Do not to persist an empty raw_template_files obj. If we
+            # already have a not null self.files_id, the (immutable)
+            # raw_templated_object has already been persisted so just
+            # return the id.
+            return self.files_id
+        rtf_obj = raw_template_files.RawTemplateFiles.create(
+            ctxt, {'files': self.files})
+        self.files_id = rtf_obj.id
+        _d[self.files_id] = self.files
+        return self.files_id
+
+    def update(self, files):
+        # Sets up the next call to store() to create a new
+        # raw_template_files db obj. It seems like we *could* just
+        # update the existing raw_template_files obj, but the problem
+        # with that is other heat-engine processes' _d dictionaries
+        # would have stale data for a given raw_template_files.id with
+        # no way of knowing whether that data should be refreshed or
+        # not. So, just avoid the potential for weird race conditions
+        # and create another db obj in the next store().
+        if len(files) == 0:
+            return
+        if not isinstance(files, dict):
+            raise ValueError(_('Expected dict, got %(cname)s for files, '
+                               '(value is %(val)s)') %
+                             {'cname': files.__class__,
+                              'val': str(files)})
+
+        self._refresh_if_needed()
+        if self.files:
+            new_files = self.files.copy()
+            new_files.update(files)
+        else:
+            new_files = files
+        self.files_id = None  # not persisted yet
+        self.files = ReadOnlyDict(new_files)

heat/objects/base.py

@@ -17,6 +17,10 @@
 from oslo_versionedobjects import base as ovoo_base
 
 
+class HeatObjectRegistry(ovoo_base.VersionedObjectRegistry):
+    pass
+
+
 class HeatObject(ovoo_base.VersionedObject):
     OBJ_PROJECT_NAMESPACE = 'heat'
     VERSION = '1.0'

heat/objects/raw_template.py

@@ -28,14 +28,21 @@ from heat.objects import base as heat_base
 from heat.objects import fields as heat_fields
 
 
+@heat_base.HeatObjectRegistry.register
 class RawTemplate(
     heat_base.HeatObject,
     base.VersionedObjectDictCompat,
     base.ComparableVersionedObject,
 ):
+    # Version 1.0: Initial version
+    # Version 1.1: Added files_id
+    VERSION = '1.1'
+
     fields = {
-        'id': fields.StringField(),
+        'id': fields.IntegerField(),
+        # TODO(cwolfe): remove deprecated files in future release
         'files': heat_fields.JsonField(nullable=True),
+        'files_id': fields.IntegerField(nullable=True),
         'template': heat_fields.JsonField(),
         'environment': heat_fields.JsonField(),
     }
@@ -86,6 +93,10 @@ class RawTemplate(
 
     @classmethod
     def update_by_id(cls, context, template_id, values):
+        # Only save template files in the new raw_template_files
+        # table, not in the old location of raw_template.files
+        if 'files_id' in values and values['files_id']:
+            values['files'] = None
         return cls._from_db_object(
             context, cls(),
             db_api.raw_template_update(context, template_id, values))

heat/objects/raw_template_files.py

@@ -0,0 +1,51 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+"""RawTemplateFiles object."""
+
+from oslo_versionedobjects import base
+from oslo_versionedobjects import fields
+
+from heat.db import api as db_api
+from heat.objects import base as heat_base
+from heat.objects import fields as heat_fields
+
+
+@heat_base.HeatObjectRegistry.register
+class RawTemplateFiles(
+    heat_base.HeatObject,
+    base.VersionedObjectDictCompat,
+    base.ComparableVersionedObject,
+):
+    # Version 1.0: Initial Version
+    VERSION = '1.0'
+
+    fields = {
+        'id': fields.IntegerField(),
+        'files': heat_fields.JsonField(read_only=True),
+    }
+
+    @staticmethod
+    def _from_db_object(context, tmpl_files, db_tmpl_files):
+        for field in tmpl_files.fields:
+            tmpl_files[field] = db_tmpl_files[field]
+
+        tmpl_files._context = context
+        tmpl_files.obj_reset_changes()
+        return tmpl_files
+
+    @classmethod
+    def create(cls, context, values):
+        return cls._from_db_object(context, cls(),
+                                   db_api.raw_template_files_create(context,
+                                                                    values))

heat/tests/db/test_sqlalchemy_api.py

@@ -39,6 +39,7 @@ from heat.engine.resources.aws.ec2 import instance as instances
 from heat.engine import scheduler
 from heat.engine import stack as parser
 from heat.engine import template as tmpl
+from heat.engine import template_files
 from heat.tests import common
 from heat.tests.openstack.nova import fakes as fakes_nova
 from heat.tests import utils
@@ -1376,8 +1377,12 @@ def create_raw_template(context, **kwargs):
     t = template_format.parse(wp_template)
     template = {
         'template': t,
-        'files': {'foo': 'bar'}
     }
+    if 'files' not in kwargs and 'files_id' not in kwargs:
+        # modern raw_templates have associated raw_template_files db obj
+        tf = template_files.TemplateFiles({'foo': 'bar'})
+        tf.store()
+        kwargs['files_id'] = tf.files_id
     template.update(kwargs)
     return db_api.raw_template_create(context, template)
 
@@ -1505,7 +1510,6 @@ class DBAPIRawTemplateTest(common.HeatTestCase):
         tp = create_raw_template(self.ctx, template=t)
         self.assertIsNotNone(tp.id)
         self.assertEqual(t, tp.template)
-        self.assertEqual({'foo': 'bar'}, tp.files)
 
     def test_raw_template_get(self):
         t = template_format.parse(wp_template)
@@ -1962,26 +1966,31 @@ class DBAPIStackTest(common.HeatTestCase):
         now = timeutils.utcnow()
         delta = datetime.timedelta(seconds=3600 * 7)
         deleted = [now - delta * i for i in range(1, 6)]
-        templates = [create_raw_template(self.ctx) for i in range(5)]
+        tmpl_files = [template_files.TemplateFiles(
+            {'foo': 'file contents %d' % i}) for i in range(5)]
+        [tmpl_file.store(self.ctx) for tmpl_file in tmpl_files]
+        templates = [create_raw_template(self.ctx,
+                                         files_id=tmpl_files[i].files_id
+                                         ) for i in range(5)]
         creds = [create_user_creds(self.ctx) for i in range(5)]
         stacks = [create_stack(self.ctx, templates[i], creds[i],
                                deleted_at=deleted[i]) for i in range(5)]
 
         db_api.purge_deleted(age=1, granularity='days')
         self._deleted_stack_existance(utils.dummy_context(), stacks,
-                                      (0, 1, 2), (3, 4))
+                                      tmpl_files, (0, 1, 2), (3, 4))
 
         db_api.purge_deleted(age=22, granularity='hours')
         self._deleted_stack_existance(utils.dummy_context(), stacks,
-                                      (0, 1, 2), (3, 4))
+                                      tmpl_files, (0, 1, 2), (3, 4))
 
         db_api.purge_deleted(age=1100, granularity='minutes')
         self._deleted_stack_existance(utils.dummy_context(), stacks,
-                                      (0, 1), (2, 3, 4))
+                                      tmpl_files, (0, 1), (2, 3, 4))
 
         db_api.purge_deleted(age=3600, granularity='seconds')
         self._deleted_stack_existance(utils.dummy_context(), stacks,
-                                      (), (0, 1, 2, 3, 4))
+                                      tmpl_files, (), (0, 1, 2, 3, 4))
 
     def test_purge_deleted_prev_raw_template(self):
         now = timeutils.utcnow()
@@ -1997,10 +2006,44 @@ class DBAPIStackTest(common.HeatTestCase):
                                               show_deleted=True))
         self.assertIsNotNone(db_api.raw_template_get(ctx, templates[1].id))
 
-    def _deleted_stack_existance(self, ctx, stacks, existing, deleted):
+    def test_dont_purge_shared_raw_template_files(self):
+        now = timeutils.utcnow()
+        delta = datetime.timedelta(seconds=3600 * 7)
+        deleted = [now - delta * i for i in range(1, 6)]
+        # the last two template_files are identical to first two
+        # (so should not be purged)
+        tmpl_files = [template_files.TemplateFiles(
+            {'foo': 'more file contents'}) for i in range(3)]
+        [tmpl_file.store(self.ctx) for tmpl_file in tmpl_files]
+        templates = [create_raw_template(self.ctx,
+                                         files_id=tmpl_files[i % 3].files_id
+                                         ) for i in range(5)]
+        creds = [create_user_creds(self.ctx) for i in range(5)]
+        [create_stack(self.ctx, templates[i], creds[i],
+                      deleted_at=deleted[i]) for i in range(5)]
+        db_api.purge_deleted(age=15, granularity='hours')
+
+        # The third raw_template_files object should be purged (along
+        # with the last three stacks/templates). However, the other
+        # two are shared with existing templates, so should not be
+        # purged.
+        self.assertIsNotNone(db_api.raw_template_files_get(
+            self.ctx, tmpl_files[0].files_id))
+        self.assertIsNotNone(db_api.raw_template_files_get(
+            self.ctx, tmpl_files[1].files_id))
+        self.assertRaises(exception.NotFound,
+                          db_api.raw_template_files_get,
+                          self.ctx, tmpl_files[2].files_id)
+
+    def _deleted_stack_existance(self, ctx, stacks,
+                                 tmpl_files, existing, deleted):
+        tmpl_idx = 0
         for s in existing:
             self.assertIsNotNone(db_api.stack_get(ctx, stacks[s].id,
                                                   show_deleted=True))
+            self.assertIsNotNone(db_api.raw_template_files_get(
+                ctx, tmpl_files[tmpl_idx].files_id))
+            tmpl_idx = tmpl_idx + 1
         for s in deleted:
             self.assertIsNone(db_api.stack_get(ctx, stacks[s].id,
                                                show_deleted=True))
@@ -2010,6 +2053,9 @@ class DBAPIStackTest(common.HeatTestCase):
             self.assertRaises(exception.NotFound,
                               db_api.resource_get_all_by_stack,
                               ctx, stacks[s].id)
+            self.assertRaises(exception.NotFound,
+                              db_api.raw_template_files_get,
+                              ctx, tmpl_files[tmpl_idx].files_id)
             for r in stacks[s].resources:
                 self.assertRaises(exception.NotFound,
                                   db_api.resource_data_get_all(r.context,
@@ -2018,6 +2064,7 @@ class DBAPIStackTest(common.HeatTestCase):
                              db_api.event_get_all_by_stack(ctx,
                                                            stacks[s].id))
             self.assertIsNone(db_api.user_creds_get(stacks[s].user_creds_id))
+            tmpl_idx = tmpl_idx + 1
 
     def test_stack_get_root_id(self):
         root = create_stack(self.ctx, self.template, self.user_creds,

heat/tests/engine/service/test_stack_update.py

@@ -318,7 +318,7 @@ class ServiceStackUpdateTest(common.HeatTestCase):
             self.assertEqual(expected_env,
                              tmpl.env.user_env_as_dict())
             self.assertEqual(expected_files,
-                             tmpl.files)
+                             tmpl.files.files)
             self.assertEqual(stk.identifier(), result)
 
     def test_stack_update_existing_parameter_defaults(self):

heat/tests/test_stack_resource.py

@@ -221,7 +221,7 @@ class StackResourceTest(StackResourceBaseTest):
         """
         self.parent_stack.t.files["foo"] = "bar"
         parsed_t = self.parent_resource._parse_child_template(self.templ, None)
-        self.assertEqual({"foo": "bar"}, parsed_t.files)
+        self.assertEqual({"foo": "bar"}, parsed_t.files.files)
 
     @mock.patch('heat.engine.environment.get_child_environment')
     @mock.patch.object(stack_resource.parser, 'Stack')
@@ -313,7 +313,7 @@ class StackResourceTest(StackResourceBaseTest):
         self.parent_resource.child_params = mock.Mock(return_value={})
         self.parent_resource.preview()
         self.stack = self.parent_resource.nested()
-        self.assertEqual({"foo": "bar"}, self.stack.t.files)
+        self.assertEqual({"foo": "bar"}, self.stack.t.files.files)
 
     def test_preview_validates_nested_resources(self):
         parent_t = self.parent_stack.t

heat/tests/test_template_files.py

@@ -0,0 +1,42 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from heat.engine import template_files
+from heat.tests import common
+
+template_files_1 = {'template file 1': 'Contents of template 1',
+                    'template file 2': 'More template contents'}
+
+
+class TestTemplateFiles(common.HeatTestCase):
+
+    def test_cache_miss(self):
+        tf1 = template_files.TemplateFiles(template_files_1)
+        tf1.store()
+        # As this is the only reference to the value in _d, deleting
+        # t1.files will cause the value to get removed from _d (due to
+        # it being a WeakValueDictionary.
+        del tf1.files
+        self.assertNotIn(tf1.files_id, template_files._d)
+        # this will cause the cache refresh
+        self.assertEqual(template_files_1['template file 1'],
+                         tf1['template file 1'])
+        self.assertEqual(template_files_1, template_files._d[tf1.files_id])
+
+    def test_d_weakref_behaviour(self):
+        tf1 = template_files.TemplateFiles(template_files_1)
+        tf1.store()
+        tf2 = template_files.TemplateFiles(tf1)
+        del tf1.files
+        self.assertIn(tf2.files_id, template_files._d)
+        del tf2.files
+        self.assertNotIn(tf2.files_id, template_files._d)