Initial disk offline cache

compressor/cache.py

@@ -3,12 +3,15 @@ import socket
 import time
 
 from django.core.cache import get_cache
+from django.core.files.base import ContentFile
+from django.utils import simplejson
 from django.utils.encoding import smart_str
 from django.utils.functional import SimpleLazyObject
 from django.utils.hashcompat import md5_constructor
 from django.utils.importlib import import_module
 
 from compressor.conf import settings
+from compressor.storage import default_storage
 from compressor.utils import get_mod_func
 
 _cachekey_func = None
@@ -44,12 +47,28 @@ def get_cachekey(*args, **kwargs):
 def get_mtime_cachekey(filename):
     return get_cachekey("mtime.%s" % get_hexdigest(filename))
 
+def get_offline_hexdigest(source):
+    return get_hexdigest([smart_str(getattr(s, 's', s)) for s in source])
 
 def get_offline_cachekey(source):
-    to_hexdigest = [smart_str(getattr(s, 's', s)) for s in source]
+    return get_cachekey("offline.%s" % get_offline_hexdigest(source))
-    return get_cachekey("offline.%s" % get_hexdigest(to_hexdigest))
 
 
+def get_offline_manifest_filename():
+    output_dir = settings.COMPRESS_OUTPUT_DIR.strip('/')
+    return os.path.join(output_dir, 'manifest.json')
+
+def get_offline_manifest():
+    filename = get_offline_manifest_filename()
+    if default_storage.exists(filename):
+        return simplejson.load(default_storage.open(filename))
+    else:
+        return {}
+
+def write_offline_manifest(manifest):
+    filename = get_offline_manifest_filename()
+    default_storage.save(filename, ContentFile(simplejson.dumps(manifest)))
+
 def get_templatetag_cachekey(compressor, mode, kind):
     return get_cachekey(
         "templatetag.%s.%s.%s" % (compressor.cachekey, mode, kind))

compressor/management/commands/compress.py

@@ -21,7 +21,7 @@ try:
 except ImportError:
     CachedLoader = None
 
-from compressor.cache import cache, get_offline_cachekey
+from compressor.cache import cache, get_offline_hexdigest, write_offline_manifest
 from compressor.conf import settings
 from compressor.exceptions import OfflineGenerationError
 from compressor.templatetags.compress import CompressorNode
@@ -176,6 +176,7 @@ class Command(NoArgsCommand):
         log.write("Compressing... ")
         count = 0
         results = []
+        offline_manifest = {}
         for template, nodes in compressor_nodes.iteritems():
             context = Context(settings.COMPRESS_OFFLINE_CONTEXT)
             extra_context = {}
@@ -195,16 +196,19 @@ class Command(NoArgsCommand):
                     context['block'] = context.render_context[BLOCK_CONTEXT_KEY].pop(node._block_name)
                     if context['block']:
                         context['block'].context = context
-                key = get_offline_cachekey(node.nodelist)
+                key = get_offline_hexdigest(node.nodelist)
                 try:
                     result = node.render(context, forced=True)
                 except Exception, e:
                     raise CommandError("An error occured during rendering: "
                                        "%s" % e)
-                cache.set(key, result, settings.COMPRESS_OFFLINE_TIMEOUT)
+                offline_manifest[key] = result
                 context.pop()
                 results.append(result)
                 count += 1
+
+        write_offline_manifest(offline_manifest)
+
         log.write("done\nCompressed %d block(s) from %d template(s).\n" %
                   (count, len(compressor_nodes)))
         return count, results
@@ -252,3 +256,4 @@ class Command(NoArgsCommand):
                     "Offline compressiong is disabled. Set "
                     "COMPRESS_OFFLINE or use the --force to override.")
         self.compress(sys.stdout, **options)
+

compressor/templatetags/compress.py

@@ -2,8 +2,10 @@ from django import template
 from django.core.exceptions import ImproperlyConfigured
 
 from compressor.cache import (cache, cache_get, cache_set,
-                              get_offline_cachekey, get_templatetag_cachekey)
+                              get_offline_hexdigest, get_offline_manifest,
+                              get_templatetag_cachekey)
 from compressor.conf import settings
+from compressor.exceptions import OfflineGenerationError
 from compressor.utils import get_class
 
 register = template.Library()
@@ -39,14 +41,20 @@ class CompressorNode(template.Node):
             if request is not None:
                 return settings.COMPRESS_DEBUG_TOGGLE in request.GET
 
-    def render_offline(self, forced):
+    def render_offline(self, compressor, forced):
         """
         If enabled and in offline mode, and not forced or in debug mode
         check the offline cache and return the result if given
         """
         if (settings.COMPRESS_ENABLED and
                 settings.COMPRESS_OFFLINE) and not forced:
-            return cache.get(get_offline_cachekey(self.nodelist))
+            key = get_offline_hexdigest(self.nodelist)
+            offline_manifest = get_offline_manifest()
+            if key in offline_manifest:
+                return offline_manifest[key]
+            else:
+                raise OfflineGenerationError('You have offline compression enabled but key "%s" is missing from offline manifest. You may need to run "python manage.py compress".' % key)
+
 
     def render_cached(self, compressor, forced):
         """
@@ -61,24 +69,26 @@ class CompressorNode(template.Node):
         return None, None
 
     def render(self, context, forced=False):
-        # 1. Check if in debug mode
+        # Check if in debug mode
         if self.debug_mode(context):
             return self.nodelist.render(context)
 
-        # 2. Try offline cache.
+        # Prepare the compressor
-        cached_offline = self.render_offline(forced)
-        if cached_offline:
-            return cached_offline
-
-        # 3. Prepare the actual compressor and check cache
         context.update({'name': self.name}) 
         compressor = self.compressor_cls(content=self.nodelist.render(context),
                                          context=context)
+
+        # See if it has been rendered offline
+        cached_offline = self.render_offline(compressor, forced)
+        if cached_offline:
+            return cached_offline
+        
+        # Check cache
         cache_key, cache_content = self.render_cached(compressor, forced)
         if cache_content is not None:
             return cache_content
 
-        # 4. call compressor output method and handle exceptions
+        # call compressor output method and handle exceptions
         rendered_output = compressor.output(self.mode, forced=forced)
         if cache_key:
             cache_set(cache_key, rendered_output)

tests/tests/offline.py

@@ -5,7 +5,9 @@ from django.template import Template, Context
 from django.test import TestCase
 
 from compressor.conf import settings
+from compressor.exceptions import OfflineGenerationError
 from compressor.management.commands.compress import Command as CompressCommand
+from compressor.storage import default_storage
 
 from .base import test_dir, css_tag
 
@@ -25,6 +27,12 @@ class OfflineGenerationTestCase(TestCase):
         settings.COMPRESS_ENABLED = self._old_compress
         settings.COMPRESS_OFFLINE = self._old_compress_offline
         self.template_file.close()
+        if default_storage.exists('CACHE/manifest.json'):
+            default_storage.delete('CACHE/manifest.json')
+
+    def test_rendering_without_compressing_raises_exception(self):
+        with self.assertRaises(OfflineGenerationError):
+            self.template.render(Context({}))
 
     def test_requires_model_validation(self):
         self.assertFalse(CompressCommand.requires_model_validation)