From ecdd5ecbc746ebe6d0f84c781c282efa1390bb7c Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 21 Sep 2011 17:00:22 +0100 Subject: [PATCH] Initial disk offline cache --- compressor/cache.py | 23 ++++++++++++++-- compressor/management/commands/compress.py | 11 ++++++-- compressor/templatetags/compress.py | 32 ++++++++++++++-------- tests/tests/offline.py | 8 ++++++ 4 files changed, 58 insertions(+), 16 deletions(-) diff --git a/compressor/cache.py b/compressor/cache.py index d629497..68baabd 100644 --- a/compressor/cache.py +++ b/compressor/cache.py @@ -3,12 +3,15 @@ import socket import time from django.core.cache import get_cache +from django.core.files.base import ContentFile +from django.utils import simplejson from django.utils.encoding import smart_str from django.utils.functional import SimpleLazyObject from django.utils.hashcompat import md5_constructor from django.utils.importlib import import_module from compressor.conf import settings +from compressor.storage import default_storage from compressor.utils import get_mod_func _cachekey_func = None @@ -44,12 +47,28 @@ def get_cachekey(*args, **kwargs): def get_mtime_cachekey(filename): return get_cachekey("mtime.%s" % get_hexdigest(filename)) +def get_offline_hexdigest(source): + return get_hexdigest([smart_str(getattr(s, 's', s)) for s in source]) def get_offline_cachekey(source): - to_hexdigest = [smart_str(getattr(s, 's', s)) for s in source] - return get_cachekey("offline.%s" % get_hexdigest(to_hexdigest)) + return get_cachekey("offline.%s" % get_offline_hexdigest(source)) +def get_offline_manifest_filename(): + output_dir = settings.COMPRESS_OUTPUT_DIR.strip('/') + return os.path.join(output_dir, 'manifest.json') + +def get_offline_manifest(): + filename = get_offline_manifest_filename() + if default_storage.exists(filename): + return simplejson.load(default_storage.open(filename)) + else: + return {} + +def write_offline_manifest(manifest): + filename = get_offline_manifest_filename() + default_storage.save(filename, ContentFile(simplejson.dumps(manifest))) + def get_templatetag_cachekey(compressor, mode, kind): return get_cachekey( "templatetag.%s.%s.%s" % (compressor.cachekey, mode, kind)) diff --git a/compressor/management/commands/compress.py b/compressor/management/commands/compress.py index 5af1a5a..c41fd48 100644 --- a/compressor/management/commands/compress.py +++ b/compressor/management/commands/compress.py @@ -21,7 +21,7 @@ try: except ImportError: CachedLoader = None -from compressor.cache import cache, get_offline_cachekey +from compressor.cache import cache, get_offline_hexdigest, write_offline_manifest from compressor.conf import settings from compressor.exceptions import OfflineGenerationError from compressor.templatetags.compress import CompressorNode @@ -176,6 +176,7 @@ class Command(NoArgsCommand): log.write("Compressing... ") count = 0 results = [] + offline_manifest = {} for template, nodes in compressor_nodes.iteritems(): context = Context(settings.COMPRESS_OFFLINE_CONTEXT) extra_context = {} @@ -195,16 +196,19 @@ class Command(NoArgsCommand): context['block'] = context.render_context[BLOCK_CONTEXT_KEY].pop(node._block_name) if context['block']: context['block'].context = context - key = get_offline_cachekey(node.nodelist) + key = get_offline_hexdigest(node.nodelist) try: result = node.render(context, forced=True) except Exception, e: raise CommandError("An error occured during rendering: " "%s" % e) - cache.set(key, result, settings.COMPRESS_OFFLINE_TIMEOUT) + offline_manifest[key] = result context.pop() results.append(result) count += 1 + + write_offline_manifest(offline_manifest) + log.write("done\nCompressed %d block(s) from %d template(s).\n" % (count, len(compressor_nodes))) return count, results @@ -252,3 +256,4 @@ class Command(NoArgsCommand): "Offline compressiong is disabled. Set " "COMPRESS_OFFLINE or use the --force to override.") self.compress(sys.stdout, **options) + diff --git a/compressor/templatetags/compress.py b/compressor/templatetags/compress.py index b28cf6c..ef38143 100644 --- a/compressor/templatetags/compress.py +++ b/compressor/templatetags/compress.py @@ -2,8 +2,10 @@ from django import template from django.core.exceptions import ImproperlyConfigured from compressor.cache import (cache, cache_get, cache_set, - get_offline_cachekey, get_templatetag_cachekey) + get_offline_hexdigest, get_offline_manifest, + get_templatetag_cachekey) from compressor.conf import settings +from compressor.exceptions import OfflineGenerationError from compressor.utils import get_class register = template.Library() @@ -39,14 +41,20 @@ class CompressorNode(template.Node): if request is not None: return settings.COMPRESS_DEBUG_TOGGLE in request.GET - def render_offline(self, forced): + def render_offline(self, compressor, forced): """ If enabled and in offline mode, and not forced or in debug mode check the offline cache and return the result if given """ if (settings.COMPRESS_ENABLED and settings.COMPRESS_OFFLINE) and not forced: - return cache.get(get_offline_cachekey(self.nodelist)) + key = get_offline_hexdigest(self.nodelist) + offline_manifest = get_offline_manifest() + if key in offline_manifest: + return offline_manifest[key] + else: + raise OfflineGenerationError('You have offline compression enabled but key "%s" is missing from offline manifest. You may need to run "python manage.py compress".' % key) + def render_cached(self, compressor, forced): """ @@ -61,24 +69,26 @@ class CompressorNode(template.Node): return None, None def render(self, context, forced=False): - # 1. Check if in debug mode + # Check if in debug mode if self.debug_mode(context): return self.nodelist.render(context) - # 2. Try offline cache. - cached_offline = self.render_offline(forced) - if cached_offline: - return cached_offline - - # 3. Prepare the actual compressor and check cache + # Prepare the compressor context.update({'name': self.name}) compressor = self.compressor_cls(content=self.nodelist.render(context), context=context) + + # See if it has been rendered offline + cached_offline = self.render_offline(compressor, forced) + if cached_offline: + return cached_offline + + # Check cache cache_key, cache_content = self.render_cached(compressor, forced) if cache_content is not None: return cache_content - # 4. call compressor output method and handle exceptions + # call compressor output method and handle exceptions rendered_output = compressor.output(self.mode, forced=forced) if cache_key: cache_set(cache_key, rendered_output) diff --git a/tests/tests/offline.py b/tests/tests/offline.py index a42a9f1..e384172 100644 --- a/tests/tests/offline.py +++ b/tests/tests/offline.py @@ -5,7 +5,9 @@ from django.template import Template, Context from django.test import TestCase from compressor.conf import settings +from compressor.exceptions import OfflineGenerationError from compressor.management.commands.compress import Command as CompressCommand +from compressor.storage import default_storage from .base import test_dir, css_tag @@ -25,6 +27,12 @@ class OfflineGenerationTestCase(TestCase): settings.COMPRESS_ENABLED = self._old_compress settings.COMPRESS_OFFLINE = self._old_compress_offline self.template_file.close() + if default_storage.exists('CACHE/manifest.json'): + default_storage.delete('CACHE/manifest.json') + + def test_rendering_without_compressing_raises_exception(self): + with self.assertRaises(OfflineGenerationError): + self.template.render(Context({})) def test_requires_model_validation(self): self.assertFalse(CompressCommand.requires_model_validation)