Only cache precompilers specified in COMPRESS_CACHEABLE_PRECOMPILERS setting, and refactor caching logic into new CachedCompilerFilter class

compressor/base.py

@@ -20,7 +20,7 @@ from compressor.cache import get_hexdigest, get_mtime
 from compressor.conf import settings
 from compressor.exceptions import (CompressorError, UncompressableFileError,
         FilterDoesNotExist)
-from compressor.filters import CompilerFilter
+from compressor.filters import CachedCompilerFilter
 from compressor.storage import compressor_file_storage
 from compressor.signals import post_compress
 from compressor.utils import get_class, get_mod_func, staticfiles
@@ -254,9 +254,9 @@ class Compressor(object):
                 try:
                     mod = import_module(mod_name)
                 except (ImportError, TypeError):
-                    filter = CompilerFilter(
+                    filter = CachedCompilerFilter(
-                        content, filter_type=self.type, filename=filename,
+                        content=content, filter_type=self.type, filename=filename,
-                        charset=charset, command=filter_or_command)
+                        charset=charset, command=filter_or_command, mimetype=mimetype)
                     return True, filter.input(**kwargs)
                 try:
                     precompiler_class = getattr(mod, cls_name)

compressor/cache.py

@@ -135,6 +135,10 @@ def get_hashed_content(filename, length=12):
         return get_hexdigest(file.read(), length)
 
 
+def get_precompiler_cachekey(command, contents):
+    return hashlib.sha1('precompiler.%s.%s' % (command, contents)).hexdigest()
+
+
 def cache_get(key):
     packed_val = cache.get(key)
     if packed_val is None:

compressor/conf.py

@@ -35,6 +35,7 @@ class CompressorConf(AppConf):
         # ('text/stylus', 'stylus < {infile} > {outfile}'),
         # ('text/x-scss', 'sass --scss {infile} {outfile}'),
     )
+    CACHEABLE_PRECOMPILERS = ()
     CLOSURE_COMPILER_BINARY = 'java -jar compiler.jar'
     CLOSURE_COMPILER_ARGUMENTS = ''
     CSSTIDY_BINARY = 'csstidy'

compressor/filters/__init__.py

@@ -1,3 +1,3 @@
 # flake8: noqa
 from compressor.filters.base import (FilterBase, CallbackOutputFilter,
-                                     CompilerFilter, FilterError)
+                                     CompilerFilter, CachedCompilerFilter, FilterError)

compressor/filters/base.py

@@ -1,11 +1,9 @@
 from __future__ import absolute_import, unicode_literals
 import io
-import hashlib
 import logging
 import subprocess
 
 from platform import system
-from django.core.cache import get_cache
 
 if system() != "Windows":
     try:
@@ -29,6 +27,8 @@ except ImportError:
 from django.utils.encoding import smart_text
 from django.utils import six
 
+from compressor.cache import cache, get_precompiler_cachekey
+
 from compressor.conf import settings
 from compressor.exceptions import FilterError
 from compressor.utils import get_mod_func
@@ -36,8 +36,6 @@ from compressor.utils import get_mod_func
 
 logger = logging.getLogger("compressor.filters")
 
-cache = get_cache(settings.COMPRESS_CACHE_BACKEND)
-
 
 class FilterBase(object):
     """
@@ -144,10 +142,6 @@ class CompilerFilter(FilterBase):
         self.infile = self.outfile = None
 
     def input(self, **kwargs):
-        content_hash = hashlib.sha1(self.command + self.content.encode('utf8')).hexdigest()
-        data = cache.get(content_hash)
-        if data:
-            return data
 
         encoding = self.default_encoding
         options = dict(self.options)
@@ -217,5 +211,26 @@ class CompilerFilter(FilterBase):
                 self.infile.close()
             if self.outfile is not None:
                 self.outfile.close()
-        cache.set(content_hash, filtered, settings.COMPRESS_REBUILD_TIMEOUT)
         return smart_text(filtered)
+
+
+class CachedCompilerFilter(CompilerFilter):
+
+    def __init__(self, mimetype, **kwargs):
+        self.mimetype = mimetype
+        super(CachedCompilerFilter, self).__init__(**kwargs)
+
+    def input(self, **kwargs):
+        if self.mimetype in settings.COMPRESS_CACHEABLE_PRECOMPILERS:
+            key = self.get_cache_key()
+            data = cache.get(key)
+            if data:
+                return data
+            filtered = super(CachedCompilerFilter, self).input(**kwargs)
+            cache.set(key, filtered, settings.COMPRESS_REBUILD_TIMEOUT)
+            return filtered
+        else:
+            return super(CachedCompilerFilter, self).input(**kwargs)
+
+    def get_cache_key(self):
+        return get_precompiler_cachekey(self.command, self.content.encode('utf8'))

compressor/tests/test_filters.py

@@ -14,7 +14,7 @@ from compressor.cache import get_hashed_mtime, get_hashed_content
 from compressor.conf import settings
 from compressor.css import CssCompressor
 from compressor.utils import find_command
-from compressor.filters.base import CompilerFilter
+from compressor.filters.base import CompilerFilter, CachedCompilerFilter
 from compressor.filters.cssmin import CSSMinFilter, rCSSMinFilter
 from compressor.filters.css_default import CssAbsoluteFilter
 from compressor.filters.jsmin import JSMinFilter
@@ -51,6 +51,7 @@ class PrecompilerTestCase(TestCase):
     def setUp(self):
         self.test_precompiler = os.path.join(test_dir, 'precompiler.py')
         self.setup_infile()
+        settings.COMPRESS_CACHEABLE_PRECOMPILERS = ('text/css',)
 
     def setup_infile(self, filename='static/css/one.css'):
         self.filename = os.path.join(test_dir, filename)
@@ -104,18 +105,28 @@ class PrecompilerTestCase(TestCase):
 
     def test_precompiler_cache(self):
         command = '%s %s -f {infile} -o {outfile}' % (sys.executable, self.test_precompiler)
-        compiler = CompilerFilter(content=self.content, filename=self.filename, command=command)
+        compiler = CachedCompilerFilter(content=self.content, filename=self.filename, command=command, mimetype='text/css')
         self.assertEqual(u"body { color:#990; }", compiler.input())
         # We tell whether the precompiler actually ran by inspecting compiler.infile. If not None, the compiler had to
         # write the input out to the file for the external command. If None, it was in the cache and thus skipped.
         self.assertIsNotNone(compiler.infile)  # Not cached
 
-        compiler = CompilerFilter(content=self.content, filename=self.filename, command=command)
+        compiler = CachedCompilerFilter(content=self.content, filename=self.filename, command=command, mimetype='text/css')
         self.assertEqual(u"body { color:#990; }", compiler.input())
         self.assertIsNone(compiler.infile)  # Cached
 
         self.content += ' '  # Invalidate cache by slightly changing content
-        compiler = CompilerFilter(content=self.content, filename=self.filename, command=command)
+        compiler = CachedCompilerFilter(content=self.content, filename=self.filename, command=command, mimetype='text/css')
+        self.assertEqual(u"body { color:#990; }", compiler.input())
+        self.assertIsNotNone(compiler.infile)  # Not cached
+
+    def test_precompiler_not_cacheable(self):
+        command = '%s %s -f {infile} -o {outfile}' % (sys.executable, self.test_precompiler)
+        compiler = CachedCompilerFilter(content=self.content, filename=self.filename, command=command, mimetype='text/different')
+        self.assertEqual(u"body { color:#990; }", compiler.input())
+        self.assertIsNotNone(compiler.infile)  # Not cached
+
+        compiler = CachedCompilerFilter(content=self.content, filename=self.filename, command=command, mimetype='text/different')
         self.assertEqual(u"body { color:#990; }", compiler.input())
         self.assertIsNotNone(compiler.infile)  # Not cached