59bea8ad59
To calculate the hash for a compress block, get_offline_hexdigest() calls smart_str on each template node, which resulted in the Python repr of the templates node which is not a text node.
This caused problems when using the {% static %} tag inside a compress block, since the {% static %} repr looks something like this:
>>> template.Template('{% load static %}{% static "foo" %}').nodelist
[<django.template.defaulttags.LoadNode at 0x10e747510>, <django.template.base.SimpleNode at 0x10e747490>]
The repr changes on every compilation (since the memory location changes), which caused the checksum to change every time, and thus made the offline mode unusable.
This patch renders the blocks completely, and calculates the checksums on the entire rendered block.
272 lines
12 KiB
Python
272 lines
12 KiB
Python
import os
|
|
import sys
|
|
from types import MethodType
|
|
from fnmatch import fnmatch
|
|
from optparse import make_option
|
|
|
|
try:
|
|
from cStringIO import StringIO
|
|
except ImportError:
|
|
from StringIO import StringIO
|
|
|
|
from django.core.management.base import NoArgsCommand, CommandError
|
|
from django.template import Context, Template, TemplateDoesNotExist, TemplateSyntaxError
|
|
from django.utils.datastructures import SortedDict
|
|
from django.utils.importlib import import_module
|
|
from django.template.loader import get_template
|
|
from django.template.loader_tags import ExtendsNode, BlockNode, BLOCK_CONTEXT_KEY
|
|
|
|
try:
|
|
from django.template.loaders.cached import Loader as CachedLoader
|
|
except ImportError:
|
|
CachedLoader = None
|
|
|
|
from compressor.cache import get_offline_hexdigest, write_offline_manifest
|
|
from compressor.conf import settings
|
|
from compressor.exceptions import OfflineGenerationError
|
|
from compressor.templatetags.compress import CompressorNode
|
|
from compressor.utils import walk, any
|
|
|
|
|
|
def patched_get_parent(self, context):
|
|
# Patch template returned by get_parent to make sure their _render method is
|
|
# just returning the context instead of actually rendering stuff.
|
|
compiled_template = self._old_get_parent(context)
|
|
compiled_template._render = MethodType(lambda self, c: c, compiled_template)
|
|
return compiled_template
|
|
|
|
|
|
class Command(NoArgsCommand):
|
|
help = "Compress content outside of the request/response cycle"
|
|
option_list = NoArgsCommand.option_list + (
|
|
make_option('--extension', '-e', action='append', dest='extensions',
|
|
help='The file extension(s) to examine (default: ".html", '
|
|
'separate multiple extensions with commas, or use -e '
|
|
'multiple times)'),
|
|
make_option('-f', '--force', default=False, action='store_true',
|
|
help="Force the generation of compressed content even if the "
|
|
"COMPRESS_ENABLED setting is not True.", dest='force'),
|
|
make_option('--follow-links', default=False, action='store_true',
|
|
help="Follow symlinks when traversing the COMPRESS_ROOT "
|
|
"(which defaults to MEDIA_ROOT). Be aware that using this "
|
|
"can lead to infinite recursion if a link points to a parent "
|
|
"directory of itself.", dest='follow_links'),
|
|
)
|
|
|
|
requires_model_validation = False
|
|
|
|
def get_loaders(self):
|
|
from django.template.loader import template_source_loaders
|
|
if template_source_loaders is None:
|
|
try:
|
|
from django.template.loader import (
|
|
find_template as finder_func)
|
|
except ImportError:
|
|
from django.template.loader import (
|
|
find_template_source as finder_func)
|
|
try:
|
|
source, name = finder_func('test')
|
|
except TemplateDoesNotExist:
|
|
pass
|
|
from django.template.loader import template_source_loaders
|
|
loaders = []
|
|
# If template loader is CachedTemplateLoader, return the loaders
|
|
# that it wraps around. So if we have
|
|
# TEMPLATE_LOADERS = (
|
|
# ('django.template.loaders.cached.Loader', (
|
|
# 'django.template.loaders.filesystem.Loader',
|
|
# 'django.template.loaders.app_directories.Loader',
|
|
# )),
|
|
# )
|
|
# The loaders will return django.template.loaders.filesystem.Loader
|
|
# and django.template.loaders.app_directories.Loader
|
|
for loader in template_source_loaders:
|
|
if CachedLoader is not None and isinstance(loader, CachedLoader):
|
|
loaders.extend(loader.loaders)
|
|
else:
|
|
loaders.append(loader)
|
|
return loaders
|
|
|
|
def compress(self, log=None, **options):
|
|
"""
|
|
Searches templates containing 'compress' nodes and compresses them
|
|
"offline" -- outside of the request/response cycle.
|
|
|
|
The result is cached with a cache-key derived from the content of the
|
|
compress nodes (not the content of the possibly linked files!).
|
|
"""
|
|
extensions = options.get('extensions')
|
|
extensions = self.handle_extensions(extensions or ['html'])
|
|
verbosity = int(options.get("verbosity", 0))
|
|
if not log:
|
|
log = StringIO()
|
|
if not settings.TEMPLATE_LOADERS:
|
|
raise OfflineGenerationError("No template loaders defined. You "
|
|
"must set TEMPLATE_LOADERS in your "
|
|
"settings.")
|
|
paths = set()
|
|
for loader in self.get_loaders():
|
|
try:
|
|
module = import_module(loader.__module__)
|
|
get_template_sources = getattr(module,
|
|
'get_template_sources', None)
|
|
if get_template_sources is None:
|
|
get_template_sources = loader.get_template_sources
|
|
paths.update(list(get_template_sources('')))
|
|
except (ImportError, AttributeError):
|
|
# Yeah, this didn't work out so well, let's move on
|
|
pass
|
|
if not paths:
|
|
raise OfflineGenerationError("No template paths found. None of "
|
|
"the configured template loaders "
|
|
"provided template paths. See "
|
|
"http://django.me/template-loaders "
|
|
"for more information on template "
|
|
"loaders.")
|
|
if verbosity > 1:
|
|
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
|
|
templates = set()
|
|
for path in paths:
|
|
for root, dirs, files in walk(path,
|
|
followlinks=options.get('followlinks', False)):
|
|
templates.update(os.path.join(root, name)
|
|
for name in files if not name.startswith('.') and
|
|
any(fnmatch(name, "*%s" % glob) for glob in extensions))
|
|
if not templates:
|
|
raise OfflineGenerationError("No templates found. Make sure your "
|
|
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
|
|
"settings are correct.")
|
|
if verbosity > 1:
|
|
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
|
|
|
|
compressor_nodes = SortedDict()
|
|
for template_name in templates:
|
|
try:
|
|
template_file = open(template_name)
|
|
try:
|
|
template = Template(template_file.read().decode(
|
|
settings.FILE_CHARSET))
|
|
finally:
|
|
template_file.close()
|
|
except IOError: # unreadable file -> ignore
|
|
if verbosity > 0:
|
|
log.write("Unreadable template at: %s\n" % template_name)
|
|
continue
|
|
except TemplateSyntaxError: # broken template -> ignore
|
|
if verbosity > 0:
|
|
log.write("Invalid template at: %s\n" % template_name)
|
|
continue
|
|
except TemplateDoesNotExist: # non existent template -> ignore
|
|
if verbosity > 0:
|
|
log.write("Non-existent template at: %s\n" % template_name)
|
|
continue
|
|
except UnicodeDecodeError:
|
|
if verbosity > 0:
|
|
log.write("UnicodeDecodeError while trying to read "
|
|
"template %s\n" % template_name)
|
|
nodes = list(self.walk_nodes(template))
|
|
if nodes:
|
|
template.template_name = template_name
|
|
compressor_nodes.setdefault(template, []).extend(nodes)
|
|
|
|
if not compressor_nodes:
|
|
raise OfflineGenerationError(
|
|
"No 'compress' template tags found in templates.")
|
|
|
|
if verbosity > 0:
|
|
log.write("Found 'compress' tags in:\n\t" +
|
|
"\n\t".join((t.template_name for t in compressor_nodes.keys())) + "\n")
|
|
|
|
log.write("Compressing... ")
|
|
count = 0
|
|
results = []
|
|
offline_manifest = {}
|
|
for template, nodes in compressor_nodes.iteritems():
|
|
context = Context(settings.COMPRESS_OFFLINE_CONTEXT)
|
|
extra_context = {}
|
|
firstnode = template.nodelist[0]
|
|
if isinstance(firstnode, ExtendsNode):
|
|
# If this template has a ExtendsNode, we apply our patch to
|
|
# generate the necessary context, and then use it for all the
|
|
# nodes in it, just in case (we don't know which nodes were
|
|
# in a block)
|
|
firstnode._old_get_parent = firstnode.get_parent
|
|
firstnode.get_parent = MethodType(patched_get_parent, firstnode)
|
|
try:
|
|
extra_context = firstnode.render(context)
|
|
context.render_context = extra_context.render_context
|
|
except (IOError, TemplateSyntaxError, TemplateDoesNotExist):
|
|
# That first node we are trying to render might cause more errors
|
|
# that we didn't catch when simply creating a Template instance
|
|
# above, so we need to catch that (and ignore it, just like above)
|
|
# as well.
|
|
if verbosity > 0:
|
|
log.write("Caught error when rendering extend node from template %s\n" % template.template_name)
|
|
continue
|
|
for node in nodes:
|
|
context.push()
|
|
if extra_context and node._block_name:
|
|
context['block'] = context.render_context[BLOCK_CONTEXT_KEY].pop(node._block_name)
|
|
if context['block']:
|
|
context['block'].context = context
|
|
key = get_offline_hexdigest(node.nodelist.render(context))
|
|
try:
|
|
result = node.render(context, forced=True)
|
|
except Exception, e:
|
|
raise CommandError("An error occured during rendering %s: "
|
|
"%s" % (template.template_name, e))
|
|
offline_manifest[key] = result
|
|
context.pop()
|
|
results.append(result)
|
|
count += 1
|
|
|
|
write_offline_manifest(offline_manifest)
|
|
|
|
log.write("done\nCompressed %d block(s) from %d template(s).\n" %
|
|
(count, len(compressor_nodes)))
|
|
return count, results
|
|
|
|
def walk_nodes(self, node, block_name=None):
|
|
for node in getattr(node, "nodelist", []):
|
|
if isinstance(node, BlockNode):
|
|
block_name = node.name
|
|
if isinstance(node, CompressorNode):
|
|
node._block_name = block_name
|
|
yield node
|
|
else:
|
|
for node in self.walk_nodes(node, block_name=block_name):
|
|
yield node
|
|
|
|
def handle_extensions(self, extensions=('html',)):
|
|
"""
|
|
organizes multiple extensions that are separated with commas or
|
|
passed by using --extension/-e multiple times.
|
|
|
|
for example: running 'django-admin compress -e js,txt -e xhtml -a'
|
|
would result in a extension list: ['.js', '.txt', '.xhtml']
|
|
|
|
>>> handle_extensions(['.html', 'html,js,py,py,py,.py', 'py,.py'])
|
|
['.html', '.js']
|
|
>>> handle_extensions(['.html, txt,.tpl'])
|
|
['.html', '.tpl', '.txt']
|
|
"""
|
|
ext_list = []
|
|
for ext in extensions:
|
|
ext_list.extend(ext.replace(' ', '').split(','))
|
|
for i, ext in enumerate(ext_list):
|
|
if not ext.startswith('.'):
|
|
ext_list[i] = '.%s' % ext_list[i]
|
|
return set(ext_list)
|
|
|
|
def handle_noargs(self, **options):
|
|
if not settings.COMPRESS_ENABLED and not options.get("force"):
|
|
raise CommandError(
|
|
"Compressor is disabled. Set the COMPRESS_ENABLED "
|
|
"settting or use --force to override.")
|
|
if not settings.COMPRESS_OFFLINE:
|
|
if not options.get("force"):
|
|
raise CommandError(
|
|
"Offline compression is disabled. Set "
|
|
"COMPRESS_OFFLINE or use the --force to override.")
|
|
self.compress(sys.stdout, **options)
|