diff --git a/compressor/exceptions.py b/compressor/exceptions.py index 07d79a1..c2d7c60 100644 --- a/compressor/exceptions.py +++ b/compressor/exceptions.py @@ -38,3 +38,17 @@ class FilterDoesNotExist(Exception): Raised when a filter class cannot be found. """ pass + + +class TemplateDoesNotExist(Exception): + """ + This exception is raised when a template does not exist. + """ + pass + + +class TemplateSyntaxError(Exception): + """ + This exception is raised when a template syntax error is encountered. + """ + pass diff --git a/compressor/management/commands/compress.py b/compressor/management/commands/compress.py index a7efaca..ac57d77 100644 --- a/compressor/management/commands/compress.py +++ b/compressor/management/commands/compress.py @@ -2,19 +2,17 @@ import io import os import sys -from types import MethodType + from fnmatch import fnmatch from optparse import make_option from django.core.management.base import NoArgsCommand, CommandError -from django.template import (Context, Template, - TemplateDoesNotExist, TemplateSyntaxError) +import django.template +from django.template import Context from django.utils import six from django.utils.datastructures import SortedDict from django.utils.importlib import import_module from django.template.loader import get_template # noqa Leave this in to preload template locations -from django.template.loader_tags import (ExtendsNode, BlockNode, - BLOCK_CONTEXT_KEY) try: from django.template.loaders.cached import Loader as CachedLoader @@ -23,7 +21,8 @@ except ImportError: from compressor.cache import get_offline_hexdigest, write_offline_manifest from compressor.conf import settings -from compressor.exceptions import OfflineGenerationError +from compressor.exceptions import (OfflineGenerationError, TemplateSyntaxError, + TemplateDoesNotExist) from compressor.templatetags.compress import CompressorNode if six.PY3: @@ -37,77 +36,6 @@ else: from StringIO import StringIO -def patched_render(self, context): - # 'Fake' _render method that just returns the context instead of - # rendering. It also checks whether the first node is an extend node or - # not, to be able to handle complex inheritance chain. - self._render_firstnode = MethodType(patched_render_firstnode, self) - self._render_firstnode(context) - - # Cleanup, uninstall our _render monkeypatch now that it has been called - self._render = self._old_render - return context - - -def patched_render_firstnode(self, context): - # If this template has a ExtendsNode, we want to find out what - # should be put in render_context to make the {% block ... %} - # tags work. - # - # We can't fully render the base template(s) (we don't have the - # full context vars - only what's necessary to render the compress - # nodes!), therefore we hack the ExtendsNode we found, patching - # its get_parent method so that rendering the ExtendsNode only - # gives us the blocks content without doing any actual rendering. - extra_context = {} - try: - firstnode = self.nodelist[0] - except IndexError: - firstnode = None - if isinstance(firstnode, ExtendsNode): - firstnode._log = self._log - firstnode._log_verbosity = self._log_verbosity - firstnode._old_get_parent = firstnode.get_parent - firstnode.get_parent = MethodType(patched_get_parent, firstnode) - try: - extra_context = firstnode.render(context) - context.render_context = extra_context.render_context - # We aren't rendering {% block %} tags, but we want - # {{ block.super }} inside {% compress %} inside {% block %}s to - # work. Therefore, we need to pop() the last block context for - # each block name, to emulate what would have been done if the - # {% block %} had been fully rendered. - for blockname in firstnode.blocks.keys(): - context.render_context[BLOCK_CONTEXT_KEY].pop(blockname) - except (IOError, TemplateSyntaxError, TemplateDoesNotExist): - # That first node we are trying to render might cause more errors - # that we didn't catch when simply creating a Template instance - # above, so we need to catch that (and ignore it, just like above) - # as well. - if self._log_verbosity > 0: - self._log.write("Caught error when rendering extend node from " - "template %s\n" % getattr(self, 'name', self)) - return None - finally: - # Cleanup, uninstall our get_parent monkeypatch now that it has been called - firstnode.get_parent = firstnode._old_get_parent - return extra_context - - -def patched_get_parent(self, context): - # Patch template returned by extendsnode's get_parent to make sure their - # _render method is just returning the context instead of actually - # rendering stuff. - # In addition, this follows the inheritance chain by looking if the first - # node of the template is an extend node itself. - compiled_template = self._old_get_parent(context) - compiled_template._log = self._log - compiled_template._log_verbosity = self._log_verbosity - compiled_template._old_render = compiled_template._render - compiled_template._render = MethodType(patched_render, compiled_template) - return compiled_template - - class Command(NoArgsCommand): help = "Compress content outside of the request/response cycle" option_list = NoArgsCommand.option_list + ( @@ -123,6 +51,9 @@ class Command(NoArgsCommand): "(which defaults to STATIC_ROOT). Be aware that using this " "can lead to infinite recursion if a link points to a parent " "directory of itself.", dest='follow_links'), + make_option('--engine', default="django", action="store", + help="Specifies the templating engine. jinja2 or django", + dest="engine"), ) requires_model_validation = False @@ -140,7 +71,7 @@ class Command(NoArgsCommand): # Force django to calculate template_source_loaders from # TEMPLATE_LOADERS settings, by asking to find a dummy template source, name = finder_func('test') - except TemplateDoesNotExist: + except django.template.TemplateDoesNotExist: pass # Reload template_source_loaders now that it has been calculated ; # it should contain the list of valid, instanciated template loaders @@ -216,11 +147,21 @@ class Command(NoArgsCommand): if verbosity > 1: log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n") + engine = options.get("engine", "django") + if engine == "jinja2": + # TODO load jinja settings + from compressor.parser.jinja2 import Jinja2Parser + parser = Jinja2Parser(charset=settings.FILE_CHARSET, globals={}, filters={}, options={}) + elif engine == "django": + from compressor.parser.dj import DjangoParser + parser = DjangoParser(charset=settings.FILE_CHARSET) + else: + raise OfflineGenerationError("Invalid templating engine specified.") + compressor_nodes = SortedDict() for template_name in templates: try: - with io.open(template_name, mode='rb') as file: - template = Template(file.read().decode(settings.FILE_CHARSET)) + template = parser.parse(template_name) except IOError: # unreadable file -> ignore if verbosity > 0: log.write("Unreadable template at: %s\n" % template_name) @@ -237,7 +178,7 @@ class Command(NoArgsCommand): if verbosity > 0: log.write("UnicodeDecodeError while trying to read " "template %s\n" % template_name) - nodes = list(self.walk_nodes(template)) + nodes = list(parser.walk_nodes(template)) if nodes: template.template_name = template_name compressor_nodes.setdefault(template, []).extend(nodes) @@ -261,22 +202,17 @@ class Command(NoArgsCommand): context = Context(settings.COMPRESS_OFFLINE_CONTEXT) template._log = log template._log_verbosity = verbosity - template._render_firstnode = MethodType(patched_render_firstnode, template) - extra_context = template._render_firstnode(context) - if extra_context is None: - # Something is wrong - ignore this template + + if not parser.process_template(template, context): continue + for node in nodes: context.push() - if extra_context and node._block_name: - # Give a block context to the node if it was found inside - # a {% block %}. - context['block'] = context.render_context[BLOCK_CONTEXT_KEY].get_block(node._block_name) - if context['block']: - context['block'].context = context - key = get_offline_hexdigest(node.nodelist.render(context)) + parser.process_node(template, context, node) + rendered = parser.render_nodelist(template, context, node) + key = get_offline_hexdigest(rendered) try: - result = node.render(context, forced=True) + result = parser.render_node(template, context, node) except Exception as e: raise CommandError("An error occured during rendering %s: " "%s" % (template.template_name, e)) @@ -291,23 +227,6 @@ class Command(NoArgsCommand): (count, len(compressor_nodes))) return count, results - def get_nodelist(self, node): - # Check if node is an ```if``` switch with true and false branches - if hasattr(node, 'nodelist_true') and hasattr(node, 'nodelist_false'): - return node.nodelist_true + node.nodelist_false - return getattr(node, "nodelist", []) - - def walk_nodes(self, node, block_name=None): - for node in self.get_nodelist(node): - if isinstance(node, BlockNode): - block_name = node.name - if isinstance(node, CompressorNode) and node.is_offline_compression_enabled(forced=True): - node._block_name = block_name - yield node - else: - for node in self.walk_nodes(node, block_name=block_name): - yield node - def handle_extensions(self, extensions=('html',)): """ organizes multiple extensions that are separated with commas or diff --git a/compressor/parser/dj.py b/compressor/parser/dj.py new file mode 100644 index 0000000..5686073 --- /dev/null +++ b/compressor/parser/dj.py @@ -0,0 +1,141 @@ +from __future__ import absolute_import + +import io +from types import MethodType + +from django import template +from django.template import Template +from django.template.loader_tags import (ExtendsNode, BlockNode, + BLOCK_CONTEXT_KEY) + +from compressor.exceptions import TemplateSyntaxError +from compressor.templatetags.compress import CompressorNode + + +def patched_render(self, context): + # 'Fake' _render method that just returns the context instead of + # rendering. It also checks whether the first node is an extend node or + # not, to be able to handle complex inheritance chain. + self._render_firstnode = MethodType(patched_render_firstnode, self) + self._render_firstnode(context) + + # Cleanup, uninstall our _render monkeypatch now that it has been called + self._render = self._old_render + return context + + +def patched_render_firstnode(self, context): + ###from django.template.loader_tags import (ExtendsNode, + ### BLOCK_CONTEXT_KEY) + + # If this template has a ExtendsNode, we want to find out what + # should be put in render_context to make the {% block ... %} + # tags work. + # + # We can't fully render the base template(s) (we don't have the + # full context vars - only what's necessary to render the compress + # nodes!), therefore we hack the ExtendsNode we found, patching + # its get_parent method so that rendering the ExtendsNode only + # gives us the blocks content without doing any actual rendering. + extra_context = {} + try: + firstnode = self.nodelist[0] + except IndexError: + firstnode = None + if isinstance(firstnode, ExtendsNode): + firstnode._log = self._log + firstnode._log_verbosity = self._log_verbosity + firstnode._old_get_parent = firstnode.get_parent + firstnode.get_parent = MethodType(patched_get_parent, firstnode) + try: + extra_context = firstnode.render(context) + context.render_context = extra_context.render_context + # We aren't rendering {% block %} tags, but we want + # {{ block.super }} inside {% compress %} inside {% block %}s to + # work. Therefore, we need to pop() the last block context for + # each block name, to emulate what would have been done if the + # {% block %} had been fully rendered. + for blockname in firstnode.blocks.keys(): + context.render_context[BLOCK_CONTEXT_KEY].pop(blockname) + except (IOError, template.TemplateSyntaxError, + template.TemplateDoesNotExist): + # That first node we are trying to render might cause more errors + # that we didn't catch when simply creating a Template instance + # above, so we need to catch that (and ignore it, just like above) + # as well. + if self._log_verbosity > 0: + self._log.write("Caught error when rendering extend node from " + "template %s\n" % getattr(self, 'name', self)) + return None + finally: + # Cleanup, uninstall our get_parent monkeypatch now that it has been called + firstnode.get_parent = firstnode._old_get_parent + return extra_context + + +def patched_get_parent(self, context): + # Patch template returned by extendsnode's get_parent to make sure their + # _render method is just returning the context instead of actually + # rendering stuff. + # In addition, this follows the inheritance chain by looking if the first + # node of the template is an extend node itself. + compiled_template = self._old_get_parent(context) + compiled_template._log = self._log + compiled_template._log_verbosity = self._log_verbosity + compiled_template._old_render = compiled_template._render + compiled_template._render = MethodType(patched_render, compiled_template) + return compiled_template + + +class DjangoParser(object): + def __init__(self, charset): + self.charset = charset + + def parse(self, template_name): + with io.open(template_name, mode='rb') as file: + try: + return Template(file.read().decode(self.charset)) + except template.TemplateSyntaxError as e: + raise TemplateSyntaxError(str(e)) + + def process_template(self, template, context): + template._render_firstnode = MethodType(patched_render_firstnode, template) + template._extra_context = template._render_firstnode(context) + + if template._extra_context is None: + # Something is wrong - ignore this template + return False + + return True + + def process_node(self, template, context, node): + if template._extra_context and node._block_name: + # Give a block context to the node if it was found inside + # a {% block %}. + context['block'] = context.render_context[BLOCK_CONTEXT_KEY].get_block(node._block_name) + + if context['block']: + context['block'].context = context + + def render_nodelist(self, template, context, node): + return node.nodelist.render(context) + + def render_node(self, template, context, node): + return node.render(context, forced=True) + + def get_nodelist(self, node): + # Check if node is an ```if``` switch with true and false branches + if hasattr(node, 'nodelist_true') and hasattr(node, 'nodelist_false'): + return node.nodelist_true + node.nodelist_false + return getattr(node, "nodelist", []) + + def walk_nodes(self, node, block_name=None): + for node in self.get_nodelist(node): + if isinstance(node, BlockNode): + block_name = node.name + if isinstance(node, CompressorNode) and node.is_offline_compression_enabled(forced=True): + node._block_name = block_name + yield node + else: + for node in self.walk_nodes(node, block_name=block_name): + yield node diff --git a/compressor/parser/jinja2.py b/compressor/parser/jinja2.py new file mode 100644 index 0000000..a556ae9 --- /dev/null +++ b/compressor/parser/jinja2.py @@ -0,0 +1,90 @@ +from __future__ import absolute_import + +import io + +from django.template.defaulttags import IfNode + +import jinja2 +from jinja2.nodes import CallBlock, Call, ExtensionAttribute + +from compressor.contrib.jinja2ext import CompressorExtension +from compressor.exceptions import TemplateSyntaxError + +# TODO: +COMPRESSOR_JINJA2_ENV = { + +} +COMPRESSOR_JINJA2_GLOBALS = {} +COMPRESSOR_JINJA2_FILTERS = {} + + +def flatten_context(context): + if hasattr(context, 'dicts'): + context_dict = {} + + for d in context.dicts: + context_dict.update(d) + + return context_dict + else: + return context + + +class Jinja2Parser(object): + COMPRESSOR_ID = 'compressor.contrib.jinja2ext.CompressorExtension' + + def __init__(self, charset, filters, globals, options): + self.charset = charset + self.env = jinja2.Environment(extensions=[CompressorExtension], + **options) + self.env.globals.update(globals) + self.env.filters.update(filters) + + def parse(self, template_name): + with io.open(template_name, mode='rb') as file: + try: + template = self.env.parse(file.read().decode(self.charset)) + except jinja2.TemplateSyntaxError as e: + raise TemplateSyntaxError(str(e)) + + return template + + def process_template(self, template, context): + return True + + def process_node(self, template, context, node): + context.update(self.env.globals) + context.update(self.env.filters) + + def render_nodelist(self, template, context, node): + compiled_node = self.env.compile(jinja2.nodes.Template(node.body)) + template = jinja2.Template.from_code(self.env, compiled_node, {}) + flat_context = flatten_context(context) + + return template.render(flat_context) + + def render_node(self, template, context, node): + context['compress_forced'] = True + compiled_node = self.env.compile(jinja2.nodes.Template([node])) + template = jinja2.Template.from_code(self.env, compiled_node, {}) + flat_context = flatten_context(context) + + return template.render(flat_context) + + def get_nodelist(self, node): + if (isinstance(node, IfNode) and + hasattr(node, 'nodelist_true') and + hasattr(node, 'nodelist_false')): + return node.nodelist_true + node.nodelist_false + return getattr(node, "body", getattr(node, "nodes", [])) + + def walk_nodes(self, node, block_name=None): + for node in self.get_nodelist(node): + if (isinstance(node, CallBlock) and + isinstance(node.call, Call) and + isinstance(node.call.node, ExtensionAttribute) and + node.call.node.identifier == self.COMPRESSOR_ID): + yield node + else: + for node in self.walk_nodes(node, block_name=block_name): + yield node