openstack/deb-python-django-compressor
Code Issues Proposed changes

added jinja2 compress offline support

Browse Source
This commit is contained in:
Lucas Tan
2014-03-05 23:14:40 +08:00
parent aba8c5f6b5
commit 786bbd2eb2
4 changed files with 274 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
compressor/exceptions.py
View File

@@ -38,3 +38,17 @@ class FilterDoesNotExist(Exception):
Raised when a filter class cannot be found.
"""
pass
class TemplateDoesNotExist(Exception):
"""
This exception is raised when a template does not exist.
"""
pass
class TemplateSyntaxError(Exception):
"""
This exception is raised when a template syntax error is encountered.
"""
pass

139
compressor/management/commands/compress.py
View File

@@ -2,19 +2,17 @@
import io
import os
import sys
from types import MethodType
from fnmatch import fnmatch
from optparse import make_option
from django.core.management.base import NoArgsCommand, CommandError
from django.template import (Context, Template,
TemplateDoesNotExist, TemplateSyntaxError)
import django.template
from django.template import Context
from django.utils import six
from django.utils.datastructures import SortedDict
from django.utils.importlib import import_module
from django.template.loader import get_template # noqa Leave this in to preload template locations
from django.template.loader_tags import (ExtendsNode, BlockNode,
BLOCK_CONTEXT_KEY)
try:
from django.template.loaders.cached import Loader as CachedLoader
@@ -23,7 +21,8 @@ except ImportError:
from compressor.cache import get_offline_hexdigest, write_offline_manifest
from compressor.conf import settings
from compressor.exceptions import OfflineGenerationError
from compressor.exceptions import (OfflineGenerationError, TemplateSyntaxError,
TemplateDoesNotExist)
from compressor.templatetags.compress import CompressorNode
if six.PY3:
@@ -37,77 +36,6 @@ else:
from StringIO import StringIO
def patched_render(self, context):
# 'Fake' _render method that just returns the context instead of
# rendering. It also checks whether the first node is an extend node or
# not, to be able to handle complex inheritance chain.
self._render_firstnode = MethodType(patched_render_firstnode, self)
self._render_firstnode(context)
# Cleanup, uninstall our _render monkeypatch now that it has been called
self._render = self._old_render
return context
def patched_render_firstnode(self, context):
# If this template has a ExtendsNode, we want to find out what
# should be put in render_context to make the {% block ... %}
# tags work.
#
# We can't fully render the base template(s) (we don't have the
# full context vars - only what's necessary to render the compress
# nodes!), therefore we hack the ExtendsNode we found, patching
# its get_parent method so that rendering the ExtendsNode only
# gives us the blocks content without doing any actual rendering.
extra_context = {}
try:
firstnode = self.nodelist[0]
except IndexError:
firstnode = None
if isinstance(firstnode, ExtendsNode):
firstnode._log = self._log
firstnode._log_verbosity = self._log_verbosity
firstnode._old_get_parent = firstnode.get_parent
firstnode.get_parent = MethodType(patched_get_parent, firstnode)
try:
extra_context = firstnode.render(context)
context.render_context = extra_context.render_context
# We aren't rendering {% block %} tags, but we want
# {{ block.super }} inside {% compress %} inside {% block %}s to
# work. Therefore, we need to pop() the last block context for
# each block name, to emulate what would have been done if the
# {% block %} had been fully rendered.
for blockname in firstnode.blocks.keys():
context.render_context[BLOCK_CONTEXT_KEY].pop(blockname)
except (IOError, TemplateSyntaxError, TemplateDoesNotExist):
# That first node we are trying to render might cause more errors
# that we didn't catch when simply creating a Template instance
# above, so we need to catch that (and ignore it, just like above)
# as well.
if self._log_verbosity > 0:
self._log.write("Caught error when rendering extend node from "
"template %s\n" % getattr(self, 'name', self))
return None
finally:
# Cleanup, uninstall our get_parent monkeypatch now that it has been called
firstnode.get_parent = firstnode._old_get_parent
return extra_context
def patched_get_parent(self, context):
# Patch template returned by extendsnode's get_parent to make sure their
# _render method is just returning the context instead of actually
# rendering stuff.
# In addition, this follows the inheritance chain by looking if the first
# node of the template is an extend node itself.
compiled_template = self._old_get_parent(context)
compiled_template._log = self._log
compiled_template._log_verbosity = self._log_verbosity
compiled_template._old_render = compiled_template._render
compiled_template._render = MethodType(patched_render, compiled_template)
return compiled_template
class Command(NoArgsCommand):
help = "Compress content outside of the request/response cycle"
option_list = NoArgsCommand.option_list + (
@@ -123,6 +51,9 @@ class Command(NoArgsCommand):
"(which defaults to STATIC_ROOT). Be aware that using this "
"can lead to infinite recursion if a link points to a parent "
"directory of itself.", dest='follow_links'),
make_option('--engine', default="django", action="store",
help="Specifies the templating engine. jinja2 or django",
dest="engine"),
)
requires_model_validation = False
@@ -140,7 +71,7 @@ class Command(NoArgsCommand):
# Force django to calculate template_source_loaders from
# TEMPLATE_LOADERS settings, by asking to find a dummy template
source, name = finder_func('test')
except TemplateDoesNotExist:
except django.template.TemplateDoesNotExist:
pass
# Reload template_source_loaders now that it has been calculated ;
# it should contain the list of valid, instanciated template loaders
@@ -216,11 +147,21 @@ class Command(NoArgsCommand):
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
engine = options.get("engine", "django")
if engine == "jinja2":
# TODO load jinja settings
from compressor.parser.jinja2 import Jinja2Parser
parser = Jinja2Parser(charset=settings.FILE_CHARSET, globals={}, filters={}, options={})
elif engine == "django":
from compressor.parser.dj import DjangoParser
parser = DjangoParser(charset=settings.FILE_CHARSET)
else:
raise OfflineGenerationError("Invalid templating engine specified.")
compressor_nodes = SortedDict()
for template_name in templates:
try:
with io.open(template_name, mode='rb') as file:
template = Template(file.read().decode(settings.FILE_CHARSET))
template = parser.parse(template_name)
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
@@ -237,7 +178,7 @@ class Command(NoArgsCommand):
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
nodes = list(self.walk_nodes(template))
nodes = list(parser.walk_nodes(template))
if nodes:
template.template_name = template_name
compressor_nodes.setdefault(template, []).extend(nodes)
@@ -261,22 +202,17 @@ class Command(NoArgsCommand):
context = Context(settings.COMPRESS_OFFLINE_CONTEXT)
template._log = log
template._log_verbosity = verbosity
template._render_firstnode = MethodType(patched_render_firstnode, template)
extra_context = template._render_firstnode(context)
if extra_context is None:
# Something is wrong - ignore this template
if not parser.process_template(template, context):
continue
for node in nodes:
context.push()
if extra_context and node._block_name:
# Give a block context to the node if it was found inside
# a {% block %}.
context['block'] = context.render_context[BLOCK_CONTEXT_KEY].get_block(node._block_name)
if context['block']:
context['block'].context = context
key = get_offline_hexdigest(node.nodelist.render(context))
parser.process_node(template, context, node)
rendered = parser.render_nodelist(template, context, node)
key = get_offline_hexdigest(rendered)
try:
result = node.render(context, forced=True)
result = parser.render_node(template, context, node)
except Exception as e:
raise CommandError("An error occured during rendering %s: "
"%s" % (template.template_name, e))
@@ -291,23 +227,6 @@ class Command(NoArgsCommand):
(count, len(compressor_nodes)))
return count, results
def get_nodelist(self, node):
# Check if node is an ```if``` switch with true and false branches
if hasattr(node, 'nodelist_true') and hasattr(node, 'nodelist_false'):
return node.nodelist_true + node.nodelist_false
return getattr(node, "nodelist", [])
def walk_nodes(self, node, block_name=None):
for node in self.get_nodelist(node):
if isinstance(node, BlockNode):
block_name = node.name
if isinstance(node, CompressorNode) and node.is_offline_compression_enabled(forced=True):
node._block_name = block_name
yield node
else:
for node in self.walk_nodes(node, block_name=block_name):
yield node
def handle_extensions(self, extensions=('html',)):
"""
organizes multiple extensions that are separated with commas or

141
compressor/parser/dj.py Normal file
View File

@@ -0,0 +1,141 @@
from __future__ import absolute_import
import io
from types import MethodType
from django import template
from django.template import Template
from django.template.loader_tags import (ExtendsNode, BlockNode,
BLOCK_CONTEXT_KEY)
from compressor.exceptions import TemplateSyntaxError
from compressor.templatetags.compress import CompressorNode
def patched_render(self, context):
# 'Fake' _render method that just returns the context instead of
# rendering. It also checks whether the first node is an extend node or
# not, to be able to handle complex inheritance chain.
self._render_firstnode = MethodType(patched_render_firstnode, self)
self._render_firstnode(context)
# Cleanup, uninstall our _render monkeypatch now that it has been called
self._render = self._old_render
return context
def patched_render_firstnode(self, context):
###from django.template.loader_tags import (ExtendsNode,
### BLOCK_CONTEXT_KEY)
# If this template has a ExtendsNode, we want to find out what
# should be put in render_context to make the {% block ... %}
# tags work.
#
# We can't fully render the base template(s) (we don't have the
# full context vars - only what's necessary to render the compress
# nodes!), therefore we hack the ExtendsNode we found, patching
# its get_parent method so that rendering the ExtendsNode only
# gives us the blocks content without doing any actual rendering.
extra_context = {}
try:
firstnode = self.nodelist[0]
except IndexError:
firstnode = None
if isinstance(firstnode, ExtendsNode):
firstnode._log = self._log
firstnode._log_verbosity = self._log_verbosity
firstnode._old_get_parent = firstnode.get_parent
firstnode.get_parent = MethodType(patched_get_parent, firstnode)
try:
extra_context = firstnode.render(context)
context.render_context = extra_context.render_context
# We aren't rendering {% block %} tags, but we want
# {{ block.super }} inside {% compress %} inside {% block %}s to
# work. Therefore, we need to pop() the last block context for
# each block name, to emulate what would have been done if the
# {% block %} had been fully rendered.
for blockname in firstnode.blocks.keys():
context.render_context[BLOCK_CONTEXT_KEY].pop(blockname)
except (IOError, template.TemplateSyntaxError,
template.TemplateDoesNotExist):
# That first node we are trying to render might cause more errors
# that we didn't catch when simply creating a Template instance
# above, so we need to catch that (and ignore it, just like above)
# as well.
if self._log_verbosity > 0:
self._log.write("Caught error when rendering extend node from "
"template %s\n" % getattr(self, 'name', self))
return None
finally:
# Cleanup, uninstall our get_parent monkeypatch now that it has been called
firstnode.get_parent = firstnode._old_get_parent
return extra_context
def patched_get_parent(self, context):
# Patch template returned by extendsnode's get_parent to make sure their
# _render method is just returning the context instead of actually
# rendering stuff.
# In addition, this follows the inheritance chain by looking if the first
# node of the template is an extend node itself.
compiled_template = self._old_get_parent(context)
compiled_template._log = self._log
compiled_template._log_verbosity = self._log_verbosity
compiled_template._old_render = compiled_template._render
compiled_template._render = MethodType(patched_render, compiled_template)
return compiled_template
class DjangoParser(object):
def __init__(self, charset):
self.charset = charset
def parse(self, template_name):
with io.open(template_name, mode='rb') as file:
try:
return Template(file.read().decode(self.charset))
except template.TemplateSyntaxError as e:
raise TemplateSyntaxError(str(e))
def process_template(self, template, context):
template._render_firstnode = MethodType(patched_render_firstnode, template)
template._extra_context = template._render_firstnode(context)
if template._extra_context is None:
# Something is wrong - ignore this template
return False
return True
def process_node(self, template, context, node):
if template._extra_context and node._block_name:
# Give a block context to the node if it was found inside
# a {% block %}.
context['block'] = context.render_context[BLOCK_CONTEXT_KEY].get_block(node._block_name)
if context['block']:
context['block'].context = context
def render_nodelist(self, template, context, node):
return node.nodelist.render(context)
def render_node(self, template, context, node):
return node.render(context, forced=True)
def get_nodelist(self, node):
# Check if node is an ```if``` switch with true and false branches
if hasattr(node, 'nodelist_true') and hasattr(node, 'nodelist_false'):
return node.nodelist_true + node.nodelist_false
return getattr(node, "nodelist", [])
def walk_nodes(self, node, block_name=None):
for node in self.get_nodelist(node):
if isinstance(node, BlockNode):
block_name = node.name
if isinstance(node, CompressorNode) and node.is_offline_compression_enabled(forced=True):
node._block_name = block_name
yield node
else:
for node in self.walk_nodes(node, block_name=block_name):
yield node

90
compressor/parser/jinja2.py Normal file
View File

@@ -0,0 +1,90 @@
from __future__ import absolute_import
import io
from django.template.defaulttags import IfNode
import jinja2
from jinja2.nodes import CallBlock, Call, ExtensionAttribute
from compressor.contrib.jinja2ext import CompressorExtension
from compressor.exceptions import TemplateSyntaxError
# TODO:
COMPRESSOR_JINJA2_ENV = {
}
COMPRESSOR_JINJA2_GLOBALS = {}
COMPRESSOR_JINJA2_FILTERS = {}
def flatten_context(context):
if hasattr(context, 'dicts'):
context_dict = {}
for d in context.dicts:
context_dict.update(d)
return context_dict
else:
return context
class Jinja2Parser(object):
COMPRESSOR_ID = 'compressor.contrib.jinja2ext.CompressorExtension'
def __init__(self, charset, filters, globals, options):
self.charset = charset
self.env = jinja2.Environment(extensions=[CompressorExtension],
**options)
self.env.globals.update(globals)
self.env.filters.update(filters)
def parse(self, template_name):
with io.open(template_name, mode='rb') as file:
try:
template = self.env.parse(file.read().decode(self.charset))
except jinja2.TemplateSyntaxError as e:
raise TemplateSyntaxError(str(e))
return template
def process_template(self, template, context):
return True
def process_node(self, template, context, node):
context.update(self.env.globals)
context.update(self.env.filters)
def render_nodelist(self, template, context, node):
compiled_node = self.env.compile(jinja2.nodes.Template(node.body))
template = jinja2.Template.from_code(self.env, compiled_node, {})
flat_context = flatten_context(context)
return template.render(flat_context)
def render_node(self, template, context, node):
context['compress_forced'] = True
compiled_node = self.env.compile(jinja2.nodes.Template([node]))
template = jinja2.Template.from_code(self.env, compiled_node, {})
flat_context = flatten_context(context)
return template.render(flat_context)
def get_nodelist(self, node):
if (isinstance(node, IfNode) and
hasattr(node, 'nodelist_true') and
hasattr(node, 'nodelist_false')):
return node.nodelist_true + node.nodelist_false
return getattr(node, "body", getattr(node, "nodes", []))
def walk_nodes(self, node, block_name=None):
for node in self.get_nodelist(node):
if (isinstance(node, CallBlock) and
isinstance(node.call, Call) and
isinstance(node.call.node, ExtensionAttribute) and
node.call.node.identifier == self.COMPRESSOR_ID):
yield node
else:
for node in self.walk_nodes(node, block_name=block_name):
yield node