diff --git a/AUTHORS b/AUTHORS index 245eafd..f72b6a6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -57,7 +57,8 @@ Julien Phalip Justin Lilly Luis Nell Lukas Lehner -Lukasz Balcerzak +Łukasz Balcerzak +Łukasz Langa Maciek Szczesniak Maor Ben-Dayan Mark Lavin @@ -89,4 +90,4 @@ Ulrich Petri Ulysses V Vladislav Poluhin wesleyb -Wilson Júnior \ No newline at end of file +Wilson Júnior diff --git a/Makefile b/Makefile index 598b837..ae5eec1 100644 --- a/Makefile +++ b/Makefile @@ -2,3 +2,5 @@ test: flake8 compressor --ignore=E501,E128 coverage run --branch --source=compressor `which django-admin.py` test --settings=compressor.test_settings compressor coverage report --omit=compressor/test*,compressor/filters/jsmin/rjsmin*,compressor/filters/cssmin/cssmin*,compressor/utils/stringformat* + +.PHONY: test diff --git a/compressor/parser/html5lib.py b/compressor/parser/html5lib.py index 86d598f..2113983 100644 --- a/compressor/parser/html5lib.py +++ b/compressor/parser/html5lib.py @@ -20,39 +20,42 @@ class Html5LibParser(ParserBase): self.html5lib = html5lib def _serialize(self, elem): - fragment = self.html5lib.treebuilders.simpletree.DocumentFragment() - fragment.appendChild(elem) - return self.html5lib.serialize(fragment, - quote_attr_values=True, omit_optional_tags=False) + return self.html5lib.serialize( + elem, tree="etree", quote_attr_values=True, + omit_optional_tags=False, use_trailing_solidus=True, + ) def _find(self, *names): - for node in self.html.childNodes: - if node.type == 5 and node.name in names: - yield node + for elem in self.html: + if elem.tag in names: + yield elem @cached_property def html(self): try: - return self.html5lib.parseFragment(self.content) + return self.html5lib.parseFragment(self.content, treebuilder="etree") except ImportError as err: raise ImproperlyConfigured("Error while importing html5lib: %s" % err) except Exception as err: raise ParserError("Error while initializing Parser: %s" % err) def css_elems(self): - return self._find('style', 'link') + return self._find('{http://www.w3.org/1999/xhtml}link', + '{http://www.w3.org/1999/xhtml}style') def js_elems(self): - return self._find('script') + return self._find('{http://www.w3.org/1999/xhtml}script') def elem_attribs(self, elem): - return elem.attributes + return elem.attrib def elem_content(self, elem): - return elem.childNodes[0].value + return smart_unicode(elem.text) def elem_name(self, elem): - return elem.name + if '}' in elem.tag: + return elem.tag.split('}')[1] + return elem.tag def elem_str(self, elem): # This method serializes HTML in a way that does not pass all tests. diff --git a/compressor/storage.py b/compressor/storage.py index b2be9b3..ca6dfe8 100644 --- a/compressor/storage.py +++ b/compressor/storage.py @@ -2,8 +2,8 @@ from __future__ import unicode_literals import errno import gzip import os -from os import path from datetime import datetime +import time from django.core.files.storage import FileSystemStorage, get_storage_class from django.utils.functional import LazyObject, SimpleLazyObject @@ -28,13 +28,13 @@ class CompressorFileStorage(FileSystemStorage): *args, **kwargs) def accessed_time(self, name): - return datetime.fromtimestamp(path.getatime(self.path(name))) + return datetime.fromtimestamp(os.path.getatime(self.path(name))) def created_time(self, name): - return datetime.fromtimestamp(path.getctime(self.path(name))) + return datetime.fromtimestamp(os.path.getctime(self.path(name))) def modified_time(self, name): - return datetime.fromtimestamp(path.getmtime(self.path(name))) + return datetime.fromtimestamp(os.path.getmtime(self.path(name))) def get_available_name(self, name): """ @@ -67,14 +67,26 @@ class GzipCompressorFileStorage(CompressorFileStorage): """ def save(self, filename, content): filename = super(GzipCompressorFileStorage, self).save(filename, content) - # workaround for http://bugs.python.org/issue13664 name = os.path.basename(filename).encode('latin1', 'replace') - f_in = open(self.path(filename), 'rb') - f_out = gzip.GzipFile(name, fileobj=open('%s.gz' % self.path(filename), 'wb')) - f_out.write(f_in.read()) - f_out.close() - f_in.close() + orig_path = self.path(filename) + compressed_path = '%s.gz' % orig_path + + f_in = open(orig_path, 'rb') + f_out = open('%s.gz' % compressed_path, 'wb') + try: + f_out = gzip.GzipFile(name, fileobj=f_out) + f_out.write(f_in.read()) + finally: + f_out.close() + f_in.close() + # Ensure the file timestamps match. + # os.stat() returns nanosecond resolution on Linux, but os.utime() + # only sets microsecond resolution. Set times on both files to + # ensure they are equal. + stamp = time.time() + os.utime(orig_path, (stamp, stamp)) + os.utime(compressed_path, (stamp, stamp)) return filename diff --git a/compressor/tests/test_parsers.py b/compressor/tests/test_parsers.py index 1fa9f51..364b7b6 100644 --- a/compressor/tests/test_parsers.py +++ b/compressor/tests/test_parsers.py @@ -19,7 +19,6 @@ except ImportError: from compressor.base import SOURCE_HUNK, SOURCE_FILE from compressor.conf import settings -from compressor.css import CssCompressor from compressor.tests.test_base import CompressorTestCase try: @@ -29,7 +28,6 @@ except ImportError: class ParserTestCase(object): - def setUp(self): self.old_parser = settings.COMPRESS_PARSER settings.COMPRESS_PARSER = self.parser_cls @@ -47,34 +45,86 @@ class LxmlParserTests(ParserTestCase, CompressorTestCase): @ut2.skipIf(html5lib is None, 'html5lib not found') class Html5LibParserTests(ParserTestCase, CompressorTestCase): parser_cls = 'compressor.parser.Html5LibParser' - - def setUp(self): - super(Html5LibParserTests, self).setUp() - # special version of the css since the parser sucks - self.css = """\ - - -""" - self.css_node = CssCompressor(self.css) + # Special test variants required since xml.etree holds attributes + # as a plain dictionary, e.g. key order is unpredictable. def test_css_split(self): - out = [ - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, 'css', 'one.css'), 'css/one.css', ''), - (SOURCE_HUNK, 'p { border:5px solid green;}', None, ''), - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, 'css', 'two.css'), 'css/two.css', ''), - ] split = self.css_node.split_contents() - split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3])) for x in split] - self.assertEqual(out, split) + out0 = ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'css', 'one.css'), + 'css/one.css', + '{http://www.w3.org/1999/xhtml}link', + {'rel': 'stylesheet', 'href': '/static/css/one.css', + 'type': 'text/css'}, + ) + self.assertEqual(out0, split[0][:3] + (split[0][3].tag, + split[0][3].attrib)) + out1 = ( + SOURCE_HUNK, + 'p { border:5px solid green;}', + None, + '', + ) + self.assertEqual(out1, split[1][:3] + + (self.css_node.parser.elem_str(split[1][3]),)) + out2 = ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'css', 'two.css'), + 'css/two.css', + '{http://www.w3.org/1999/xhtml}link', + {'rel': 'stylesheet', 'href': '/static/css/two.css', + 'type': 'text/css'}, + ) + self.assertEqual(out2, split[2][:3] + (split[2][3].tag, + split[2][3].attrib)) def test_js_split(self): - out = [ - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, 'js', 'one.js'), 'js/one.js', ''), - (SOURCE_HUNK, 'obj.value = "value";', None, ''), - ] split = self.js_node.split_contents() - split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3])) for x in split] - self.assertEqual(out, split) + out0 = ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'js', 'one.js'), + 'js/one.js', + '{http://www.w3.org/1999/xhtml}script', + {'src': '/static/js/one.js', 'type': 'text/javascript'}, + None, + ) + self.assertEqual(out0, split[0][:3] + (split[0][3].tag, + split[0][3].attrib, + split[0][3].text)) + out1 = ( + SOURCE_HUNK, + 'obj.value = "value";', + None, + '{http://www.w3.org/1999/xhtml}script', + {'type': 'text/javascript'}, + 'obj.value = "value";', + ) + self.assertEqual(out1, split[1][:3] + (split[1][3].tag, + split[1][3].attrib, + split[1][3].text)) + + def test_css_return_if_off(self): + settings.COMPRESS_ENABLED = False + # Yes, they are semantically equal but attributes might be + # scrambled in unpredictable order. A more elaborate check + # would require parsing both arguments with a different parser + # and then evaluating the result, which no longer is + # a meaningful unit test. + self.assertEqual(len(self.css), len(self.css_node.output())) + + def test_js_return_if_off(self): + try: + enabled = settings.COMPRESS_ENABLED + precompilers = settings.COMPRESS_PRECOMPILERS + settings.COMPRESS_ENABLED = False + settings.COMPRESS_PRECOMPILERS = {} + # As above. + self.assertEqual(len(self.js), len(self.js_node.output())) + finally: + settings.COMPRESS_ENABLED = enabled + settings.COMPRESS_PRECOMPILERS = precompilers + @ut2.skipIf(BeautifulSoup is None, 'BeautifulSoup not found')