diff --git a/compressor/base.py b/compressor/base.py index 4e91d4d..f5f2dc8 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -1,17 +1,17 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals import os import codecs import urllib +import six from django.core.files.base import ContentFile from django.template import Context from django.template.loader import render_to_string -from django.utils.encoding import smart_unicode from django.utils.importlib import import_module from django.utils.safestring import mark_safe from compressor.cache import get_hexdigest, get_mtime - +from compressor.utils.compat import smart_text from compressor.conf import settings from compressor.exceptions import (CompressorError, UncompressableFileError, FilterDoesNotExist) @@ -34,7 +34,7 @@ class Compressor(object): type = None def __init__(self, content=None, output_prefix=None, context=None, *args, **kwargs): - self.content = content or "" + self.content = content or "" # rendered contents of {% compress %} tag self.output_prefix = output_prefix or "compressed" self.output_dir = settings.COMPRESS_OUTPUT_DIR.strip('/') self.charset = settings.DEFAULT_CHARSET @@ -65,6 +65,10 @@ class Compressor(object): return "compressor/%s_%s.html" % (self.type, mode) def get_basename(self, url): + """ + Takes full path to a static file (eg. "/static/css/style.css") and + returns path with storage's base url removed (eg. "css/style.css"). + """ try: base_url = self.storage.base_url except AttributeError: @@ -78,6 +82,17 @@ class Compressor(object): return basename.split("?", 1)[0] def get_filepath(self, content, basename=None): + """ + Returns file path for an output file based on contents. + + Returned path is relative to compressor storage's base url, for + example "CACHE/css/e41ba2cc6982.css". + + When `basename` argument is provided then file name (without extension) + will be used as a part of returned file name, for example: + + get_filepath(content, "my_file.css") -> 'CACHE/css/my_file.e41ba2cc6982.css' + """ parts = [] if basename: filename = os.path.split(basename)[1] @@ -86,6 +101,11 @@ class Compressor(object): return os.path.join(self.output_dir, self.output_prefix, '.'.join(parts)) def get_filename(self, basename): + """ + Returns full path to a file, for example: + + get_filename('css/one.css') -> '/full/path/to/static/css/one.css' + """ filename = None # first try finding the file in the root try: @@ -110,13 +130,16 @@ class Compressor(object): self.finders and " or with staticfiles." or ".")) def get_filecontent(self, filename, charset): - with codecs.open(filename, 'rb', charset) as fd: + """ + Reads file contents using given `charset` and returns it as text. + """ + with codecs.open(filename, 'r', charset) as fd: try: return fd.read() - except IOError, e: + except IOError as e: raise UncompressableFileError("IOError while processing " "'%s': %s" % (filename, e)) - except UnicodeDecodeError, e: + except UnicodeDecodeError as e: raise UncompressableFileError("UnicodeDecodeError while " "processing '%s' with " "charset %s: %s" % @@ -143,7 +166,7 @@ class Compressor(object): def hunks(self, forced=False): """ - The heart of content parsing, iterates of the + The heart of content parsing, iterates over the list of split contents and looks at its kind to decide what to do with it. Should yield a bunch of precompiled and/or rendered hunks. @@ -170,11 +193,11 @@ class Compressor(object): if enabled: value = self.filter(value, **options) - yield smart_unicode(value, charset.lower()) + yield smart_text(value, charset.lower()) else: if precompiled: value = self.handle_output(kind, value, forced=True, basename=basename) - yield smart_unicode(value, charset.lower()) + yield smart_text(value, charset.lower()) else: yield self.parser.elem_str(elem) @@ -243,11 +266,10 @@ class Compressor(object): any custom modification. Calls other mode specific methods or simply returns the content directly. """ - content = self.filter_input(forced) - if not content: - return '' + output = '\n'.join(self.filter_input(forced)) - output = '\n'.join(c.encode(self.charset) for c in content) + if not output: + return '' if settings.COMPRESS_ENABLED or forced: filtered_output = self.filter_output(output) diff --git a/compressor/cache.py b/compressor/cache.py index 1caeded..937f44b 100644 --- a/compressor/cache.py +++ b/compressor/cache.py @@ -1,3 +1,4 @@ +import json import hashlib import os import socket @@ -5,31 +6,31 @@ import time from django.core.cache import get_cache from django.core.files.base import ContentFile -from django.utils import simplejson -from django.utils.encoding import smart_str +from django.utils.encoding import force_str from django.utils.functional import SimpleLazyObject from django.utils.importlib import import_module from compressor.conf import settings from compressor.storage import default_storage from compressor.utils import get_mod_func +from compressor.utils.compat import force_bytes _cachekey_func = None def get_hexdigest(plaintext, length=None): - digest = hashlib.md5(smart_str(plaintext)).hexdigest() + digest = hashlib.md5(force_bytes(plaintext)).hexdigest() if length: return digest[:length] return digest def simple_cachekey(key): - return 'django_compressor.%s' % smart_str(key) + return 'django_compressor.%s' % force_str(key) def socket_cachekey(key): - return "django_compressor.%s.%s" % (socket.gethostname(), smart_str(key)) + return "django_compressor.%s.%s" % (socket.gethostname(), force_str(key)) def get_cachekey(*args, **kwargs): @@ -39,7 +40,7 @@ def get_cachekey(*args, **kwargs): mod_name, func_name = get_mod_func( settings.COMPRESS_CACHE_KEY_FUNCTION) _cachekey_func = getattr(import_module(mod_name), func_name) - except (AttributeError, ImportError), e: + except (AttributeError, ImportError) as e: raise ImportError("Couldn't import cache key function %s: %s" % (settings.COMPRESS_CACHE_KEY_FUNCTION, e)) return _cachekey_func(*args, **kwargs) @@ -70,7 +71,8 @@ def get_offline_manifest(): if _offline_manifest is None: filename = get_offline_manifest_filename() if default_storage.exists(filename): - _offline_manifest = simplejson.load(default_storage.open(filename)) + with default_storage.open(filename) as fp: + _offline_manifest = json.loads(fp.read().decode('utf8')) else: _offline_manifest = {} return _offline_manifest @@ -83,8 +85,8 @@ def flush_offline_manifest(): def write_offline_manifest(manifest): filename = get_offline_manifest_filename() - default_storage.save(filename, - ContentFile(simplejson.dumps(manifest, indent=2))) + content = json.dumps(manifest, indent=2).encode('utf8') + default_storage.save(filename, ContentFile(content)) flush_offline_manifest() @@ -118,12 +120,10 @@ def get_hashed_content(filename, length=12): filename = os.path.realpath(filename) except OSError: return None - hash_file = open(filename) - try: - content = hash_file.read() - finally: - hash_file.close() - return get_hexdigest(content, length) + + # should we make sure that file is utf-8 encoded? + with open(filename, 'rb') as file: + return get_hexdigest(file.read(), length) def cache_get(key): diff --git a/compressor/conf.py b/compressor/conf.py index 5ba7bee..63bf86b 100644 --- a/compressor/conf.py +++ b/compressor/conf.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import os from django.conf import settings from django.core.exceptions import ImproperlyConfigured diff --git a/compressor/contrib/jinja2ext.py b/compressor/contrib/jinja2ext.py index baf76d5..6c2f792 100644 --- a/compressor/contrib/jinja2ext.py +++ b/compressor/contrib/jinja2ext.py @@ -10,7 +10,7 @@ class CompressorExtension(CompressorMixin, Extension): tags = set(['compress']) def parse(self, parser): - lineno = parser.stream.next().lineno + lineno = next(parser.stream).lineno kindarg = parser.parse_expression() # Allow kind to be defined as jinja2 name node if isinstance(kindarg, nodes.Name): diff --git a/compressor/filters/base.py b/compressor/filters/base.py index 641cf6b..a593cdd 100644 --- a/compressor/filters/base.py +++ b/compressor/filters/base.py @@ -1,7 +1,9 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals +import io import logging import subprocess +import six from django.core.exceptions import ImproperlyConfigured from django.core.files.temp import NamedTemporaryFile from django.utils.importlib import import_module @@ -9,13 +11,18 @@ from django.utils.importlib import import_module from compressor.conf import settings from compressor.exceptions import FilterError from compressor.utils import get_mod_func -from compressor.utils.stringformat import FormattableString as fstr + logger = logging.getLogger("compressor.filters") class FilterBase(object): + """ + A base class for filters that does nothing. + Subclasses should implement `input` and/or `output` methods which must + return a string (unicode under python 2) or raise a NotImplementedError. + """ def __init__(self, content, filter_type=None, filename=None, verbose=0): self.type = filter_type self.content = content @@ -31,6 +38,16 @@ class FilterBase(object): class CallbackOutputFilter(FilterBase): + """ + A filter which takes function path in `callback` attribute, imports it + and uses that function to filter output string:: + + class MyFilter(CallbackOutputFilter): + callback = 'path.to.my.callback' + + Callback should be a function which takes a string as first argument and + returns a string (unicode under python 2). + """ callback = None args = [] kwargs = {} @@ -39,12 +56,13 @@ class CallbackOutputFilter(FilterBase): def __init__(self, *args, **kwargs): super(CallbackOutputFilter, self).__init__(*args, **kwargs) if self.callback is None: - raise ImproperlyConfigured("The callback filter %s must define" - "a 'callback' attribute." % self) + raise ImproperlyConfigured( + "The callback filter %s must define a 'callback' attribute." % + self.__class__.__name__) try: mod_name, func_name = get_mod_func(self.callback) func = getattr(import_module(mod_name), func_name) - except ImportError, e: + except ImportError: if self.dependencies: if len(self.dependencies) == 1: warning = "dependency (%s) is" % self.dependencies[0] @@ -53,17 +71,19 @@ class CallbackOutputFilter(FilterBase): ", ".join([dep for dep in self.dependencies])) else: warning = "" - raise ImproperlyConfigured("The callback %s couldn't be imported. " - "Make sure the %s correctly installed." - % (self.callback, warning)) - except AttributeError, e: - raise ImproperlyConfigured("An error occured while importing the " + raise ImproperlyConfigured( + "The callback %s couldn't be imported. Make sure the %s " + "correctly installed." % (self.callback, warning)) + except AttributeError as e: + raise ImproperlyConfigured("An error occurred while importing the " "callback filter %s: %s" % (self, e)) else: self._callback_func = func def output(self, **kwargs): - return self._callback_func(self.content, *self.args, **self.kwargs) + ret = self._callback_func(self.content, *self.args, **self.kwargs) + assert isinstance(ret, six.text_type) + return ret class CompilerFilter(FilterBase): @@ -73,71 +93,87 @@ class CompilerFilter(FilterBase): """ command = None options = () + encoding = 'utf8' def __init__(self, content, command=None, *args, **kwargs): super(CompilerFilter, self).__init__(content, *args, **kwargs) self.cwd = None + if command: self.command = command if self.command is None: raise FilterError("Required attribute 'command' not given") + if isinstance(self.options, dict): + # turn dict into a tuple new_options = () - for item in kwargs.iteritems(): + for item in kwargs.items(): new_options += (item,) self.options = new_options - for item in kwargs.iteritems(): + + # append kwargs to self.options + for item in kwargs.items(): self.options += (item,) - self.stdout = subprocess.PIPE - self.stdin = subprocess.PIPE - self.stderr = subprocess.PIPE - self.infile, self.outfile = None, None + + self.stdout = self.stdin = self.stderr = subprocess.PIPE + self.infile = self.outfile = None def input(self, **kwargs): + encoding = self.encoding options = dict(self.options) - if self.infile is None: - if "{infile}" in self.command: - if self.filename is None: - self.infile = NamedTemporaryFile(mode="w") - self.infile.write(self.content.encode('utf8')) - self.infile.flush() - options["infile"] = self.infile.name - else: - self.infile = open(self.filename) - options["infile"] = self.filename + + if self.infile is None and "{infile}" in self.command: + # create temporary input file if needed + if self.filename is None: + self.infile = NamedTemporaryFile(mode='wb') + self.infile.write(self.content.encode(self.encoding)) + self.infile.flush() + options["infile"] = self.infile.name + else: + self.infile = open(self.filename) + options["infile"] = self.filename if "{outfile}" in self.command and not "outfile" in options: + # create temporary output file if needed ext = self.type and ".%s" % self.type or "" self.outfile = NamedTemporaryFile(mode='r+', suffix=ext) options["outfile"] = self.outfile.name + try: - command = fstr(self.command).format(**options) - proc = subprocess.Popen(command, shell=True, cwd=self.cwd, - stdout=self.stdout, stdin=self.stdin, stderr=self.stderr) + command = self.command.format(**options) + proc = subprocess.Popen( + command, shell=True, cwd=self.cwd, stdout=self.stdout, + stdin=self.stdin, stderr=self.stderr) if self.infile is None: - filtered, err = proc.communicate(self.content.encode('utf8')) + # if infile is None then send content to process' stdin + filtered, err = proc.communicate( + self.content.encode(encoding)) else: filtered, err = proc.communicate() - except (IOError, OSError), e: + filtered, err = filtered.decode(encoding), err.decode(encoding) + except (IOError, OSError) as e: raise FilterError('Unable to apply %s (%r): %s' % (self.__class__.__name__, self.command, e)) else: if proc.wait() != 0: + # command failed, raise FilterError exception if not err: err = ('Unable to apply %s (%s)' % (self.__class__.__name__, self.command)) if filtered: err += '\n%s' % filtered raise FilterError(err) + if self.verbose: self.logger.debug(err) + outfile_path = options.get('outfile') if outfile_path: - self.outfile = open(outfile_path, 'r') + with io.open(outfile_path, 'r', encoding=encoding) as file: + filtered = file.read() finally: if self.infile is not None: self.infile.close() if self.outfile is not None: - filtered = self.outfile.read() self.outfile.close() return filtered diff --git a/compressor/filters/cssmin/cssmin.py b/compressor/filters/cssmin/cssmin.py index 3dc0cc7..155a32d 100644 --- a/compressor/filters/cssmin/cssmin.py +++ b/compressor/filters/cssmin/cssmin.py @@ -28,13 +28,15 @@ # """`cssmin` - A Python port of the YUI CSS compressor.""" +import re try: from cStringIO import StringIO except ImportError: - from StringIO import StringIO # noqa -import re - + try: + from StringIO import StringIO + except ImportError: + from io import StringIO # python 3 __version__ = '0.1.4' diff --git a/compressor/filters/datauri.py b/compressor/filters/datauri.py index 29ae40f..cf824ea 100644 --- a/compressor/filters/datauri.py +++ b/compressor/filters/datauri.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import os import re import mimetypes @@ -39,7 +40,8 @@ class DataUriFilter(FilterBase): if not url.startswith('data:'): path = self.get_file_path(url) if os.stat(path).st_size <= settings.COMPRESS_DATA_URI_MAX_SIZE: - data = b64encode(open(path, 'rb').read()) + with open(path, 'rb') as file: + data = b64encode(file.read()).decode('ascii') return 'url("data:%s;base64,%s")' % ( mimetypes.guess_type(path)[0], data) return 'url("%s")' % url diff --git a/compressor/management/commands/compress.py b/compressor/management/commands/compress.py index 1ae0778..3795713 100644 --- a/compressor/management/commands/compress.py +++ b/compressor/management/commands/compress.py @@ -1,15 +1,11 @@ # flake8: noqa +import io import os import sys from types import MethodType from fnmatch import fnmatch from optparse import make_option -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO # noqa - from django.core.management.base import NoArgsCommand, CommandError from django.template import (Context, Template, TemplateDoesNotExist, TemplateSyntaxError) @@ -29,6 +25,7 @@ from compressor.cache import get_offline_hexdigest, write_offline_manifest from compressor.conf import settings from compressor.exceptions import OfflineGenerationError from compressor.templatetags.compress import CompressorNode +from compressor.utils.compat import StringIO def patched_render(self, context): @@ -213,17 +210,13 @@ class Command(NoArgsCommand): compressor_nodes = SortedDict() for template_name in templates: try: - template_file = open(template_name) - try: - template = Template(template_file.read().decode( - settings.FILE_CHARSET)) - finally: - template_file.close() + with io.open(template_name, encoding=settings.FILE_CHARSET) as file: + template = Template(file.read()) except IOError: # unreadable file -> ignore if verbosity > 0: log.write("Unreadable template at: %s\n" % template_name) continue - except TemplateSyntaxError, e: # broken template -> ignore + except TemplateSyntaxError as e: # broken template -> ignore if verbosity > 0: log.write("Invalid template %s: %s\n" % (template_name, e)) continue @@ -255,7 +248,7 @@ class Command(NoArgsCommand): count = 0 results = [] offline_manifest = SortedDict() - for template, nodes in compressor_nodes.iteritems(): + for template, nodes in compressor_nodes.items(): context = Context(settings.COMPRESS_OFFLINE_CONTEXT) template._log = log template._log_verbosity = verbosity @@ -275,7 +268,7 @@ class Command(NoArgsCommand): key = get_offline_hexdigest(node.nodelist.render(context)) try: result = node.render(context, forced=True) - except Exception, e: + except Exception as e: raise CommandError("An error occured during rendering %s: " "%s" % (template.template_name, e)) offline_manifest[key] = result @@ -331,7 +324,7 @@ class Command(NoArgsCommand): if not settings.COMPRESS_ENABLED and not options.get("force"): raise CommandError( "Compressor is disabled. Set the COMPRESS_ENABLED " - "settting or use --force to override.") + "setting or use --force to override.") if not settings.COMPRESS_OFFLINE: if not options.get("force"): raise CommandError( diff --git a/compressor/parser/__init__.py b/compressor/parser/__init__.py index bc8c18c..8da936b 100644 --- a/compressor/parser/__init__.py +++ b/compressor/parser/__init__.py @@ -1,3 +1,4 @@ +import six from django.utils.functional import LazyObject from django.utils.importlib import import_module @@ -11,8 +12,9 @@ from compressor.parser.html5lib import Html5LibParser # noqa class AutoSelectParser(LazyObject): options = ( - ('lxml.html', LxmlParser), # lxml, extremely fast - ('HTMLParser', HtmlParser), # fast and part of the Python stdlib + # TODO: make lxml.html parser first again + (six.moves.html_parser.__name__, HtmlParser), # fast and part of the Python stdlib + ('lxml.html', LxmlParser), # lxml, extremely fast ) def __init__(self, content): diff --git a/compressor/parser/beautifulsoup.py b/compressor/parser/beautifulsoup.py index 498cde8..2132868 100644 --- a/compressor/parser/beautifulsoup.py +++ b/compressor/parser/beautifulsoup.py @@ -1,10 +1,10 @@ from __future__ import absolute_import from django.core.exceptions import ImproperlyConfigured -from django.utils.encoding import smart_unicode from compressor.exceptions import ParserError from compressor.parser import ParserBase from compressor.utils.decorators import cached_property +from compressor.utils.compat import smart_text class BeautifulSoupParser(ParserBase): @@ -14,9 +14,9 @@ class BeautifulSoupParser(ParserBase): try: from BeautifulSoup import BeautifulSoup return BeautifulSoup(self.content) - except ImportError, err: + except ImportError as err: raise ImproperlyConfigured("Error while importing BeautifulSoup: %s" % err) - except Exception, err: + except Exception as err: raise ParserError("Error while initializing Parser: %s" % err) def css_elems(self): @@ -35,4 +35,4 @@ class BeautifulSoupParser(ParserBase): return elem.name def elem_str(self, elem): - return smart_unicode(elem) + return smart_text(elem) diff --git a/compressor/parser/default_htmlparser.py b/compressor/parser/default_htmlparser.py index 8425d77..db16c99 100644 --- a/compressor/parser/default_htmlparser.py +++ b/compressor/parser/default_htmlparser.py @@ -1,13 +1,12 @@ -from HTMLParser import HTMLParser -from django.utils.encoding import smart_unicode +import six from compressor.exceptions import ParserError from compressor.parser import ParserBase +from compressor.utils.compat import smart_text -class DefaultHtmlParser(ParserBase, HTMLParser): - +class DefaultHtmlParser(ParserBase, six.moves.html_parser.HTMLParser): def __init__(self, content): - HTMLParser.__init__(self) + six.moves.html_parser.HTMLParser.__init__(self) self.content = content self._css_elems = [] self._js_elems = [] @@ -15,7 +14,7 @@ class DefaultHtmlParser(ParserBase, HTMLParser): try: self.feed(self.content) self.close() - except Exception, err: + except Exception as err: lineno = err.lineno line = self.content.splitlines()[lineno] raise ParserError("Error while initializing HtmlParser: %s (line: %s)" % (err, repr(line))) @@ -65,7 +64,7 @@ class DefaultHtmlParser(ParserBase, HTMLParser): return elem['attrs_dict'] def elem_content(self, elem): - return smart_unicode(elem['text']) + return smart_text(elem['text']) def elem_str(self, elem): tag = {} diff --git a/compressor/parser/html5lib.py b/compressor/parser/html5lib.py index 7fee590..c1a4a72 100644 --- a/compressor/parser/html5lib.py +++ b/compressor/parser/html5lib.py @@ -1,10 +1,10 @@ from __future__ import absolute_import -from django.utils.encoding import smart_unicode from django.core.exceptions import ImproperlyConfigured from compressor.exceptions import ParserError from compressor.parser import ParserBase from compressor.utils.decorators import cached_property +from compressor.utils.compat import smart_text class Html5LibParser(ParserBase): @@ -29,9 +29,9 @@ class Html5LibParser(ParserBase): def html(self): try: return self.html5lib.parseFragment(self.content) - except ImportError, err: + except ImportError as err: raise ImproperlyConfigured("Error while importing html5lib: %s" % err) - except Exception, err: + except Exception as err: raise ParserError("Error while initializing Parser: %s" % err) def css_elems(self): @@ -53,4 +53,4 @@ class Html5LibParser(ParserBase): # This method serializes HTML in a way that does not pass all tests. # However, this method is only called in tests anyway, so it doesn't # really matter. - return smart_unicode(self._serialize(elem)) + return smart_text(self._serialize(elem)) diff --git a/compressor/parser/lxml.py b/compressor/parser/lxml.py index 7bbb561..48cb50d 100644 --- a/compressor/parser/lxml.py +++ b/compressor/parser/lxml.py @@ -1,10 +1,10 @@ from __future__ import absolute_import from django.core.exceptions import ImproperlyConfigured -from django.utils.encoding import smart_unicode from compressor.exceptions import ParserError from compressor.parser import ParserBase from compressor.utils.decorators import cached_property +from compressor.utils.compat import smart_text class LxmlParser(ParserBase): @@ -16,9 +16,9 @@ class LxmlParser(ParserBase): self.fromstring = fromstring self.soupparser = soupparser self.tostring = tostring - except ImportError, err: + except ImportError as err: raise ImproperlyConfigured("Error while importing lxml: %s" % err) - except Exception, err: + except Exception as err: raise ParserError("Error while initializing Parser: %s" % err) super(LxmlParser, self).__init__(content) @@ -43,13 +43,13 @@ class LxmlParser(ParserBase): return elem.attrib def elem_content(self, elem): - return smart_unicode(elem.text) + return smart_text(elem.text) def elem_name(self, elem): return elem.tag def elem_str(self, elem): - elem_as_string = smart_unicode( + elem_as_string = smart_text( self.tostring(elem, method='html', encoding=unicode)) if elem.tag == 'link': # This makes testcases happy diff --git a/compressor/storage.py b/compressor/storage.py index 757b5a6..0e55639 100644 --- a/compressor/storage.py +++ b/compressor/storage.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import errno import gzip import os @@ -50,7 +51,7 @@ class CompressorFileStorage(FileSystemStorage): """ try: super(CompressorFileStorage, self).delete(name) - except OSError, e: + except OSError as e: if e.errno != errno.ENOENT: raise @@ -69,7 +70,7 @@ class GzipCompressorFileStorage(CompressorFileStorage): # workaround for http://bugs.python.org/issue13664 name = os.path.basename(filename).encode('latin1', errors='replace') - out = gzip.GzipFile(name, fileobj=open(filename + ".gz", 'wb')) + out = gzip.GzipFile(name, fileobj=open("%s.gz" % filename, 'wb')) out.writelines(open(self.path(filename), 'rb')) out.close() return filename diff --git a/compressor/templatetags/compress.py b/compressor/templatetags/compress.py index 870668a..bd53f85 100644 --- a/compressor/templatetags/compress.py +++ b/compressor/templatetags/compress.py @@ -1,5 +1,6 @@ from django import template from django.core.exceptions import ImproperlyConfigured +import six from compressor.cache import (cache_get, cache_set, get_offline_hexdigest, get_offline_manifest, get_templatetag_cachekey) @@ -50,7 +51,7 @@ class CompressorMixin(object): Check if offline compression is enabled or forced Defaults to just checking the settings and forced argument, - but can be overriden to completely disable compression for + but can be overridden to completely disable compression for a subclass, for instance. """ return (settings.COMPRESS_ENABLED and @@ -107,6 +108,7 @@ class CompressorMixin(object): rendered_output = self.render_output(compressor, mode, forced=forced) if cache_key: cache_set(cache_key, rendered_output) + assert isinstance(rendered_output, six.text_type) return rendered_output except Exception: if settings.DEBUG or forced: diff --git a/compressor/tests/precompiler.py b/compressor/tests/precompiler.py index 4c01964..059a322 100644 --- a/compressor/tests/precompiler.py +++ b/compressor/tests/precompiler.py @@ -28,7 +28,7 @@ def main(): with open(options.outfile, 'w') as f: f.write(content) else: - print content + print(content) if __name__ == '__main__': diff --git a/compressor/tests/test_base.py b/compressor/tests/test_base.py index 8678e32..48dbb3e 100644 --- a/compressor/tests/test_base.py +++ b/compressor/tests/test_base.py @@ -1,8 +1,8 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals import os import re -from BeautifulSoup import BeautifulSoup +from bs4 import BeautifulSoup from django.core.cache.backends import locmem from django.test import TestCase @@ -14,9 +14,14 @@ from compressor.js import JsCompressor from compressor.exceptions import FilterDoesNotExist +def make_soup(markup): + # we use html.parser instead of lxml because it doesn't work on python 3.3 + return BeautifulSoup(markup, 'html.parser') + + def css_tag(href, **kwargs): rendered_attrs = ''.join(['%s="%s" ' % (k, v) for k, v in kwargs.items()]) - template = u'' + template = '' return template % (href, rendered_attrs) @@ -51,20 +56,34 @@ class CompressorTestCase(TestCase): def test_css_split(self): out = [ - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css', u'one.css'), u'css/one.css', u''), - (SOURCE_HUNK, u'p { border:5px solid green;}', None, u''), - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css', u'two.css'), u'css/two.css', u''), + ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'css', 'one.css'), + 'css/one.css', '', + ), + ( + SOURCE_HUNK, + 'p { border:5px solid green;}', + None, + '', + ), + ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'css', 'two.css'), + 'css/two.css', + '', + ), ] split = self.css_node.split_contents() split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3])) for x in split] self.assertEqual(out, split) def test_css_hunks(self): - out = ['body { background:#990; }', u'p { border:5px solid green;}', 'body { color:#fff; }'] + out = ['body { background:#990; }', 'p { border:5px solid green;}', 'body { color:#fff; }'] self.assertEqual(out, list(self.css_node.hunks())) def test_css_output(self): - out = u'body { background:#990; }\np { border:5px solid green;}\nbody { color:#fff; }' + out = 'body { background:#990; }\np { border:5px solid green;}\nbody { color:#fff; }' hunks = '\n'.join([h for h in self.css_node.hunks()]) self.assertEqual(out, hunks) @@ -89,28 +108,38 @@ class CompressorTestCase(TestCase): def test_js_split(self): out = [ - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'js', u'one.js'), u'js/one.js', ''), - (SOURCE_HUNK, u'obj.value = "value";', None, ''), + ( + SOURCE_FILE, + os.path.join(settings.COMPRESS_ROOT, 'js', 'one.js'), + 'js/one.js', + '', + ), + ( + SOURCE_HUNK, + 'obj.value = "value";', + None, + '', + ), ] split = self.js_node.split_contents() split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3])) for x in split] self.assertEqual(out, split) def test_js_hunks(self): - out = ['obj = {};', u'obj.value = "value";'] + out = ['obj = {};', 'obj.value = "value";'] self.assertEqual(out, list(self.js_node.hunks())) def test_js_output(self): - out = u'' + out = '' self.assertEqual(out, self.js_node.output()) def test_js_override_url(self): - self.js_node.context.update({'url': u'This is not a url, just a text'}) - out = u'' + self.js_node.context.update({'url': 'This is not a url, just a text'}) + out = '' self.assertEqual(out, self.js_node.output()) def test_css_override_url(self): - self.css_node.context.update({'url': u'This is not a url, just a text'}) + self.css_node.context.update({'url': 'This is not a url, just a text'}) output = css_tag('/static/CACHE/css/e41ba2cc6982.css') self.assertEqual(output, self.css_node.output().strip()) @@ -126,20 +155,20 @@ class CompressorTestCase(TestCase): settings.COMPRESS_PRECOMPILERS = precompilers def test_js_return_if_on(self): - output = u'' + output = '' self.assertEqual(output, self.js_node.output()) def test_custom_output_dir(self): try: old_output_dir = settings.COMPRESS_OUTPUT_DIR settings.COMPRESS_OUTPUT_DIR = 'custom' - output = u'' + output = '' self.assertEqual(output, JsCompressor(self.js).output()) settings.COMPRESS_OUTPUT_DIR = '' - output = u'' + output = '' self.assertEqual(output, JsCompressor(self.js).output()) settings.COMPRESS_OUTPUT_DIR = '/custom/nested/' - output = u'' + output = '' self.assertEqual(output, JsCompressor(self.js).output()) finally: settings.COMPRESS_OUTPUT_DIR = old_output_dir @@ -153,7 +182,7 @@ class CompressorTestCase(TestCase): ) css = '' css_node = CssCompressor(css) - output = BeautifulSoup(css_node.output('inline')) + output = make_soup(css_node.output('inline')) self.assertEqual(output.text, 'OUTPUT') finally: settings.COMPRESS_PRECOMPILERS = original_precompilers @@ -182,16 +211,16 @@ class CssMediaTestCase(TestCase): def test_css_output(self): css_node = CssCompressor(self.css) - links = BeautifulSoup(css_node.output()).findAll('link') - media = [u'screen', u'print', u'all', None] + links = make_soup(css_node.output()).findAll('link') + media = ['screen', 'print', 'all', None] self.assertEqual(len(links), 4) self.assertEqual(media, [l.get('media', None) for l in links]) def test_avoid_reordering_css(self): css = self.css + '' css_node = CssCompressor(css) - media = [u'screen', u'print', u'all', None, u'print'] - links = BeautifulSoup(css_node.output()).findAll('link') + media = ['screen', 'print', 'all', None, 'print'] + links = make_soup(css_node.output()).findAll('link') self.assertEqual(media, [l.get('media', None) for l in links]) def test_passthough_when_compress_disabled(self): @@ -205,10 +234,10 @@ class CssMediaTestCase(TestCase): """ css_node = CssCompressor(css) - output = BeautifulSoup(css_node.output()).findAll(['link', 'style']) - self.assertEqual([u'/static/css/one.css', u'/static/css/two.css', None], + output = make_soup(css_node.output()).findAll(['link', 'style']) + self.assertEqual(['/static/css/one.css', '/static/css/two.css', None], [l.get('href', None) for l in output]) - self.assertEqual([u'screen', u'screen', u'screen'], + self.assertEqual(['screen', 'screen', 'screen'], [l.get('media', None) for l in output]) settings.COMPRESS_PRECOMPILERS = original_precompilers diff --git a/compressor/tests/test_filters.py b/compressor/tests/test_filters.py index 90c4036..1618a01 100644 --- a/compressor/tests/test_filters.py +++ b/compressor/tests/test_filters.py @@ -1,14 +1,17 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals +import io import os import sys -from unittest2 import skipIf +import textwrap +import six from django.test import TestCase from compressor.cache import get_hashed_mtime, get_hashed_content from compressor.conf import settings from compressor.css import CssCompressor from compressor.utils import find_command +from compressor.utils.compat import unittest as ut2 from compressor.filters.base import CompilerFilter from compressor.filters.cssmin import CSSMinFilter from compressor.filters.css_default import CssAbsoluteFilter @@ -16,56 +19,54 @@ from compressor.filters.template import TemplateFilter from compressor.tests.test_base import test_dir +@ut2.skipIf(find_command(settings.COMPRESS_CSSTIDY_BINARY) is None, + 'CSStidy binary %r not found' % settings.COMPRESS_CSSTIDY_BINARY) class CssTidyTestCase(TestCase): def test_tidy(self): - content = """ -/* Some comment */ -font,th,td,p{ -color: black; -} -""" + content = textwrap.dedent("""\ + /* Some comment */ + font,th,td,p{ + color: black; + } + """) from compressor.filters.csstidy import CSSTidyFilter + ret = CSSTidyFilter(content).input() + self.assertIsInstance(ret, six.text_type) self.assertEqual( "font,th,td,p{color:#000;}", CSSTidyFilter(content).input()) -CssTidyTestCase = skipIf( - find_command(settings.COMPRESS_CSSTIDY_BINARY) is None, - 'CSStidy binary %r not found' % settings.COMPRESS_CSSTIDY_BINARY, -)(CssTidyTestCase) - class PrecompilerTestCase(TestCase): - def setUp(self): self.filename = os.path.join(test_dir, 'static/css/one.css') - with open(self.filename) as f: - self.content = f.read() + with io.open(self.filename, encoding=settings.FILE_CHARSET) as file: + self.content = file.read() self.test_precompiler = os.path.join(test_dir, 'precompiler.py') def test_precompiler_infile_outfile(self): command = '%s %s -f {infile} -o {outfile}' % (sys.executable, self.test_precompiler) compiler = CompilerFilter(content=self.content, filename=self.filename, command=command) - self.assertEqual(u"body { color:#990; }", compiler.input()) + self.assertEqual("body { color:#990; }", compiler.input()) def test_precompiler_infile_stdout(self): command = '%s %s -f {infile}' % (sys.executable, self.test_precompiler) compiler = CompilerFilter(content=self.content, filename=None, command=command) - self.assertEqual(u"body { color:#990; }%s" % os.linesep, compiler.input()) + self.assertEqual("body { color:#990; }%s" % os.linesep, compiler.input()) def test_precompiler_stdin_outfile(self): command = '%s %s -o {outfile}' % (sys.executable, self.test_precompiler) compiler = CompilerFilter(content=self.content, filename=None, command=command) - self.assertEqual(u"body { color:#990; }", compiler.input()) + self.assertEqual("body { color:#990; }", compiler.input()) def test_precompiler_stdin_stdout(self): command = '%s %s' % (sys.executable, self.test_precompiler) compiler = CompilerFilter(content=self.content, filename=None, command=command) - self.assertEqual(u"body { color:#990; }%s" % os.linesep, compiler.input()) + self.assertEqual("body { color:#990; }%s" % os.linesep, compiler.input()) def test_precompiler_stdin_stdout_filename(self): command = '%s %s' % (sys.executable, self.test_precompiler) compiler = CompilerFilter(content=self.content, filename=self.filename, command=command) - self.assertEqual(u"body { color:#990; }%s" % os.linesep, compiler.input()) + self.assertEqual("body { color:#990; }%s" % os.linesep, compiler.input()) class CssMinTestCase(TestCase): @@ -77,7 +78,7 @@ class CssMinTestCase(TestCase): } -""" + """ output = "p{background:#369 url('../../images/image.gif')}" self.assertEqual(output, CSSMinFilter(content).output()) @@ -210,14 +211,14 @@ class CssAbsolutizingTestCase(TestCase): 'hash1': self.hashing_func(os.path.join(settings.COMPRESS_ROOT, 'img/python.png')), 'hash2': self.hashing_func(os.path.join(settings.COMPRESS_ROOT, 'img/add.png')), } - self.assertEqual([u"""\ + self.assertEqual(["""\ p { background: url('/static/img/python.png?%(hash1)s'); } p { background: url('/static/img/python.png?%(hash1)s'); } p { background: url('/static/img/python.png?%(hash1)s'); } p { background: url('/static/img/python.png?%(hash1)s'); } p { filter: progid:DXImageTransform.Microsoft.AlphaImageLoader(src='/static/img/python.png?%(hash1)s'); } """ % hash_dict, - u"""\ + """\ p { background: url('/static/img/add.png?%(hash2)s'); } p { background: url('/static/img/add.png?%(hash2)s'); } p { background: url('/static/img/add.png?%(hash2)s'); } @@ -264,7 +265,7 @@ class CssDataUriTestCase(TestCase): def test_data_uris(self): datauri_hash = get_hashed_mtime(os.path.join(settings.COMPRESS_ROOT, 'img/python.png')) - out = [u'''.add { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAJvSURBVDjLpZPrS5NhGIf9W7YvBYOkhlkoqCklWChv2WyKik7blnNris72bi6dus0DLZ0TDxW1odtopDs4D8MDZuLU0kXq61CijSIIasOvv94VTUfLiB74fXngup7nvrnvJABJ/5PfLnTTdcwOj4RsdYmo5glBWP6iOtzwvIKSWstI0Wgx80SBblpKtE9KQs/We7EaWoT/8wbWP61gMmCH0lMDvokT4j25TiQU/ITFkek9Ow6+7WH2gwsmahCPdwyw75uw9HEO2gUZSkfyI9zBPCJOoJ2SMmg46N61YO/rNoa39Xi41oFuXysMfh36/Fp0b7bAfWAH6RGi0HglWNCbzYgJaFjRv6zGuy+b9It96N3SQvNKiV9HvSaDfFEIxXItnPs23BzJQd6DDEVM0OKsoVwBG/1VMzpXVWhbkUM2K4oJBDYuGmbKIJ0qxsAbHfRLzbjcnUbFBIpx/qH3vQv9b3U03IQ/HfFkERTzfFj8w8jSpR7GBE123uFEYAzaDRIqX/2JAtJbDat/COkd7CNBva2cMvq0MGxp0PRSCPF8BXjWG3FgNHc9XPT71Ojy3sMFdfJRCeKxEsVtKwFHwALZfCUk3tIfNR8XiJwc1LmL4dg141JPKtj3WUdNFJqLGFVPC4OkR4BxajTWsChY64wmCnMxsWPCHcutKBxMVp5mxA1S+aMComToaqTRUQknLTH62kHOVEE+VQnjahscNCy0cMBWsSI0TCQcZc5ALkEYckL5A5noWSBhfm2AecMAjbcRWV0pUTh0HE64TNf0mczcnnQyu/MilaFJCae1nw2fbz1DnVOxyGTlKeZft/Ff8x1BRssfACjTwQAAAABJRU5ErkJggg=="); } + out = ['''.add { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAJvSURBVDjLpZPrS5NhGIf9W7YvBYOkhlkoqCklWChv2WyKik7blnNris72bi6dus0DLZ0TDxW1odtopDs4D8MDZuLU0kXq61CijSIIasOvv94VTUfLiB74fXngup7nvrnvJABJ/5PfLnTTdcwOj4RsdYmo5glBWP6iOtzwvIKSWstI0Wgx80SBblpKtE9KQs/We7EaWoT/8wbWP61gMmCH0lMDvokT4j25TiQU/ITFkek9Ow6+7WH2gwsmahCPdwyw75uw9HEO2gUZSkfyI9zBPCJOoJ2SMmg46N61YO/rNoa39Xi41oFuXysMfh36/Fp0b7bAfWAH6RGi0HglWNCbzYgJaFjRv6zGuy+b9It96N3SQvNKiV9HvSaDfFEIxXItnPs23BzJQd6DDEVM0OKsoVwBG/1VMzpXVWhbkUM2K4oJBDYuGmbKIJ0qxsAbHfRLzbjcnUbFBIpx/qH3vQv9b3U03IQ/HfFkERTzfFj8w8jSpR7GBE123uFEYAzaDRIqX/2JAtJbDat/COkd7CNBva2cMvq0MGxp0PRSCPF8BXjWG3FgNHc9XPT71Ojy3sMFdfJRCeKxEsVtKwFHwALZfCUk3tIfNR8XiJwc1LmL4dg141JPKtj3WUdNFJqLGFVPC4OkR4BxajTWsChY64wmCnMxsWPCHcutKBxMVp5mxA1S+aMComToaqTRUQknLTH62kHOVEE+VQnjahscNCy0cMBWsSI0TCQcZc5ALkEYckL5A5noWSBhfm2AecMAjbcRWV0pUTh0HE64TNf0mczcnnQyu/MilaFJCae1nw2fbz1DnVOxyGTlKeZft/Ff8x1BRssfACjTwQAAAABJRU5ErkJggg=="); } .add-with-hash { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAJvSURBVDjLpZPrS5NhGIf9W7YvBYOkhlkoqCklWChv2WyKik7blnNris72bi6dus0DLZ0TDxW1odtopDs4D8MDZuLU0kXq61CijSIIasOvv94VTUfLiB74fXngup7nvrnvJABJ/5PfLnTTdcwOj4RsdYmo5glBWP6iOtzwvIKSWstI0Wgx80SBblpKtE9KQs/We7EaWoT/8wbWP61gMmCH0lMDvokT4j25TiQU/ITFkek9Ow6+7WH2gwsmahCPdwyw75uw9HEO2gUZSkfyI9zBPCJOoJ2SMmg46N61YO/rNoa39Xi41oFuXysMfh36/Fp0b7bAfWAH6RGi0HglWNCbzYgJaFjRv6zGuy+b9It96N3SQvNKiV9HvSaDfFEIxXItnPs23BzJQd6DDEVM0OKsoVwBG/1VMzpXVWhbkUM2K4oJBDYuGmbKIJ0qxsAbHfRLzbjcnUbFBIpx/qH3vQv9b3U03IQ/HfFkERTzfFj8w8jSpR7GBE123uFEYAzaDRIqX/2JAtJbDat/COkd7CNBva2cMvq0MGxp0PRSCPF8BXjWG3FgNHc9XPT71Ojy3sMFdfJRCeKxEsVtKwFHwALZfCUk3tIfNR8XiJwc1LmL4dg141JPKtj3WUdNFJqLGFVPC4OkR4BxajTWsChY64wmCnMxsWPCHcutKBxMVp5mxA1S+aMComToaqTRUQknLTH62kHOVEE+VQnjahscNCy0cMBWsSI0TCQcZc5ALkEYckL5A5noWSBhfm2AecMAjbcRWV0pUTh0HE64TNf0mczcnnQyu/MilaFJCae1nw2fbz1DnVOxyGTlKeZft/Ff8x1BRssfACjTwQAAAABJRU5ErkJggg=="); } .python { background-image: url("/static/img/python.png?%s"); } .datauri { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9YGARc5KB0XV+IAAAAddEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIFRoZSBHSU1Q72QlbgAAAF1JREFUGNO9zL0NglAAxPEfdLTs4BZM4DIO4C7OwQg2JoQ9LE1exdlYvBBeZ7jqch9//q1uH4TLzw4d6+ErXMMcXuHWxId3KOETnnXXV6MJpcq2MLaI97CER3N0 vr4MkhoXe0rZigAAAABJRU5ErkJggg=="); } diff --git a/compressor/tests/test_jinja2ext.py b/compressor/tests/test_jinja2ext.py index cb2e012..916e946 100644 --- a/compressor/tests/test_jinja2ext.py +++ b/compressor/tests/test_jinja2ext.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement +from __future__ import with_statement, unicode_literals from django.test import TestCase @@ -53,13 +53,13 @@ class TestJinja2CompressorExtension(TestCase): settings.COMPRESS_ENABLED = org_COMPRESS_ENABLED def test_empty_tag(self): - template = self.env.from_string(u"""{% compress js %}{% block js %} + template = self.env.from_string("""{% compress js %}{% block js %} {% endblock %}{% endcompress %}""") context = {'STATIC_URL': settings.COMPRESS_URL} - self.assertEqual(u'', template.render(context)) + self.assertEqual('', template.render(context)) def test_css_tag(self): - template = self.env.from_string(u"""{% compress css -%} + template = self.env.from_string("""{% compress css -%} @@ -69,7 +69,7 @@ class TestJinja2CompressorExtension(TestCase): self.assertEqual(out, template.render(context)) def test_nonascii_css_tag(self): - template = self.env.from_string(u"""{% compress css -%} + template = self.env.from_string("""{% compress css -%} {% endcompress %}""") @@ -78,34 +78,34 @@ class TestJinja2CompressorExtension(TestCase): self.assertEqual(out, template.render(context)) def test_js_tag(self): - template = self.env.from_string(u"""{% compress js -%} + template = self.env.from_string("""{% compress js -%} {% endcompress %}""") context = {'STATIC_URL': settings.COMPRESS_URL} - out = u'' + out = '' self.assertEqual(out, template.render(context)) def test_nonascii_js_tag(self): - template = self.env.from_string(u"""{% compress js -%} + template = self.env.from_string("""{% compress js -%} {% endcompress %}""") context = {'STATIC_URL': settings.COMPRESS_URL} - out = u'' + out = '' self.assertEqual(out, template.render(context)) def test_nonascii_latin1_js_tag(self): - template = self.env.from_string(u"""{% compress js -%} + template = self.env.from_string("""{% compress js -%} {% endcompress %}""") context = {'STATIC_URL': settings.COMPRESS_URL} - out = u'' + out = '' self.assertEqual(out, template.render(context)) def test_css_inline(self): - template = self.env.from_string(u"""{% compress css, inline -%} + template = self.env.from_string("""{% compress css, inline -%} {% endcompress %}""") @@ -117,7 +117,7 @@ class TestJinja2CompressorExtension(TestCase): self.assertEqual(out, template.render(context)) def test_js_inline(self): - template = self.env.from_string(u"""{% compress js, inline -%} + template = self.env.from_string("""{% compress js, inline -%} {% endcompress %}""") @@ -128,11 +128,11 @@ class TestJinja2CompressorExtension(TestCase): def test_nonascii_inline_css(self): org_COMPRESS_ENABLED = settings.COMPRESS_ENABLED settings.COMPRESS_ENABLED = False - template = self.env.from_string(u'{% compress css %}' - u'{% endcompress %}') - out = u'' + template = self.env.from_string('{% compress css %}' + '{% endcompress %}') + out = '' settings.COMPRESS_ENABLED = org_COMPRESS_ENABLED context = {'STATIC_URL': settings.COMPRESS_URL} self.assertEqual(out, template.render(context)) diff --git a/compressor/tests/test_offline.py b/compressor/tests/test_offline.py index a988afd..73e8084 100644 --- a/compressor/tests/test_offline.py +++ b/compressor/tests/test_offline.py @@ -1,8 +1,8 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals +import io import os -from StringIO import StringIO -from unittest2 import skipIf +import six import django from django.template import Template, Context from django.test import TestCase @@ -13,6 +13,7 @@ from compressor.conf import settings from compressor.exceptions import OfflineGenerationError from compressor.management.commands.compress import Command as CompressCommand from compressor.storage import default_storage +from compressor.utils.compat import StringIO, unittest as ut2 class OfflineTestCaseMixin(object): @@ -39,14 +40,14 @@ class OfflineTestCaseMixin(object): settings.COMPRESS_ENABLED = True settings.COMPRESS_OFFLINE = True self.template_path = os.path.join(settings.TEMPLATE_DIRS[0], self.template_name) - self.template_file = open(self.template_path) - self.template = Template(self.template_file.read().decode(settings.FILE_CHARSET)) + + with io.open(self.template_path, encoding=settings.FILE_CHARSET) as file: + self.template = Template(file.read()) def tearDown(self): settings.COMPRESS_ENABLED = self._old_compress settings.COMPRESS_OFFLINE = self._old_compress_offline settings.TEMPLATE_DIRS = self._old_template_dirs - self.template_file.close() manifest_path = os.path.join('CACHE', 'manifest.json') if default_storage.exists(manifest_path): default_storage.delete(manifest_path) @@ -55,7 +56,7 @@ class OfflineTestCaseMixin(object): count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity) self.assertEqual(1, count) self.assertEqual([ - u'' % (self.expected_hash, ), + '' % (self.expected_hash, ), ], result) rendered_template = self.template.render(Context(settings.COMPRESS_OFFLINE_CONTEXT)) self.assertEqual(rendered_template, "".join(result) + "\n") @@ -97,8 +98,8 @@ class OfflineGenerationBlockSuperTestCaseWithExtraContent(OfflineTestCaseMixin, count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity) self.assertEqual(2, count) self.assertEqual([ - u'', - u'' + '', + '' ], result) rendered_template = self.template.render(Context(settings.COMPRESS_OFFLINE_CONTEXT)) self.assertEqual(rendered_template, "".join(result) + "\n") @@ -129,7 +130,7 @@ class OfflineGenerationStaticTemplateTagTestCase(OfflineTestCaseMixin, TestCase) templates_dir = "test_static_templatetag" expected_hash = "dfa2bb387fa8" # This test uses {% static %} which was introduced in django 1.4 -OfflineGenerationStaticTemplateTagTestCase = skipIf( +OfflineGenerationStaticTemplateTagTestCase = ut2.skipIf( django.VERSION[1] < 4, 'Django 1.4 not found' )(OfflineGenerationStaticTemplateTagTestCase) @@ -156,8 +157,8 @@ class OfflineGenerationTestCaseErrors(OfflineTestCaseMixin, TestCase): def test_offline(self): count, result = CompressCommand().compress(log=self.log, verbosity=self.verbosity) self.assertEqual(2, count) - self.assertIn(u'', result) - self.assertIn(u'', result) + self.assertIn('', result) + self.assertIn('', result) class OfflineGenerationTestCaseWithError(OfflineTestCaseMixin, TestCase): @@ -209,7 +210,7 @@ class OfflineGenerationTestCase(OfflineTestCaseMixin, TestCase): default_storage.delete(manifest_path) self.assertEqual(1, count) self.assertEqual([ - u'' % (self.expected_hash, ), + '' % (self.expected_hash, ), ], result) rendered_template = self.template.render(Context(settings.COMPRESS_OFFLINE_CONTEXT)) self.assertEqual(rendered_template, "".join(result) + "\n") diff --git a/compressor/tests/test_parsers.py b/compressor/tests/test_parsers.py index 04ec924..3269c17 100644 --- a/compressor/tests/test_parsers.py +++ b/compressor/tests/test_parsers.py @@ -1,6 +1,5 @@ from __future__ import with_statement import os -from unittest2 import skipIf try: import lxml @@ -22,6 +21,7 @@ from compressor.base import SOURCE_HUNK, SOURCE_FILE from compressor.conf import settings from compressor.css import CssCompressor from compressor.tests.test_base import CompressorTestCase +from compressor.utils.compat import unittest as ut2 class ParserTestCase(object): @@ -35,11 +35,12 @@ class ParserTestCase(object): settings.COMPRESS_PARSER = self.old_parser +@ut2.skipIf(lxml is None, 'lxml not found') class LxmlParserTests(ParserTestCase, CompressorTestCase): parser_cls = 'compressor.parser.LxmlParser' -LxmlParserTests = skipIf(lxml is None, 'lxml not found')(LxmlParserTests) +@ut2.skipIf(html5lib is None, 'html5lib not found') class Html5LibParserTests(ParserTestCase, CompressorTestCase): parser_cls = 'compressor.parser.Html5LibParser' @@ -54,9 +55,9 @@ class Html5LibParserTests(ParserTestCase, CompressorTestCase): def test_css_split(self): out = [ - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css', u'one.css'), u'css/one.css', u''), - (SOURCE_HUNK, u'p { border:5px solid green;}', None, u''), - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css', u'two.css'), u'css/two.css', u''), + (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, 'css', 'one.css'), 'css/one.css', ''), + (SOURCE_HUNK, 'p { border:5px solid green;}', None, ''), + (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, 'css', 'two.css'), 'css/two.css', ''), ] split = self.css_node.split_contents() split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3])) for x in split] @@ -64,23 +65,18 @@ class Html5LibParserTests(ParserTestCase, CompressorTestCase): def test_js_split(self): out = [ - (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'js', u'one.js'), u'js/one.js', u''), - (SOURCE_HUNK, u'obj.value = "value";', None, u''), + (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, 'js', 'one.js'), 'js/one.js', ''), + (SOURCE_HUNK, 'obj.value = "value";', None, ''), ] split = self.js_node.split_contents() split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3])) for x in split] self.assertEqual(out, split) -Html5LibParserTests = skipIf( - html5lib is None, 'html5lib not found')(Html5LibParserTests) - +@ut2.skipIf(BeautifulSoup is None, 'BeautifulSoup not found') class BeautifulSoupParserTests(ParserTestCase, CompressorTestCase): parser_cls = 'compressor.parser.BeautifulSoupParser' -BeautifulSoupParserTests = skipIf( - BeautifulSoup is None, 'BeautifulSoup not found')(BeautifulSoupParserTests) - class HtmlParserTests(ParserTestCase, CompressorTestCase): parser_cls = 'compressor.parser.HtmlParser' diff --git a/compressor/tests/test_storages.py b/compressor/tests/test_storages.py index 713002e..24d3764 100644 --- a/compressor/tests/test_storages.py +++ b/compressor/tests/test_storages.py @@ -1,4 +1,4 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals import errno import os @@ -23,7 +23,7 @@ class StorageTestCase(TestCase): base.default_storage = self._storage def test_css_tag_with_storage(self): - template = u"""{% load compress %}{% compress css %} + template = """{% load compress %}{% compress css %} @@ -40,7 +40,7 @@ class StorageTestCase(TestCase): def race_remove(path): "Patched os.remove to raise ENOENT (No such file or directory)" original_remove(path) - raise OSError(errno.ENOENT, u'Fake ENOENT') + raise OSError(errno.ENOENT, 'Fake ENOENT') try: os.remove = race_remove diff --git a/compressor/tests/test_templatetags.py b/compressor/tests/test_templatetags.py index 151b785..1bb415c 100644 --- a/compressor/tests/test_templatetags.py +++ b/compressor/tests/test_templatetags.py @@ -1,4 +1,4 @@ -from __future__ import with_statement +from __future__ import with_statement, unicode_literals import os import sys @@ -34,12 +34,12 @@ class TemplatetagTestCase(TestCase): settings.COMPRESS_ENABLED = self.old_enabled def test_empty_tag(self): - template = u"""{% load compress %}{% compress js %}{% block js %} + template = """{% load compress %}{% compress js %}{% block js %} {% endblock %}{% endcompress %}""" - self.assertEqual(u'', render(template, self.context)) + self.assertEqual('', render(template, self.context)) def test_css_tag(self): - template = u"""{% load compress %}{% compress css %} + template = """{% load compress %}{% compress css %} @@ -50,7 +50,7 @@ class TemplatetagTestCase(TestCase): maxDiff = None def test_uppercase_rel(self): - template = u"""{% load compress %}{% compress css %} + template = """{% load compress %}{% compress css %} @@ -59,7 +59,7 @@ class TemplatetagTestCase(TestCase): self.assertEqual(out, render(template, self.context)) def test_nonascii_css_tag(self): - template = u"""{% load compress %}{% compress css %} + template = """{% load compress %}{% compress css %} {% endcompress %} @@ -68,40 +68,40 @@ class TemplatetagTestCase(TestCase): self.assertEqual(out, render(template, self.context)) def test_js_tag(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} {% endcompress %} """ - out = u'' + out = '' self.assertEqual(out, render(template, self.context)) def test_nonascii_js_tag(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} {% endcompress %} """ - out = u'' + out = '' self.assertEqual(out, render(template, self.context)) def test_nonascii_latin1_js_tag(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} {% endcompress %} """ - out = u'' + out = '' self.assertEqual(out, render(template, self.context)) def test_compress_tag_with_illegal_arguments(self): - template = u"""{% load compress %}{% compress pony %} + template = """{% load compress %}{% compress pony %} {% endcompress %}""" self.assertRaises(TemplateSyntaxError, render, template, {}) def test_debug_toggle(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} {% endcompress %} @@ -111,12 +111,12 @@ class TemplatetagTestCase(TestCase): GET = {settings.COMPRESS_DEBUG_TOGGLE: 'true'} context = dict(self.context, request=MockDebugRequest()) - out = u""" + out = """ """ self.assertEqual(out, render(template, context)) def test_named_compress_tag(self): - template = u"""{% load compress %}{% compress js inline foo %} + template = """{% load compress %}{% compress js inline foo %} {% endcompress %} """ @@ -151,14 +151,14 @@ class PrecompilerTemplatetagTestCase(TestCase): settings.COMPRESS_PRECOMPILERS = self.old_precompilers def test_compress_coffeescript_tag(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} {% endcompress %}""" out = script(src="/static/CACHE/js/e920d58f166d.js") self.assertEqual(out, render(template, self.context)) def test_compress_coffeescript_tag_and_javascript_tag(self): - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} {% endcompress %}""" @@ -169,7 +169,7 @@ class PrecompilerTemplatetagTestCase(TestCase): self.old_enabled = settings.COMPRESS_ENABLED settings.COMPRESS_ENABLED = False try: - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} {% endcompress %}""" @@ -183,7 +183,7 @@ class PrecompilerTemplatetagTestCase(TestCase): self.old_enabled = settings.COMPRESS_ENABLED settings.COMPRESS_ENABLED = False try: - template = u"""{% load compress %}{% compress js %} + template = """{% load compress %}{% compress js %} {% endcompress %}""" out = script("# this is a comment.\n") @@ -195,7 +195,7 @@ class PrecompilerTemplatetagTestCase(TestCase): self.old_enabled = settings.COMPRESS_ENABLED settings.COMPRESS_ENABLED = False try: - template = u""" + template = """ {% load compress %}{% compress js %} @@ -210,7 +210,7 @@ class PrecompilerTemplatetagTestCase(TestCase): self.old_enabled = settings.COMPRESS_ENABLED settings.COMPRESS_ENABLED = False try: - template = u""" + template = """ {% load compress %}{% compress js %} @@ -232,7 +232,7 @@ class PrecompilerTemplatetagTestCase(TestCase): settings.COMPRESS_ENABLED = False assert(settings.COMPRESS_PRECOMPILERS) try: - template = u""" + template = """ {% load compress %}{% compress css %} @@ -250,7 +250,7 @@ class PrecompilerTemplatetagTestCase(TestCase): settings.COMPRESS_ENABLED = False assert(settings.COMPRESS_PRECOMPILERS) try: - template = u""" + template = """ {% load compress %}{% compress css %} @@ -272,9 +272,9 @@ def script(content="", src="", scripttype="text/javascript"): >>> script('#this is a comment', scripttype="text/applescript") '' """ - out_script = u'' % content + out_script += 'src="%s" ' % src + return out_script[:-1] + '>%s' % content diff --git a/compressor/utils/__init__.py b/compressor/utils/__init__.py index c842b73..c1d3b02 100644 --- a/compressor/utils/__init__.py +++ b/compressor/utils/__init__.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals import os +import six from compressor.exceptions import FilterError @@ -10,10 +12,10 @@ def get_class(class_string, exception=FilterError): """ if not hasattr(class_string, '__bases__'): try: - class_string = class_string.encode('ascii') + class_string = str(class_string) mod_name, class_name = get_mod_func(class_string) if class_name: - return getattr(__import__(mod_name, {}, {}, ['']), class_name) + return getattr(__import__(mod_name, {}, {}, [str('')]), class_name) except (ImportError, AttributeError): raise exception('Failed to import %s' % class_string) @@ -47,7 +49,7 @@ def find_command(cmd, paths=None, pathext=None): """ if paths is None: paths = os.environ.get('PATH', '').split(os.pathsep) - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = [paths] # check if there are funny path extensions for executables, e.g. Windows if pathext is None: diff --git a/compressor/utils/compat.py b/compressor/utils/compat.py new file mode 100644 index 0000000..91428ab --- /dev/null +++ b/compressor/utils/compat.py @@ -0,0 +1,28 @@ +import six + +try: + from django.utils.encoding import force_text, force_bytes + from django.utils.encoding import smart_text, smart_bytes +except ImportError: + # django < 1.4.2 + from django.utils.encoding import force_unicode as force_text + from django.utils.encoding import force_str as force_bytes + from django.utils.encoding import smart_unicode as smart_text + from django.utils.encoding import smart_str as smart_bytes + + +try: + from django.utils import unittest +except ImportError: + import unittest2 as unittest + + +if six.PY3: + # there is an 'io' module in python 2.6+, but io.StringIO does not + # accept regular strings, just unicode objects + from io import StringIO +else: + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO diff --git a/compressor/utils/staticfiles.py b/compressor/utils/staticfiles.py index 169d427..28026f2 100644 --- a/compressor/utils/staticfiles.py +++ b/compressor/utils/staticfiles.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals from django.core.exceptions import ImproperlyConfigured