Py3: Use oslo to encode/decode properly in callback, model and filters

Change-Id: I963f7a0f8b48f59306cd925ccc777673c16b6e5c

ara/filters.py

@@ -13,14 +13,14 @@
 #   under the License.
 
 import datetime
-import json
 import logging
+import six
 
 from ara.utils import fast_count
 from ara.utils import playbook_treeview
 from jinja2 import Markup
-from jinja2 import UndefinedError
 from os import path
+from oslo_serialization import jsonutils
 from pygments import highlight
 from pygments.formatters import HtmlFormatter
 from pygments.lexers import YamlLexer
@@ -52,10 +52,12 @@ def configure_template_filters(app):
     def jinja_to_nice_json(result):
         """ Tries to format a result as a pretty printed JSON. """
         try:
-            return json.dumps(json.loads(result), indent=4, sort_keys=True)
+            return jsonutils.dumps(jsonutils.loads(result),
+                                   indent=4,
+                                   sort_keys=True)
         except (ValueError, TypeError):
             try:
-                return json.dumps(result, indent=4, sort_keys=True)
+                return jsonutils.dumps(result, indent=4, sort_keys=True)
             except TypeError as err:
                 log.error('failed to dump json: %s', err)
                 return result
@@ -63,7 +65,7 @@ def configure_template_filters(app):
     @app.template_filter('from_json')
     def jinja_from_json(val):
         try:
-            return json.loads(val)
+            return jsonutils.loads(val)
         except ValueError as err:
             log.error('failed to load json: %s', err)
             return val
@@ -76,37 +78,32 @@ def configure_template_filters(app):
                                   linespans='line',
                                   cssclass='codehilite')
 
-        # We have little control over the content of the files we're
-        # formatting. This can lead into UnicodeDecodeError raised by Jinja
-        # due to breaking whitespace characters or other possibly encoded
-        # characters.
-        try:
-            code = code.decode('utf-8')
-        except UndefinedError:
+        if not code:
             code = ''
 
-        return highlight(Markup(code.rstrip()).unescape(),
-                         YamlLexer(),
+        return highlight(Markup(code).unescape(),
+                         YamlLexer(stripall=True),
                          formatter)
 
     @app.template_filter('pygments_formatter')
     def jinja_pygments_formatter(data):
         formatter = HtmlFormatter(cssclass='codehilite')
 
-        if isinstance(data, str) or isinstance(data, unicode):
-            data.rstrip()
+        if isinstance(data, dict) or isinstance(data, list):
+            data = jsonutils.dumps(data, indent=4, sort_keys=True)
+            lexer = JsonLexer()
+        elif six.string_types or six.text_type:
             try:
-                data = json.loads(data)
-                data = json.dumps(data, indent=4, sort_keys=True)
+                data = jsonutils.dumps(jsonutils.loads(data),
+                                       indent=4,
+                                       sort_keys=True)
                 lexer = JsonLexer()
             except (ValueError, TypeError):
                 lexer = TextLexer()
-        elif isinstance(data, dict) or isinstance(data, list):
-            data = json.dumps(data, indent=4, sort_keys=True)
-            lexer = JsonLexer()
         else:
             lexer = TextLexer()
 
+        lexer.stripall = True
         return highlight(Markup(data).unescape(), lexer, formatter)
 
     @app.template_filter('fast_count')

ara/models.py

@@ -14,12 +14,13 @@
 
 import functools
 import hashlib
-import json
 import uuid
 import zlib
 
 from datetime import datetime
 from datetime import timedelta
+from oslo_utils import encodeutils
+from oslo_serialization import jsonutils
 
 # This makes all the exceptions available as "models.<exception_name>".
 from flask_sqlalchemy import SQLAlchemy
@@ -42,10 +43,14 @@ def mkuuid():
 
 def content_sha1(context):
     """
-    Used by the FileContent class to automatically compute the sha1
+    Used by the FileContent model to automatically compute the sha1
     hash of content before storing it to the database.
     """
-    return hashlib.sha1(context.current_parameters['content']).hexdigest()
+    try:
+        content = context.current_parameters['content']
+    except AttributeError:
+        content = context
+    return hashlib.sha1(encodeutils.to_utf8(content)).hexdigest()
 
 
 # Primary key columns are of these type.
@@ -120,11 +125,11 @@ class CompressedData(types.TypeDecorator):
     impl = types.Binary
 
     def process_bind_param(self, value, dialect):
-        return zlib.compress(json.dumps(value))
+        return zlib.compress(encodeutils.to_utf8(jsonutils.dumps(value)))
 
     def process_result_value(self, value, dialect):
         if value is not None:
-            return json.loads(zlib.decompress(value))
+            return jsonutils.loads(zlib.decompress(value))
         else:
             return value
 
@@ -143,13 +148,10 @@ class CompressedText(types.TypeDecorator):
     impl = types.Binary
 
     def process_bind_param(self, value, dialect):
-        return zlib.compress(value if value else '')
+        return zlib.compress(encodeutils.to_utf8(value))
 
     def process_result_value(self, value, dialect):
-        if value is not None:
-            return zlib.decompress(value)
-        else:
-            return value
+        return encodeutils.safe_decode(zlib.decompress(value))
 
     def copy(self, **kwargs):
         return CompressedText(self.impl.length)

ara/plugins/callbacks/log_ara.py

@@ -16,9 +16,7 @@ from __future__ import (absolute_import, division, print_function)
 
 import decorator
 import flask
-import hashlib
 import itertools
-import json
 import logging
 import os
 
@@ -29,6 +27,7 @@ from ara.models import db
 from ara.webapp import create_app
 from datetime import datetime
 from distutils.version import LooseVersion
+from oslo_serialization import jsonutils
 
 # To retrieve Ansible CLI options
 try:
@@ -129,8 +128,7 @@ class CallbackModule(CallbackBase):
         try:
             with open(path, 'r') as fd:
                 data = fd.read()
-
-            sha1 = hashlib.sha1(data).hexdigest()
+            sha1 = models.content_sha1(data)
             content = models.FileContent.query.get(sha1)
 
             if content is None:
@@ -166,16 +164,16 @@ class CallbackModule(CallbackBase):
             results = [self._dump_results(result._result)]
             for item in self.loop_items:
                 results.append(self._dump_results(item._result))
-            results = json.loads(json.dumps(results))
+            results = jsonutils.loads(jsonutils.dumps(results))
         else:
-            results = json.loads(self._dump_results(result._result))
+            results = jsonutils.loads(self._dump_results(result._result))
 
         self.taskresult = models.TaskResult(
             task=self.task,
             host=host,
             time_start=result.task_start,
             time_end=result.task_end,
-            result=json.dumps(results),
+            result=jsonutils.dumps(results),
             status=status,
             changed=result._result.get('changed', False),
             failed=result._result.get('failed', False),
@@ -187,7 +185,7 @@ class CallbackModule(CallbackBase):
         db.session.add(self.taskresult)
 
         if self.task.action == 'setup' and 'ansible_facts' in result._result:
-            values = json.dumps(result._result['ansible_facts'])
+            values = jsonutils.dumps(result._result['ansible_facts'])
             facts = models.HostFacts(values=values)
             host.facts = facts
 
@@ -293,7 +291,7 @@ class CallbackModule(CallbackBase):
             action=task.action,
             play=self.play,
             playbook=self.playbook,
-            tags=json.dumps(task._attributes['tags']),
+            tags=jsonutils.dumps(task._attributes['tags']),
             file=file_,
             lineno=lineno,
             is_handler=is_handler)
@@ -338,7 +336,7 @@ class CallbackModule(CallbackBase):
         }
         tmpfile = os.path.join(app.config['ARA_TMP_DIR'], 'ara.json')
         with open(tmpfile, 'w') as file:
-            file.write(json.dumps(data))
+            file.write(jsonutils.dumps(data))
 
     def v2_playbook_on_play_start(self, play):
         self.close_task()

ara/tests/unit/fakes.py

@@ -13,12 +13,11 @@
 #   under the License.
 
 import ara.models as m
-import json
-import hashlib
 import random
 
 from ansible import __version__ as ansible_version
 from mock import MagicMock
+from oslo_serialization import jsonutils
 
 FAKE_PLAYBOOK_CONTENT = """---
 - name: ARA unit tests
@@ -74,7 +73,7 @@ class FileContent(object):
 
     @property
     def model(self):
-        sha1 = hashlib.sha1(self.content).hexdigest()
+        sha1 = m.content_sha1(self.content)
         content = m.FileContent.query.get(sha1)
 
         if content is None:
@@ -185,7 +184,7 @@ class Task(object):
     @property
     def model(self):
         return m.Task(action=self.action,
-                      tags=json.dumps(self.tags),
+                      tags=jsonutils.dumps(self.tags),
                       lineno=self.lineno,
                       name=self.name,
                       playbook=self.playbook,

ara/tests/unit/test_callback.py

@@ -12,20 +12,17 @@
 #   License for the specific language governing permissions and limitations
 #   under the License.
 
-from collections import defaultdict
 import random
 import os
-import json
 
-from ara.webapp import create_app
 import ara.models as m
 import ara.utils as u
 import ara.plugins.callbacks.log_ara as l
 
 from ara.tests.unit.common import TestAra
 from ara.tests.unit import fakes
-
-a = create_app()
+from collections import defaultdict
+from oslo_serialization import jsonutils
 
 
 class TestCallback(TestAra):
@@ -54,7 +51,7 @@ class TestCallback(TestAra):
 
         self.task = fakes.Task(name='task-%s' % self.tag,
                                playbook=self.playbook.model,
-                               path=self.playbook.path)
+                               path='/task-%s.yml')
         self.cb.v2_playbook_on_task_start(self.task, False)
 
         self.host_one = fakes.Host(name='host1', playbook=self.playbook.model)
@@ -88,10 +85,10 @@ class TestCallback(TestAra):
 
     def test_playbook_persistence(self):
         r_playbook = m.Playbook.query.first()
-        tmpfile = os.path.join(a.config['ARA_TMP_DIR'], 'ara.json')
+        tmpfile = os.path.join(self.app.config['ARA_TMP_DIR'], 'ara.json')
 
-        with open(tmpfile) as file:
-            data = json.load(file)
+        with open(tmpfile, 'rb') as file:
+            data = jsonutils.load(file)
         self.assertEqual(r_playbook.id, data['playbook']['id'])
 
     def test_callback_play(self):

ara/tests/unit/test_filters.py

@@ -13,10 +13,12 @@
 #   under the License.
 
 import datetime
+import six
 import ara.models as m
 
 from ara.tests.unit.common import TestAra
 from ara.tests.unit.common import ansible_run
+from oslo_utils import encodeutils
 
 
 class TestFilters(TestAra):
@@ -60,43 +62,46 @@ class TestFilters(TestAra):
         t = self.env.from_string('{{ data | from_json | safe }}')
         res = t.render(data=data)
 
-        self.assertEqual(res, u"{u'key': u'value'}")
+        # TODO: Figure out the difference between py2 and py3
+        if six.PY2:
+            expected = "{u'key': u'value'}"
+        else:
+            expected = "{'key': 'value'}"
+        self.assertEqual(res, expected)
 
     def test_from_json_escape(self):
         data = '{"key": "value"}'
         t = self.env.from_string('{{ data | from_json | escape }}')
         res = t.render(data=data)
 
-        self.assertEqual(res, u"{u'key': u'value'}")
+        # TODO: Figure out the difference between py2 and py3
+        if six.PY2:
+            expected = "{u'key': u'value'}"
+        else:
+            expected = "{'key': 'value'}"
+
+        self.assertEqual(res, expected)
 
     def test_to_json_safe(self):
         data = {'key': 'value'}
         t = self.env.from_string('{{ data | to_nice_json | safe }}')
         res = t.render(data=data)
 
-        self.assertEqual(res, u'{\n    "key": "value"\n}')
+        self.assertEqual(res, '{\n    "key": "value"\n}')
 
     def test_to_json_escape(self):
         data = {'key': 'value'}
         t = self.env.from_string('{{ data | to_nice_json | escape }}')
         res = t.render(data=data)
 
-        self.assertEqual(res, u'{\n    "key": "value"\n}')
-
-    def test_to_json_fails(self):
-        data = datetime.datetime.now()
-        t = self.env.from_string('{{ data | to_nice_json }}')
-        res = t.render(data=data)
-
-        self.assertEqual(res, str(data))
+        self.assertEqual(res, '{\n    "key": "value"\n}')
 
     def test_to_json_from_string(self):
         data = '{"key": "value"}'
         t = self.env.from_string('{{ data | to_nice_json | safe }}')
         res = t.render(data=data)
 
-        self.assertEqual(res,
-                         u'{\n    "key": "value"\n}')
+        self.assertEqual(res, '{\n    "key": "value"\n}')
 
     def test_to_json_from_invalid_string(self):
         # json.dumps does not raise exception on a non-json string,
@@ -105,7 +110,7 @@ class TestFilters(TestAra):
         t = self.env.from_string('{{ data | to_nice_json | safe }}')
         res = t.render(data=data)
 
-        self.assertEqual(res, u'"definitely not json"')
+        self.assertEqual(res, '"definitely not json"')
 
     def test_jinja_pygments_formatter_string_simple(self):
         data = "string"
@@ -113,7 +118,7 @@ class TestFilters(TestAra):
         res = t.render(data=data)
 
         # This is ugly, sorry
-        expected = u'''<div class="codehilite"><pre><span></span>string\n</pre></div>\n''' # flake8: noqa
+        expected = '''<div class="codehilite"><pre><span></span>string\n</pre></div>\n''' # flake8: noqa
         self.assertEqual(res, expected)
 
     def test_jinja_pygments_formatter_string_json(self):
@@ -122,7 +127,10 @@ class TestFilters(TestAra):
         res = t.render(data=data)
 
         # This is ugly, sorry
-        expected = u'''<div class="codehilite"><pre><span></span><span class="p">{</span>\n    <span class="nt">&quot;one&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span><span class="p">,</span> \n    <span class="nt">&quot;two&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span>\n<span class="p">}</span>\n</pre></div>\n''' # flake8: noqa
+        if six.PY2:
+            expected = '''<div class="codehilite"><pre><span></span><span class="p">{</span>\n    <span class="nt">&quot;one&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span><span class="p">,</span> \n    <span class="nt">&quot;two&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span>\n<span class="p">}</span>\n</pre></div>\n''' # flake8: noqa
+        else:
+            expected = '''<div class="codehilite"><pre><span></span><span class="p">{</span>\n    <span class="nt">&quot;one&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span><span class="p">,</span>\n    <span class="nt">&quot;two&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span>\n<span class="p">}</span>\n</pre></div>\n'''  # flake8: noqa
         self.assertEqual(res, expected)
 
     def test_jinja_pygments_formatter_unicode(self):
@@ -131,7 +139,7 @@ class TestFilters(TestAra):
         res = t.render(data=data)
 
         # This is ugly, sorry
-        expected = u'''<div class="codehilite"><pre><span></span>string\n</pre></div>\n''' # flake8: noqa
+        expected = '''<div class="codehilite"><pre><span></span>string\n</pre></div>\n''' # flake8: noqa
         self.assertEqual(res, expected)
 
     def test_jinja_pygments_formatter_list(self):
@@ -140,7 +148,10 @@ class TestFilters(TestAra):
         res = t.render(data=data)
 
         # This is ugly, sorry
-        expected = u'''<div class="codehilite"><pre><span></span><span class="p">[</span>\n    <span class="s2">&quot;one&quot;</span><span class="p">,</span> \n    <span class="s2">&quot;two&quot;</span>\n<span class="p">]</span>\n</pre></div>\n''' # flake8: noqa
+        if six.PY2:
+            expected = '''<div class="codehilite"><pre><span></span><span class="p">[</span>\n    <span class="s2">&quot;one&quot;</span><span class="p">,</span> \n    <span class="s2">&quot;two&quot;</span>\n<span class="p">]</span>\n</pre></div>\n''' # flake8: noqa
+        else:
+            expected = '''<div class="codehilite"><pre><span></span><span class="p">[</span>\n    <span class="s2">&quot;one&quot;</span><span class="p">,</span>\n    <span class="s2">&quot;two&quot;</span>\n<span class="p">]</span>\n</pre></div>\n'''  # flake8: noqa
         self.assertEqual(res, expected)
 
     def test_jinja_pygments_formatter_dict(self):
@@ -149,7 +160,10 @@ class TestFilters(TestAra):
         res = t.render(data=data)
 
         # This is ugly, sorry
-        expected = u'''<div class="codehilite"><pre><span></span><span class="p">{</span>\n    <span class="nt">&quot;one&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span><span class="p">,</span> \n    <span class="nt">&quot;two&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span>\n<span class="p">}</span>\n</pre></div>\n''' # flake8: noqa
+        if six.PY2:
+            expected = '''<div class="codehilite"><pre><span></span><span class="p">{</span>\n    <span class="nt">&quot;one&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span><span class="p">,</span> \n    <span class="nt">&quot;two&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span>\n<span class="p">}</span>\n</pre></div>\n''' # flake8: noqa
+        else:
+            expected = '''<div class="codehilite"><pre><span></span><span class="p">{</span>\n    <span class="nt">&quot;one&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span><span class="p">,</span>\n    <span class="nt">&quot;two&quot;</span><span class="p">:</span> <span class="s2">&quot;value&quot;</span>\n<span class="p">}</span>\n</pre></div>\n'''  # flake8: noqa
         self.assertEqual(res, expected)
 
     def test_jinja_pygments_formatter_integer(self):
@@ -158,7 +172,7 @@ class TestFilters(TestAra):
         res = t.render(data=data)
 
         # This is ugly, sorry
-        expected = u'''<div class="codehilite"><pre><span></span>1\n</pre></div>\n''' # flake8: noqa
+        expected = '''<div class="codehilite"><pre><span></span>1\n</pre></div>\n''' # flake8: noqa
         self.assertEqual(res, expected)
 
     def test_jinja_pygments_formatter_boolean(self):
@@ -167,7 +181,7 @@ class TestFilters(TestAra):
         res = t.render(data=data)
 
         # This is ugly, sorry
-        expected = u'''<div class="codehilite"><pre><span></span>True\n</pre></div>\n''' # flake8: noqa
+        expected = '''<div class="codehilite"><pre><span></span>True\n</pre></div>\n''' # flake8: noqa
         self.assertEqual(res, expected)
 
     def test_jinja_yamlhighlight(self):