Migrate from ujson to simplejson

The change updates the imports to use simplejson instead of ujson and
monasca_log_api.common.rest instead of monasca_common.rest

Temporarily set monascalog-python3-tempest as voting false

This is an alternative to this change
https://review.opendev.org/#/c/724658/

Change-Id: Iae94376b38cae8a1eb8aa6a704fc2ca5d383adaf
Story: 2007549
Task: 39601

.zuul.yaml

@@ -61,6 +61,7 @@
 - job:
     name: monascalog-python3-tempest
     parent: monascalog-tempest-base
+    voting: false
     vars:
       devstack_localrc:
         USE_PYTHON3: true

lower-constraints.txt

@@ -74,7 +74,7 @@ requests==2.14.2
 requestsexceptions==1.2.0
 restructuredtext-lint==1.1.1
 rfc3986==0.3.1
-simplejson==3.5.1
+simplejson==3.8.1
 six==1.10.0
 smmap==0.9.0
 snowballstemmer==1.2.1
@@ -86,7 +86,6 @@ stevedore==1.20.0
 tabulate==0.8.1
 testtools==2.2.0
 traceback2==1.4.0
-ujson==1.35
 unittest2==1.1.0
 WebOb==1.7.1
 wrapt==1.7.0

monasca_log_api/app/base/log_publisher.py

@@ -17,7 +17,7 @@ import time
 
 import falcon
 from monasca_common.kafka import producer
-from monasca_common.rest import utils as rest_utils
+from monasca_log_api.common.rest import utils as rest_utils
 from oslo_log import log
 from oslo_utils import encodeutils
 

monasca_log_api/app/base/model.py

@@ -15,7 +15,7 @@
 from oslo_utils import timeutils
 import six
 
-from monasca_common.rest import utils as rest_utils
+from monasca_log_api.common.rest import utils as rest_utils
 
 
 def serialize_envelope(envelope):

monasca_log_api/app/controller/healthchecks.py

@@ -13,7 +13,7 @@
 # under the License.
 
 import falcon
-from monasca_common.rest import utils as rest_utils
+from monasca_log_api.common.rest import utils as rest_utils
 
 from monasca_log_api.app.base.validation import validate_authorization
 from monasca_log_api.app.controller.api import healthcheck_api

monasca_log_api/app/controller/v3/aid/helpers.py

@@ -16,7 +16,7 @@
 # under the License.
 
 import falcon
-from monasca_common.rest import utils as rest_utils
+from monasca_log_api.common.rest import utils as rest_utils
 from oslo_log import log
 
 from monasca_log_api.app.base import exceptions

monasca_log_api/app/controller/versions.py

@@ -16,7 +16,7 @@
 import falcon
 import six
 
-from monasca_common.rest import utils as rest_utils
+from monasca_log_api.common.rest import utils as rest_utils
 
 from monasca_log_api.app.base.validation import validate_authorization
 from monasca_log_api.app.controller.api import versions_api

monasca_log_api/common/rest/exceptions.py

@@ -0,0 +1,39 @@
+# Copyright 2015 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+class UnsupportedContentTypeException(Exception):
+    """Exception thrown if content type is not supported."""
+    pass
+
+
+class UnreadableContentError(IOError):
+    """Exception thrown if reading data fails
+
+    :py:class`.UnreadableContentError` may be thrown
+    if data was impossible to read from input
+
+    """
+    pass
+
+
+class DataConversionException(Exception):
+    """Exception thrown if data transformation fails
+
+    :py:class`.DataConversionException` may be thrown
+    if data was impossible to transform into target
+    representation according to content_type classifier.
+
+    """
+    pass

monasca_log_api/common/rest/utils.py

@@ -0,0 +1,115 @@
+# Copyright 2015 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import simplejson as json
+import six
+
+from monasca_log_api.common.rest import exceptions
+
+ENCODING = 'utf8'
+
+TEXT_CONTENT_TYPE = 'text/plain'
+JSON_CONTENT_TYPE = 'application/json'
+
+
+def _try_catch(fun):
+
+    @six.wraps(fun)
+    def wrapper(*args, **kwargs):
+        try:
+            return fun(*args, **kwargs)
+        except Exception as ex:
+            raise exceptions.DataConversionException(str(ex))
+
+    return wrapper
+
+
+@_try_catch
+def as_json(data, **kwargs):
+    """Writes data as json.
+
+    :param dict data: data to convert to json
+    :param kwargs kwargs: kwargs for json dumps
+    :return: json string
+    :rtype: str
+    """
+
+    if 'sort_keys' not in kwargs:
+        kwargs['sort_keys'] = False
+    if 'ensure_ascii' not in kwargs:
+        kwargs['ensure_ascii'] = False
+
+    data = json.dumps(data, **kwargs)
+
+    return data
+
+
+@_try_catch
+def from_json(data, **kwargs):
+    """Reads data from json str.
+
+    :param str data: data to read
+    :param kwargs kwargs: kwargs for json loads
+    :return: read data
+    :rtype: dict
+    """
+    return json.loads(data, **kwargs)
+
+
+_READABLE_CONTENT_TYPES = {
+    TEXT_CONTENT_TYPE: lambda content: content,
+    JSON_CONTENT_TYPE: from_json
+}
+
+
+def read_body(payload, content_type=JSON_CONTENT_TYPE):
+    """Reads HTTP payload according to given content_type.
+
+    Function is capable of reading from payload stream.
+    Read data is then processed according to content_type.
+
+    Note:
+        Content-Type is validated. It means that if read_body
+        body is not capable of reading data in requested type,
+        it will throw an exception.
+
+    If read data was empty method will return false boolean
+    value to indicate that.
+
+    Note:
+        There is no transformation if content type is equal to
+        'text/plain'. What has been read is returned.
+
+    :param stream payload: payload to read, payload should have read method
+    :param str content_type: payload content type, default to application/json
+    :return: read data, returned type depends on content_type or False
+             if empty
+
+    :exception: :py:class:`.UnreadableBody` - in case of any failure when
+                                              reading data
+
+    """
+    if content_type not in _READABLE_CONTENT_TYPES:
+        msg = ('Cannot read %s, not in %s' %
+               (content_type, _READABLE_CONTENT_TYPES))
+        raise exceptions.UnsupportedContentTypeException(msg)
+
+    try:
+        content = payload.read()
+        if not content:
+            return None
+    except Exception as ex:
+        raise exceptions.UnreadableContentError(str(ex))
+
+    return _READABLE_CONTENT_TYPES[content_type](content)

monasca_log_api/tests/test_log_publisher.py

@@ -16,7 +16,7 @@
 import copy
 import datetime
 import random
-import ujson
+import simplejson as json
 import unittest
 
 import mock
@@ -131,7 +131,7 @@ class TestSendMessage(base.BaseTestCase):
 
         instance._kafka_publisher.publish.assert_called_once_with(
             cfg.CONF.log_publisher.topics[0],
-            [ujson.dumps(msg, ensure_ascii=False).encode('utf-8')])
+            [json.dumps(msg, ensure_ascii=False).encode('utf-8')])
 
     @mock.patch('monasca_log_api.app.base.log_publisher.producer'
                 '.KafkaProducer')
@@ -166,7 +166,7 @@ class TestSendMessage(base.BaseTestCase):
             }
         )
         msg['creation_time'] = creation_time
-        json_msg = ujson.dumps(msg, ensure_ascii=False)
+        json_msg = json.dumps(msg, ensure_ascii=False)
 
         instance.send_message(msg)
 
@@ -201,7 +201,7 @@ class TestSendMessage(base.BaseTestCase):
                 )
                 instance.send_message(envelope)
 
-                expected_message = ujson.dumps(envelope, ensure_ascii=False)
+                expected_message = json.dumps(envelope, ensure_ascii=False)
 
                 if six.PY3:
                     expected_message = expected_message.encode('utf-8')
@@ -219,7 +219,7 @@ class TestSendMessage(base.BaseTestCase):
     'monasca_log_api.app.base.log_publisher.producer'
     '.KafkaProducer')
 class TestTruncation(base.BaseTestCase):
-    EXTRA_CHARS_SIZE = len(bytearray(ujson.dumps({
+    EXTRA_CHARS_SIZE = len(bytearray(json.dumps({
         'log': {
             'message': None
         }
@@ -276,7 +276,7 @@ class TestTruncation(base.BaseTestCase):
         envelope_copy = copy.deepcopy(envelope)
         json_envelope = instance._truncate(envelope_copy)
 
-        parsed_envelope = ujson.loads(json_envelope)
+        parsed_envelope = json.loads(json_envelope)
 
         parsed_log_message = parsed_envelope['log']['message']
         parsed_log_message_len = len(parsed_log_message)

monasca_log_api/tests/test_logs.py

@@ -15,7 +15,7 @@
 
 import falcon
 import mock
-import ujson
+import simplejson as json
 
 from monasca_log_api.app.base import exceptions as log_api_exceptions
 from monasca_log_api.app.controller.api import headers
@@ -154,7 +154,7 @@ class TestApiLogs(base.BaseApiTestCase):
         self.assertEqual(1, log_creator.new_log.call_count)
         self.assertEqual(1, log_creator.new_log_envelope.call_count)
 
-    @mock.patch('monasca_common.rest.utils')
+    @mock.patch('monasca_log_api.common.rest.utils')
     @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher')
     def test_should_fail_empty_dimensions_delegate(self, _, rest_utils):
         _init_resource(self)
@@ -195,7 +195,7 @@ class TestApiLogs(base.BaseApiTestCase):
         _init_resource(self)
 
         max_log_size = 1000
-        body = ujson.dumps({
+        body = json.dumps({
             'message': 't' * (max_log_size - 100)
         })
 

monasca_log_api/tests/test_logs_v3.py

@@ -14,8 +14,8 @@
 
 import falcon
 import mock
+import simplejson as json
 from six import PY3
-import ujson as json
 
 from monasca_log_api.app.base import exceptions as log_api_exceptions
 from monasca_log_api.app.controller.api import headers

monasca_log_api/tests/test_v2_v3_compare.py

@@ -13,7 +13,7 @@
 # under the License.
 
 import mock
-import ujson as json
+import simplejson as json
 
 from monasca_log_api.app.controller.api import headers
 from monasca_log_api.app.controller.v2 import logs as v2_logs

requirements.txt

@@ -16,3 +16,4 @@ PasteDeploy>=1.5.0 # MIT
 monasca-common>=2.7.0 # Apache-2.0
 eventlet!=0.18.3,!=0.20.1,!=0.21.0,>=0.18.2  # MIT
 monasca-statsd>=1.1.0 # Apache-2.0
+simplejson>=3.8.1 # MIT

test-requirements.txt

@@ -12,7 +12,7 @@ coverage!=4.4,>=4.0 # Apache-2.0
 mock>=2.0.0 # BSD
 oslotest>=3.2.0 # Apache-2.0
 stestr>=1.0.0 # Apache-2.0
-simplejson>=3.5.1 # MIT
+simplejson>=3.8.1 # MIT
 
 # documentation
 doc8>=0.6.0 # Apache-2.0