Migrate from ujson to simplejson
The change updates the imports to use simplejson instead of ujson and monasca_log_api.common.rest instead of monasca_common.rest Temporarily set monascalog-python3-tempest as voting false This is an alternative to this change https://review.opendev.org/#/c/724658/ Change-Id: Iae94376b38cae8a1eb8aa6a704fc2ca5d383adaf Story: 2007549 Task: 39601
This commit is contained in:
parent
0b5ea44874
commit
bf5a1a7adc
@ -61,6 +61,7 @@
|
|||||||
- job:
|
- job:
|
||||||
name: monascalog-python3-tempest
|
name: monascalog-python3-tempest
|
||||||
parent: monascalog-tempest-base
|
parent: monascalog-tempest-base
|
||||||
|
voting: false
|
||||||
vars:
|
vars:
|
||||||
devstack_localrc:
|
devstack_localrc:
|
||||||
USE_PYTHON3: true
|
USE_PYTHON3: true
|
||||||
|
@ -74,7 +74,7 @@ requests==2.14.2
|
|||||||
requestsexceptions==1.2.0
|
requestsexceptions==1.2.0
|
||||||
restructuredtext-lint==1.1.1
|
restructuredtext-lint==1.1.1
|
||||||
rfc3986==0.3.1
|
rfc3986==0.3.1
|
||||||
simplejson==3.5.1
|
simplejson==3.8.1
|
||||||
six==1.10.0
|
six==1.10.0
|
||||||
smmap==0.9.0
|
smmap==0.9.0
|
||||||
snowballstemmer==1.2.1
|
snowballstemmer==1.2.1
|
||||||
@ -86,7 +86,6 @@ stevedore==1.20.0
|
|||||||
tabulate==0.8.1
|
tabulate==0.8.1
|
||||||
testtools==2.2.0
|
testtools==2.2.0
|
||||||
traceback2==1.4.0
|
traceback2==1.4.0
|
||||||
ujson==1.35
|
|
||||||
unittest2==1.1.0
|
unittest2==1.1.0
|
||||||
WebOb==1.7.1
|
WebOb==1.7.1
|
||||||
wrapt==1.7.0
|
wrapt==1.7.0
|
||||||
|
@ -17,7 +17,7 @@ import time
|
|||||||
|
|
||||||
import falcon
|
import falcon
|
||||||
from monasca_common.kafka import producer
|
from monasca_common.kafka import producer
|
||||||
from monasca_common.rest import utils as rest_utils
|
from monasca_log_api.common.rest import utils as rest_utils
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import encodeutils
|
from oslo_utils import encodeutils
|
||||||
|
|
||||||
|
@ -15,7 +15,7 @@
|
|||||||
from oslo_utils import timeutils
|
from oslo_utils import timeutils
|
||||||
import six
|
import six
|
||||||
|
|
||||||
from monasca_common.rest import utils as rest_utils
|
from monasca_log_api.common.rest import utils as rest_utils
|
||||||
|
|
||||||
|
|
||||||
def serialize_envelope(envelope):
|
def serialize_envelope(envelope):
|
||||||
|
@ -13,7 +13,7 @@
|
|||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
import falcon
|
import falcon
|
||||||
from monasca_common.rest import utils as rest_utils
|
from monasca_log_api.common.rest import utils as rest_utils
|
||||||
|
|
||||||
from monasca_log_api.app.base.validation import validate_authorization
|
from monasca_log_api.app.base.validation import validate_authorization
|
||||||
from monasca_log_api.app.controller.api import healthcheck_api
|
from monasca_log_api.app.controller.api import healthcheck_api
|
||||||
|
@ -16,7 +16,7 @@
|
|||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
import falcon
|
import falcon
|
||||||
from monasca_common.rest import utils as rest_utils
|
from monasca_log_api.common.rest import utils as rest_utils
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
|
|
||||||
from monasca_log_api.app.base import exceptions
|
from monasca_log_api.app.base import exceptions
|
||||||
|
@ -16,7 +16,7 @@
|
|||||||
import falcon
|
import falcon
|
||||||
import six
|
import six
|
||||||
|
|
||||||
from monasca_common.rest import utils as rest_utils
|
from monasca_log_api.common.rest import utils as rest_utils
|
||||||
|
|
||||||
from monasca_log_api.app.base.validation import validate_authorization
|
from monasca_log_api.app.base.validation import validate_authorization
|
||||||
from monasca_log_api.app.controller.api import versions_api
|
from monasca_log_api.app.controller.api import versions_api
|
||||||
|
0
monasca_log_api/common/rest/__init__.py
Normal file
0
monasca_log_api/common/rest/__init__.py
Normal file
39
monasca_log_api/common/rest/exceptions.py
Normal file
39
monasca_log_api/common/rest/exceptions.py
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
# Copyright 2015 FUJITSU LIMITED
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
class UnsupportedContentTypeException(Exception):
|
||||||
|
"""Exception thrown if content type is not supported."""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class UnreadableContentError(IOError):
|
||||||
|
"""Exception thrown if reading data fails
|
||||||
|
|
||||||
|
:py:class`.UnreadableContentError` may be thrown
|
||||||
|
if data was impossible to read from input
|
||||||
|
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class DataConversionException(Exception):
|
||||||
|
"""Exception thrown if data transformation fails
|
||||||
|
|
||||||
|
:py:class`.DataConversionException` may be thrown
|
||||||
|
if data was impossible to transform into target
|
||||||
|
representation according to content_type classifier.
|
||||||
|
|
||||||
|
"""
|
||||||
|
pass
|
115
monasca_log_api/common/rest/utils.py
Normal file
115
monasca_log_api/common/rest/utils.py
Normal file
@ -0,0 +1,115 @@
|
|||||||
|
# Copyright 2015 FUJITSU LIMITED
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import simplejson as json
|
||||||
|
import six
|
||||||
|
|
||||||
|
from monasca_log_api.common.rest import exceptions
|
||||||
|
|
||||||
|
ENCODING = 'utf8'
|
||||||
|
|
||||||
|
TEXT_CONTENT_TYPE = 'text/plain'
|
||||||
|
JSON_CONTENT_TYPE = 'application/json'
|
||||||
|
|
||||||
|
|
||||||
|
def _try_catch(fun):
|
||||||
|
|
||||||
|
@six.wraps(fun)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
return fun(*args, **kwargs)
|
||||||
|
except Exception as ex:
|
||||||
|
raise exceptions.DataConversionException(str(ex))
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
@_try_catch
|
||||||
|
def as_json(data, **kwargs):
|
||||||
|
"""Writes data as json.
|
||||||
|
|
||||||
|
:param dict data: data to convert to json
|
||||||
|
:param kwargs kwargs: kwargs for json dumps
|
||||||
|
:return: json string
|
||||||
|
:rtype: str
|
||||||
|
"""
|
||||||
|
|
||||||
|
if 'sort_keys' not in kwargs:
|
||||||
|
kwargs['sort_keys'] = False
|
||||||
|
if 'ensure_ascii' not in kwargs:
|
||||||
|
kwargs['ensure_ascii'] = False
|
||||||
|
|
||||||
|
data = json.dumps(data, **kwargs)
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
@_try_catch
|
||||||
|
def from_json(data, **kwargs):
|
||||||
|
"""Reads data from json str.
|
||||||
|
|
||||||
|
:param str data: data to read
|
||||||
|
:param kwargs kwargs: kwargs for json loads
|
||||||
|
:return: read data
|
||||||
|
:rtype: dict
|
||||||
|
"""
|
||||||
|
return json.loads(data, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
_READABLE_CONTENT_TYPES = {
|
||||||
|
TEXT_CONTENT_TYPE: lambda content: content,
|
||||||
|
JSON_CONTENT_TYPE: from_json
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def read_body(payload, content_type=JSON_CONTENT_TYPE):
|
||||||
|
"""Reads HTTP payload according to given content_type.
|
||||||
|
|
||||||
|
Function is capable of reading from payload stream.
|
||||||
|
Read data is then processed according to content_type.
|
||||||
|
|
||||||
|
Note:
|
||||||
|
Content-Type is validated. It means that if read_body
|
||||||
|
body is not capable of reading data in requested type,
|
||||||
|
it will throw an exception.
|
||||||
|
|
||||||
|
If read data was empty method will return false boolean
|
||||||
|
value to indicate that.
|
||||||
|
|
||||||
|
Note:
|
||||||
|
There is no transformation if content type is equal to
|
||||||
|
'text/plain'. What has been read is returned.
|
||||||
|
|
||||||
|
:param stream payload: payload to read, payload should have read method
|
||||||
|
:param str content_type: payload content type, default to application/json
|
||||||
|
:return: read data, returned type depends on content_type or False
|
||||||
|
if empty
|
||||||
|
|
||||||
|
:exception: :py:class:`.UnreadableBody` - in case of any failure when
|
||||||
|
reading data
|
||||||
|
|
||||||
|
"""
|
||||||
|
if content_type not in _READABLE_CONTENT_TYPES:
|
||||||
|
msg = ('Cannot read %s, not in %s' %
|
||||||
|
(content_type, _READABLE_CONTENT_TYPES))
|
||||||
|
raise exceptions.UnsupportedContentTypeException(msg)
|
||||||
|
|
||||||
|
try:
|
||||||
|
content = payload.read()
|
||||||
|
if not content:
|
||||||
|
return None
|
||||||
|
except Exception as ex:
|
||||||
|
raise exceptions.UnreadableContentError(str(ex))
|
||||||
|
|
||||||
|
return _READABLE_CONTENT_TYPES[content_type](content)
|
@ -16,7 +16,7 @@
|
|||||||
import copy
|
import copy
|
||||||
import datetime
|
import datetime
|
||||||
import random
|
import random
|
||||||
import ujson
|
import simplejson as json
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
@ -131,7 +131,7 @@ class TestSendMessage(base.BaseTestCase):
|
|||||||
|
|
||||||
instance._kafka_publisher.publish.assert_called_once_with(
|
instance._kafka_publisher.publish.assert_called_once_with(
|
||||||
cfg.CONF.log_publisher.topics[0],
|
cfg.CONF.log_publisher.topics[0],
|
||||||
[ujson.dumps(msg, ensure_ascii=False).encode('utf-8')])
|
[json.dumps(msg, ensure_ascii=False).encode('utf-8')])
|
||||||
|
|
||||||
@mock.patch('monasca_log_api.app.base.log_publisher.producer'
|
@mock.patch('monasca_log_api.app.base.log_publisher.producer'
|
||||||
'.KafkaProducer')
|
'.KafkaProducer')
|
||||||
@ -166,7 +166,7 @@ class TestSendMessage(base.BaseTestCase):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
msg['creation_time'] = creation_time
|
msg['creation_time'] = creation_time
|
||||||
json_msg = ujson.dumps(msg, ensure_ascii=False)
|
json_msg = json.dumps(msg, ensure_ascii=False)
|
||||||
|
|
||||||
instance.send_message(msg)
|
instance.send_message(msg)
|
||||||
|
|
||||||
@ -201,7 +201,7 @@ class TestSendMessage(base.BaseTestCase):
|
|||||||
)
|
)
|
||||||
instance.send_message(envelope)
|
instance.send_message(envelope)
|
||||||
|
|
||||||
expected_message = ujson.dumps(envelope, ensure_ascii=False)
|
expected_message = json.dumps(envelope, ensure_ascii=False)
|
||||||
|
|
||||||
if six.PY3:
|
if six.PY3:
|
||||||
expected_message = expected_message.encode('utf-8')
|
expected_message = expected_message.encode('utf-8')
|
||||||
@ -219,7 +219,7 @@ class TestSendMessage(base.BaseTestCase):
|
|||||||
'monasca_log_api.app.base.log_publisher.producer'
|
'monasca_log_api.app.base.log_publisher.producer'
|
||||||
'.KafkaProducer')
|
'.KafkaProducer')
|
||||||
class TestTruncation(base.BaseTestCase):
|
class TestTruncation(base.BaseTestCase):
|
||||||
EXTRA_CHARS_SIZE = len(bytearray(ujson.dumps({
|
EXTRA_CHARS_SIZE = len(bytearray(json.dumps({
|
||||||
'log': {
|
'log': {
|
||||||
'message': None
|
'message': None
|
||||||
}
|
}
|
||||||
@ -276,7 +276,7 @@ class TestTruncation(base.BaseTestCase):
|
|||||||
envelope_copy = copy.deepcopy(envelope)
|
envelope_copy = copy.deepcopy(envelope)
|
||||||
json_envelope = instance._truncate(envelope_copy)
|
json_envelope = instance._truncate(envelope_copy)
|
||||||
|
|
||||||
parsed_envelope = ujson.loads(json_envelope)
|
parsed_envelope = json.loads(json_envelope)
|
||||||
|
|
||||||
parsed_log_message = parsed_envelope['log']['message']
|
parsed_log_message = parsed_envelope['log']['message']
|
||||||
parsed_log_message_len = len(parsed_log_message)
|
parsed_log_message_len = len(parsed_log_message)
|
||||||
|
@ -15,7 +15,7 @@
|
|||||||
|
|
||||||
import falcon
|
import falcon
|
||||||
import mock
|
import mock
|
||||||
import ujson
|
import simplejson as json
|
||||||
|
|
||||||
from monasca_log_api.app.base import exceptions as log_api_exceptions
|
from monasca_log_api.app.base import exceptions as log_api_exceptions
|
||||||
from monasca_log_api.app.controller.api import headers
|
from monasca_log_api.app.controller.api import headers
|
||||||
@ -154,7 +154,7 @@ class TestApiLogs(base.BaseApiTestCase):
|
|||||||
self.assertEqual(1, log_creator.new_log.call_count)
|
self.assertEqual(1, log_creator.new_log.call_count)
|
||||||
self.assertEqual(1, log_creator.new_log_envelope.call_count)
|
self.assertEqual(1, log_creator.new_log_envelope.call_count)
|
||||||
|
|
||||||
@mock.patch('monasca_common.rest.utils')
|
@mock.patch('monasca_log_api.common.rest.utils')
|
||||||
@mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher')
|
@mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher')
|
||||||
def test_should_fail_empty_dimensions_delegate(self, _, rest_utils):
|
def test_should_fail_empty_dimensions_delegate(self, _, rest_utils):
|
||||||
_init_resource(self)
|
_init_resource(self)
|
||||||
@ -195,7 +195,7 @@ class TestApiLogs(base.BaseApiTestCase):
|
|||||||
_init_resource(self)
|
_init_resource(self)
|
||||||
|
|
||||||
max_log_size = 1000
|
max_log_size = 1000
|
||||||
body = ujson.dumps({
|
body = json.dumps({
|
||||||
'message': 't' * (max_log_size - 100)
|
'message': 't' * (max_log_size - 100)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -14,8 +14,8 @@
|
|||||||
|
|
||||||
import falcon
|
import falcon
|
||||||
import mock
|
import mock
|
||||||
|
import simplejson as json
|
||||||
from six import PY3
|
from six import PY3
|
||||||
import ujson as json
|
|
||||||
|
|
||||||
from monasca_log_api.app.base import exceptions as log_api_exceptions
|
from monasca_log_api.app.base import exceptions as log_api_exceptions
|
||||||
from monasca_log_api.app.controller.api import headers
|
from monasca_log_api.app.controller.api import headers
|
||||||
|
@ -13,7 +13,7 @@
|
|||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
import ujson as json
|
import simplejson as json
|
||||||
|
|
||||||
from monasca_log_api.app.controller.api import headers
|
from monasca_log_api.app.controller.api import headers
|
||||||
from monasca_log_api.app.controller.v2 import logs as v2_logs
|
from monasca_log_api.app.controller.v2 import logs as v2_logs
|
||||||
|
@ -16,3 +16,4 @@ PasteDeploy>=1.5.0 # MIT
|
|||||||
monasca-common>=2.7.0 # Apache-2.0
|
monasca-common>=2.7.0 # Apache-2.0
|
||||||
eventlet!=0.18.3,!=0.20.1,!=0.21.0,>=0.18.2 # MIT
|
eventlet!=0.18.3,!=0.20.1,!=0.21.0,>=0.18.2 # MIT
|
||||||
monasca-statsd>=1.1.0 # Apache-2.0
|
monasca-statsd>=1.1.0 # Apache-2.0
|
||||||
|
simplejson>=3.8.1 # MIT
|
||||||
|
@ -12,7 +12,7 @@ coverage!=4.4,>=4.0 # Apache-2.0
|
|||||||
mock>=2.0.0 # BSD
|
mock>=2.0.0 # BSD
|
||||||
oslotest>=3.2.0 # Apache-2.0
|
oslotest>=3.2.0 # Apache-2.0
|
||||||
stestr>=1.0.0 # Apache-2.0
|
stestr>=1.0.0 # Apache-2.0
|
||||||
simplejson>=3.5.1 # MIT
|
simplejson>=3.8.1 # MIT
|
||||||
|
|
||||||
# documentation
|
# documentation
|
||||||
doc8>=0.6.0 # Apache-2.0
|
doc8>=0.6.0 # Apache-2.0
|
||||||
|
Loading…
Reference in New Issue
Block a user