From 44a27384601113cdc5068940d07b85a003ed2211 Mon Sep 17 00:00:00 2001 From: Renat Akhmerov Date: Tue, 31 Mar 2020 18:09:08 +0700 Subject: [PATCH] Add a utility for JSON serialization * This patch adds a utility that helps serialize data into a JSON string that might contain some non-standard data types like generators iterators and frozen dicts coming from YAQL. The utility uses oslo.serialization project that already takes care of iterators and any kinds of custom dicts. And in addition, it handles generators (assuming a generator represents an iterable similar to an iterator). * Unit tests. * Added YAQL into requirements and bumped the version of oslo.serialization to make sure to have the "fallback" parameter in "jsonutils.to_primitive" Change-Id: I2fe891525bc86beb92aecf9ac2d8a490837c47d3 --- lower-constraints.txt | 3 +- mistral_lib/tests/test_utils.py | 71 +++++++++++++++++++++++++++++++++ mistral_lib/utils/__init__.py | 51 +++++++++++++++++++++++ requirements.txt | 3 +- 4 files changed, 126 insertions(+), 2 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index 1099048..d2cee79 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -28,7 +28,7 @@ openstackdocstheme==1.18.1 os-client-config==1.28.0 oslo.i18n==3.15.3 oslo.log==3.36.0 -oslo.serialization==2.18.0 +oslo.serialization==2.21.1 oslo.utils==3.33.0 oslotest==3.2.0 pbr==2.0.0 @@ -54,3 +54,4 @@ testtools==2.2.0 traceback2==1.4.0 unittest2==1.1.0 wrapt==1.7.0 +yaql==1.1.3 # Apache 2.0 License diff --git a/mistral_lib/tests/test_utils.py b/mistral_lib/tests/test_utils.py index 23bdb3d..eeb754c 100644 --- a/mistral_lib/tests/test_utils.py +++ b/mistral_lib/tests/test_utils.py @@ -14,10 +14,14 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + import copy +from yaql.language import utils as yaql_utils + from mistral_lib.tests import base as tests_base from mistral_lib import utils + import testtools.matchers as ttm @@ -267,3 +271,70 @@ class TestUtils(tests_base.TestCase): payload = ["adminPass", 'fooBarBaz'] expected = ["adminPass", 'fooBarBaz'] self.assertEqual(expected, utils.mask_data(payload)) + + def test_json_serialize_frozen_dict(self): + data = yaql_utils.FrozenDict(a=1, b=2, c=iter([1, 2, 3])) + + json_str = utils.to_json_str(data) + + self.assertIsNotNone(json_str) + + self.assertIn('"a": 1', json_str) + self.assertIn('"b": 2', json_str) + self.assertIn('"c": [1, 2, 3]', json_str) + + def test_json_serialize_generator(self): + def _list_stream(_list): + for i in _list: + yield i + + gen = _list_stream( + [1, yaql_utils.FrozenDict(a=1), _list_stream([12, 15])] + ) + + self.assertEqual('[1, {"a": 1}, [12, 15]]', utils.to_json_str(gen)) + + def test_json_serialize_dict_of_generators(self): + def _f(cnt): + for i in range(1, cnt + 1): + yield i + + data = {'numbers': _f(3)} + + self.assertEqual('{"numbers": [1, 2, 3]}', utils.to_json_str(data)) + + def test_json_serialize_range(self): + self.assertEqual("[1, 2, 3, 4]", utils.to_json_str(range(1, 5))) + + def test_json_serialize_iterator_of_frozen_dicts(self): + data = iter( + [ + yaql_utils.FrozenDict(a=1, b=2, c=iter([1, 2, 3])), + yaql_utils.FrozenDict( + a=11, + b=yaql_utils.FrozenDict(b='222'), + c=iter( + [ + 1, + yaql_utils.FrozenDict( + a=iter([4, yaql_utils.FrozenDict(a=99)]) + ) + ] + ) + ) + ] + ) + + json_str = utils.to_json_str(data) + + self.assertIsNotNone(json_str) + + # Checking the first item. + self.assertIn('"a": 1', json_str) + self.assertIn('"b": 2', json_str) + self.assertIn('"c": [1, 2, 3]', json_str) + + # Checking the first item. + self.assertIn('"a": 11', json_str) + self.assertIn('"b": {"b": "222"}', json_str) + self.assertIn('"c": [1, {"a": [4, {"a": 99}]}]', json_str) diff --git a/mistral_lib/utils/__init__.py b/mistral_lib/utils/__init__.py index 2d4e9c4..56f88b4 100644 --- a/mistral_lib/utils/__init__.py +++ b/mistral_lib/utils/__init__.py @@ -17,6 +17,7 @@ import datetime import functools +import inspect import json import os from os import path @@ -28,6 +29,7 @@ import threading import eventlet from eventlet import corolocal from oslo_log import log as logging +from oslo_serialization import jsonutils from oslo_utils.strutils import mask_dict_password from oslo_utils.strutils import mask_password from oslo_utils import timeutils @@ -495,3 +497,52 @@ def mask_data(obj): return [mask_data(i) for i in obj] else: return mask_password(obj) + + +def to_json_str(obj): + """Serializes an object into a JSON string. + + :param obj: Object to serialize. + :return: JSON string. + """ + + if obj is None: + return None + + def _fallback(value): + if inspect.isgenerator(value): + result = list(value) + + # The result of the generator call may be again not primitive + # so we need to call "to_primitive" again with the same fallback + # function. Note that the endless recursion here is not a problem + # because "to_primitive" limits the depth for custom classes, + # if they are present in the object graph being traversed. + return jsonutils.to_primitive( + result, + convert_instances=True, + fallback=_fallback + ) + + return value + + # We need to convert the root of the given object graph into + # a primitive by hand so that we also enable conversion of + # object of custom classes into primitives. Otherwise, they are + # ignored by the "json" lib. + return jsonutils.dumps( + jsonutils.to_primitive(obj, convert_instances=True, fallback=_fallback) + ) + + +def from_json_str(json_str): + """Reconstructs an object from a JSON string. + + :param json_str: A JSON string. + :return: Deserialized object. + """ + + if json_str is None: + return None + + return jsonutils.loads(json_str) diff --git a/requirements.txt b/requirements.txt index c9e2689..0b1c906 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ eventlet!=0.20.1,>=0.20.0 # MIT oslo.log>=3.36.0 # Apache-2.0 pbr!=2.1.0,>=2.0.0 # Apache-2.0 -oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 +oslo.serialization>=2.21.1 # Apache-2.0 +yaql>=1.1.3 # Apache 2.0 License