Add a utility for JSON serialization

* This patch adds a utility that helps serialize data into a JSON
  string that might contain some non-standard data types like
  generators iterators and frozen dicts coming from YAQL. The
  utility uses oslo.serialization project that already takes care
  of iterators and any kinds of custom dicts. And in addition, it
  handles generators (assuming a generator represents an iterable
  similar to an iterator).
* Unit tests.
* Added YAQL into requirements and bumped the version of
  oslo.serialization to make sure to have the "fallback" parameter
  in "jsonutils.to_primitive"

Change-Id: I2fe891525bc86beb92aecf9ac2d8a490837c47d3

lower-constraints.txt

@@ -28,7 +28,7 @@ openstackdocstheme==1.18.1
 os-client-config==1.28.0
 oslo.i18n==3.15.3
 oslo.log==3.36.0
-oslo.serialization==2.18.0
+oslo.serialization==2.21.1
 oslo.utils==3.33.0
 oslotest==3.2.0
 pbr==2.0.0
@@ -54,3 +54,4 @@ testtools==2.2.0
 traceback2==1.4.0
 unittest2==1.1.0
 wrapt==1.7.0
+yaql==1.1.3 # Apache 2.0 License

mistral_lib/tests/test_utils.py

@@ -14,10 +14,14 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+
 import copy
 
+from yaql.language import utils as yaql_utils
+
 from mistral_lib.tests import base as tests_base
 from mistral_lib import utils
+
 import testtools.matchers as ttm
 
 
@@ -267,3 +271,70 @@ class TestUtils(tests_base.TestCase):
         payload = ["adminPass", 'fooBarBaz']
         expected = ["adminPass", 'fooBarBaz']
         self.assertEqual(expected, utils.mask_data(payload))
+
+    def test_json_serialize_frozen_dict(self):
+        data = yaql_utils.FrozenDict(a=1, b=2, c=iter([1, 2, 3]))
+
+        json_str = utils.to_json_str(data)
+
+        self.assertIsNotNone(json_str)
+
+        self.assertIn('"a": 1', json_str)
+        self.assertIn('"b": 2', json_str)
+        self.assertIn('"c": [1, 2, 3]', json_str)
+
+    def test_json_serialize_generator(self):
+        def _list_stream(_list):
+            for i in _list:
+                yield i
+
+        gen = _list_stream(
+            [1, yaql_utils.FrozenDict(a=1), _list_stream([12, 15])]
+        )
+
+        self.assertEqual('[1, {"a": 1}, [12, 15]]', utils.to_json_str(gen))
+
+    def test_json_serialize_dict_of_generators(self):
+        def _f(cnt):
+            for i in range(1, cnt + 1):
+                yield i
+
+        data = {'numbers': _f(3)}
+
+        self.assertEqual('{"numbers": [1, 2, 3]}', utils.to_json_str(data))
+
+    def test_json_serialize_range(self):
+        self.assertEqual("[1, 2, 3, 4]", utils.to_json_str(range(1, 5)))
+
+    def test_json_serialize_iterator_of_frozen_dicts(self):
+        data = iter(
+            [
+                yaql_utils.FrozenDict(a=1, b=2, c=iter([1, 2, 3])),
+                yaql_utils.FrozenDict(
+                    a=11,
+                    b=yaql_utils.FrozenDict(b='222'),
+                    c=iter(
+                        [
+                            1,
+                            yaql_utils.FrozenDict(
+                                a=iter([4, yaql_utils.FrozenDict(a=99)])
+                            )
+                        ]
+                    )
+                )
+            ]
+        )
+
+        json_str = utils.to_json_str(data)
+
+        self.assertIsNotNone(json_str)
+
+        # Checking the first item.
+        self.assertIn('"a": 1', json_str)
+        self.assertIn('"b": 2', json_str)
+        self.assertIn('"c": [1, 2, 3]', json_str)
+
+        # Checking the first item.
+        self.assertIn('"a": 11', json_str)
+        self.assertIn('"b": {"b": "222"}', json_str)
+        self.assertIn('"c": [1, {"a": [4, {"a": 99}]}]', json_str)

mistral_lib/utils/__init__.py

@@ -17,6 +17,7 @@
 
 import datetime
 import functools
+import inspect
 import json
 import os
 from os import path
@@ -28,6 +29,7 @@ import threading
 import eventlet
 from eventlet import corolocal
 from oslo_log import log as logging
+from oslo_serialization import jsonutils
 from oslo_utils.strutils import mask_dict_password
 from oslo_utils.strutils import mask_password
 from oslo_utils import timeutils
@@ -495,3 +497,52 @@ def mask_data(obj):
         return [mask_data(i) for i in obj]
     else:
         return mask_password(obj)
+
+
+def to_json_str(obj):
+    """Serializes an object into a JSON string.
+
+    :param obj: Object to serialize.
+    :return: JSON string.
+    """
+
+    if obj is None:
+        return None
+
+    def _fallback(value):
+        if inspect.isgenerator(value):
+            result = list(value)
+
+            # The result of the generator call may be again not primitive
+            # so we need to call "to_primitive" again with the same fallback
+            # function. Note that the endless recursion here is not a problem
+            # because "to_primitive" limits the depth for custom classes,
+            # if they are present in the object graph being traversed.
+            return jsonutils.to_primitive(
+                result,
+                convert_instances=True,
+                fallback=_fallback
+            )
+
+        return value
+
+    # We need to convert the root of the given object graph into
+    # a primitive by hand so that we also enable conversion of
+    # object of custom classes into primitives. Otherwise, they are
+    # ignored by the "json" lib.
+    return jsonutils.dumps(
+        jsonutils.to_primitive(obj, convert_instances=True, fallback=_fallback)
+    )
+
+
+def from_json_str(json_str):
+    """Reconstructs an object from a JSON string.
+
+    :param json_str: A JSON string.
+    :return: Deserialized object.
+    """
+
+    if json_str is None:
+        return None
+
+    return jsonutils.loads(json_str)

requirements.txt

@@ -5,4 +5,5 @@
 eventlet!=0.20.1,>=0.20.0 # MIT
 oslo.log>=3.36.0 # Apache-2.0
 pbr!=2.1.0,>=2.0.0 # Apache-2.0
-oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
+oslo.serialization>=2.21.1 # Apache-2.0
+yaql>=1.1.3 # Apache 2.0 License