dump() function added to DSL

Added a dsl-level yaql function 'dump' capable to serialize any given
MuranoPL object into one of three different formats identified by its
`serialization_type` argument:

* `Serializable` - a json-compliant notation with '?'-sections
                   describing type metadata including type name,
                   package and package name.
* 'Inline'       - a MuranoPL-compliant notation with dict keys being
                   instances of `MuranoType` class.
* `Mixed`        - similar to `Serializable` but type information is
                   not stringified and is present in '?'-sections as
                   objects of MuranoType class.

Function arguments also control whether object upcasting should be
honored or ignored.

Change-Id: Id36bb5daf9ebbdc42b09ad7bb956f51cfbf3c465

meta/io.murano/Classes/metadata/engine/Serialize.yaml

@@ -0,0 +1,25 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+Namespaces:
+  =: io.murano.metadata.engine
+
+Name: Serialize
+Usage: Meta
+Cardinality: One
+Inherited: true
+Applies:
+  - Property
+
+Properties:
+  as:
+    Contract: $.check($ in ['reference', 'copy'])

meta/io.murano/manifest.yaml

@@ -77,5 +77,6 @@ Classes:
   io.murano.metadata.Title: metadata/Title.yaml
   io.murano.metadata.forms.Hidden: metadata/forms/Hidden.yaml
   io.murano.metadata.forms.Section: metadata/forms/Section.yaml
+  io.murano.metadata.engine.Serialize: metadata/engine/Serialize.yaml
 
   io.murano.test.TestFixture: test/TestFixture.yaml

murano/dsl/murano_object.py

@@ -19,6 +19,7 @@ from murano.dsl import dsl
 from murano.dsl import dsl_types
 from murano.dsl import exceptions
 from murano.dsl import helpers
+from murano.dsl import serializer
 from murano.dsl import yaql_integration
 
 
@@ -244,10 +245,43 @@ class MuranoObject(dsl_types.MuranoObject):
         return '<{0}/{1} {2} ({3})>'.format(
             self.type.name, self.type.version, self.object_id, id(self))
 
-    def to_dictionary(self, include_hidden=False):
+    def to_dictionary(self, include_hidden=False,
+                      serialization_type=serializer.DumpTypes.Serializable,
+                      allow_refs=False):
+        context = helpers.get_context()
         result = {}
         for parent in self.__parents.values():
-            result.update(parent.to_dictionary(include_hidden))
+            result.update(parent.to_dictionary(
+                include_hidden, serializer.DumpTypes.Serializable,
+                allow_refs))
+        for property_name in self.type.properties:
+            if property_name in self.__properties:
+                spec = self.type.properties[property_name]
+                if (spec.usage != dsl_types.PropertyUsages.Runtime or
+                        include_hidden):
+                    prop_value = self.__properties[property_name]
+                    if isinstance(prop_value, MuranoObject) and allow_refs:
+                        meta = [m for m in spec.get_meta(context)
+                                if m.type.name == ('io.murano.metadata.'
+                                                   'engine.Serialize')]
+                        if meta and meta[0].get_property(
+                                'as', context) == 'reference':
+                            prop_value = prop_value.object_id
+                    result[property_name] = prop_value
+        if serialization_type == serializer.DumpTypes.Inline:
+            result.pop('?')
+            result = {
+                self.type: result,
+                'id': self.object_id,
+                'name': self.name
+            }
+        elif serialization_type == serializer.DumpTypes.Mixed:
+            result.update({'?': {
+                'type': self.type,
+                'id': self.object_id,
+                'name': self.name,
+            }})
+        else:
             result.update({'?': {
                 'type': self.type.name,
                 'id': self.object_id,
@@ -255,13 +289,4 @@ class MuranoObject(dsl_types.MuranoObject):
                 'classVersion': str(self.type.version),
                 'package': self.type.package.name
             }})
-        if include_hidden:
-            result.update(self.__properties)
-        else:
-            for property_name in self.type.properties:
-                if property_name in self.__properties:
-                    spec = self.type.properties[property_name]
-                    if spec.usage != dsl_types.PropertyUsages.Runtime:
-                        result[property_name] = self.__properties[
-                            property_name]
         return result

murano/dsl/serializer.py

@@ -21,24 +21,38 @@ from murano.dsl import dsl_types
 from murano.dsl import helpers
 
 
+class DumpTypes(object):
+    Serializable = 'Serializable'
+    Inline = 'Inline'
+    Mixed = 'Mixed'
+    All = {Serializable, Inline, Mixed}
+
+
 class ObjRef(object):
     def __init__(self, obj):
         self.ref_obj = obj
 
 
-def serialize(obj, executor):
+def serialize(obj, executor, serialization_type=DumpTypes.Serializable):
     with helpers.with_object_store(executor.object_store):
-        return serialize_model(obj, executor, True)[0]['Objects']
+        return serialize_model(
+            obj, executor, True,
+            make_copy=False,
+            serialize_attributes=False,
+            serialize_actions=False,
+            serialization_type=serialization_type)[0]['Objects']
 
 
 def _serialize_object(root_object, designer_attributes, allow_refs,
-                      executor):
+                      executor, serialize_actions=True,
+                      serialization_type=DumpTypes.Serializable):
     serialized_objects = set()
 
     obj = root_object
     while True:
         obj, need_another_pass = _pass12_serialize(
-            obj, None, serialized_objects, designer_attributes, executor)
+            obj, None, serialized_objects, designer_attributes, executor,
+            serialize_actions, serialization_type, allow_refs)
         if not need_another_pass:
             break
     tree = [obj]
@@ -46,7 +60,12 @@ def _serialize_object(root_object, designer_attributes, allow_refs,
     return tree[0], serialized_objects
 
 
-def serialize_model(root_object, executor, allow_refs=False):
+def serialize_model(root_object, executor,
+                    allow_refs=False,
+                    make_copy=True,
+                    serialize_attributes=True,
+                    serialize_actions=True,
+                    serialization_type=DumpTypes.Serializable):
     designer_attributes = executor.object_store.designer_attributes
 
     if root_object is None:
@@ -57,10 +76,15 @@ def serialize_model(root_object, executor, allow_refs=False):
     else:
         with helpers.with_object_store(executor.object_store):
             tree, serialized_objects = _serialize_object(
-                root_object, designer_attributes, allow_refs, executor)
+                root_object, designer_attributes, allow_refs, executor,
-            tree_copy, _ = _serialize_object(root_object, None, allow_refs,
+                serialize_actions, serialization_type)
-                                             executor)
+
-            attributes = executor.attribute_store.serialize(serialized_objects)
+            tree_copy = _serialize_object(
+                root_object, None, allow_refs, executor, serialize_actions,
+                serialization_type)[0] if make_copy else None
+
+            attributes = executor.attribute_store.serialize(
+                serialized_objects) if serialize_attributes else None
 
     return {
         'Objects': tree,
@@ -95,7 +119,8 @@ def _serialize_available_action(obj, current_actions, executor):
 
 
 def _pass12_serialize(value, parent, serialized_objects,
-                      designer_attributes_getter, executor):
+                      designer_attributes_getter, executor,
+                      serialize_actions, serialization_type, allow_refs):
     if isinstance(value, dsl.MuranoObjectInterface):
         value = value.object
     if isinstance(value, (six.string_types,
@@ -111,25 +136,42 @@ def _pass12_serialize(value, parent, serialized_objects,
         else:
             return value, False
     if isinstance(value, dsl_types.MuranoObject):
-        result = value.to_dictionary()
+
+        result = value.to_dictionary(
+            serialization_type=serialization_type, allow_refs=allow_refs)
         if designer_attributes_getter is not None:
-            result['?'].update(designer_attributes_getter(value.object_id))
+            if serialization_type == DumpTypes.Inline:
+                system_data = result
+            else:
+                system_data = result['?']
+            system_data.update(designer_attributes_getter(value.object_id))
+            if serialize_actions:
                 # deserialize and merge list of actions
-            result['?']['_actions'] = _serialize_available_action(
+                system_data['_actions'] = _serialize_available_action(
-                value, result['?'].get('_actions', {}), executor)
+                    value, system_data.get('_actions', {}), executor)
         serialized_objects.add(value.object_id)
         return _pass12_serialize(
             result, value, serialized_objects, designer_attributes_getter,
-            executor)
+            executor, serialize_actions, serialization_type, allow_refs)
     elif isinstance(value, utils.MappingType):
         result = {}
         need_another_pass = False
 
         for d_key, d_value in six.iteritems(value):
+            if (isinstance(d_key, dsl_types.MuranoType) and
+                    serialization_type == DumpTypes.Serializable):
                 result_key = str(d_key)
+            else:
+                result_key = d_key
+            if (result_key == 'type' and
+                    isinstance(d_value, dsl_types.MuranoType) and
+                    serialization_type == DumpTypes.Mixed):
+                result_value = d_value, False
+            else:
                 result_value = _pass12_serialize(
                     d_value, parent, serialized_objects,
-                designer_attributes_getter, executor)
+                    designer_attributes_getter, executor, serialize_actions,
+                    serialization_type, allow_refs)
             result[result_key] = result_value[0]
             if result_value[1]:
                 need_another_pass = True
@@ -140,7 +182,7 @@ def _pass12_serialize(value, parent, serialized_objects,
         for t in value:
             v, nmp = _pass12_serialize(
                 t, parent, serialized_objects, designer_attributes_getter,
-                executor)
+                executor, serialize_actions, serialization_type, allow_refs)
             if nmp:
                 need_another_pass = True
             result.append(v)

murano/dsl/yaql_functions.py

@@ -24,6 +24,7 @@ from murano.dsl import dsl
 from murano.dsl import dsl_types
 from murano.dsl import helpers
 from murano.dsl import reflection
+from murano.dsl import serializer
 
 
 @specs.parameter('object_', dsl.MuranoObjectParameter())
@@ -216,6 +217,11 @@ def is_instance_of(obj, type_):
     return type_.type.is_compatible(obj)
 
 
+def is_object(value):
+    return isinstance(value, (dsl_types.MuranoObject,
+                              dsl_types.MuranoTypeReference))
+
+
 @specs.name('call')
 @specs.parameter('name', yaqltypes.String())
 @specs.parameter('args', yaqltypes.Sequence())
@@ -237,6 +243,19 @@ def call_func(context, op_dot, base, name, args, kwargs,
         return base(context, name, args, kwargs, receiver)
 
 
+@specs.parameter('obj', dsl.MuranoObjectParameter(decorate=False))
+@specs.parameter('serialization_type', yaqltypes.String())
+@specs.parameter('ignore_upcasts', bool)
+def dump(obj, serialization_type=serializer.DumpTypes.Serializable,
+         ignore_upcasts=True):
+    if serialization_type not in serializer.DumpTypes.All:
+        raise ValueError('Invalid Serialization Type')
+    executor = helpers.get_executor()
+    if ignore_upcasts:
+        obj = obj.real_this
+    return serializer.serialize(obj, executor, serialization_type)
+
+
 def register(context, runtime_version):
     context.register_function(cast)
     context.register_function(new)
@@ -272,5 +291,6 @@ def register(context, runtime_version):
                 context.register_function(spec)
 
     context.register_function(type_from_name)
-
+    context.register_function(is_object)
+    context.register_function(dump)
     return context

murano/tests/unit/dsl/meta/TestDump.yaml

@@ -0,0 +1,107 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+
+Namespaces:
+  =: dumptests
+  std: io.murano
+  m: io.murano.metadata.engine
+--- # ------------------------------------------------------------------ # ---
+
+Name: DumpTarget1
+
+Properties:
+  foo:
+    Contract: $.string()
+
+  bar:
+    Contract:
+      - $.int()
+
+  baz:
+    Contract:
+      $.string(): $.int()
+
+
+Methods:
+  getOwner:
+    Body:
+      - Return: $.find(DumpTarget2).require()
+
+--- # ------------------------------------------------------------------ # ---
+
+Name: DumpTarget2
+
+Properties:
+  nested:
+    Usage: InOut
+    Contract: $.class(std:Object)
+
+  another:
+    Contract: $.class(DumpTarget1)
+
+  ref:
+    Usage: InOut
+    Contract: $.class(std:Object)
+
+--- # ------------------------------------------------------------------ # ---
+
+Name: DumpTarget3
+
+Properties:
+  a:
+    Meta:
+      - m:Serialize:
+          as: copy
+    Contract: $.class(DumpTarget1)
+  b:
+    Meta:
+      - m:Serialize:
+          as: reference
+    Contract: $.class(DumpTarget1)
+
+--- # ------------------------------------------------------------------ # ---
+Name: DumpTarget4
+Extends: DumpTarget1
+Properties:
+  qux:
+    Contract: $.string().notNull()
+
+--- # ------------------------------------------------------------------ # ---
+
+
+Name: TestDump
+
+Methods:
+  testDump:
+    Arguments:
+      - object:
+          Contract: $.class(std:Object).notNull()
+      - serializationType:
+          Contract: $.string().check($ in [Serializable, Mixed, Inline])
+          Default: 'Inline'
+    Body:
+      - Return: dump($object, $serializationType, true)
+
+  testDumpWithUpcast:
+    Arguments:
+      - object:
+          Contract: $.class(std:Object).notNull()
+      - doUpcast:
+          Contract: $.bool().notNull()
+      - passIgnoreUpcast:
+          Contract: $.bool().notNull()
+    Body:
+      - If: $doUpcast
+        Then:
+          - $object: $object.cast(DumpTarget1)
+      - Return: dump($object, Inline, $passIgnoreUpcast)

murano/tests/unit/dsl/test_dump.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+
+from murano.dsl import dsl_types
+from murano.tests.unit.dsl.foundation import object_model as om
+from murano.tests.unit.dsl.foundation import test_case
+
+
+class TestDump(test_case.DslTestCase):
+    def setUp(self):
+        super(TestDump, self).setUp()
+        self._runner = self.new_runner(om.Object('dumptests.TestDump'))
+
+    def test_dump_simple_inline(self):
+        source = om.Object('dumptests.DumpTarget1',
+                           foo='FOO', bar=[40, 41, 42], baz={'BAZ': 99})
+        result = self._runner.testDump(source, 'Inline')
+        self.assertIn('id', result)
+        res = self._get_body(result)
+        self.assertEqual('FOO', res['foo'])
+        self.assertEqual([40, 41, 42], res['bar'])
+        self.assertEqual({'BAZ': 99}, res['baz'])
+
+    def test_dump_simple_serializable(self):
+        source = om.Object('dumptests.DumpTarget1',
+                           foo='FOO', bar=[40, 41, 42], baz={'BAZ': 99})
+        result = self._runner.testDump(source, 'Serializable')
+        self.assertIn('?', result)
+        self.assertIn('classVersion', result['?'])
+        self.assertIn('package', result['?'])
+        self.assertEqual('dumptests.DumpTarget1', result['?']['type'])
+
+    def test_dump_simple_full_mixed(self):
+        source = om.Object('dumptests.DumpTarget1',
+                           foo='FOO', bar=[40, 41, 42], baz={'BAZ': 99})
+
+        result = self._runner.testDump(source, 'Mixed')
+        self.assertIn('?', result)
+        self.assertNotIn('classVersion', result['?'])
+        self.assertNotIn('package', result['?'])
+        self.assertIsInstance(result['?']['type'], dsl_types.MuranoType)
+        self.assertEqual('dumptests.DumpTarget1', result['?']['type'].name)
+
+    def test_nested(self):
+        n1 = om.Object('dumptests.DumpTarget1', foo='FOO')
+        n2 = om.Object('dumptests.DumpTarget1', foo='BAR')
+        n3 = om.Object('dumptests.DumpTarget1', foo='BAZ')
+        source = om.Object('dumptests.DumpTarget2',
+                           nested=n1, another=n2, ref=n3)
+        result = self._runner.testDump(source)
+        res = self._get_body(result)
+        self.assertIsNotNone(res['ref'])
+        self.assertIsNotNone(res['another'])
+        self.assertIsNotNone(res['nested'])
+        self.assertEqual('FOO', self._get_body(res['nested'])['foo'])
+        self.assertEqual('BAR', self._get_body(res['another'])['foo'])
+        self.assertEqual('BAZ', self._get_body(res['ref'])['foo'])
+
+    def test_same_ref_dump(self):
+        nested = om.Object('dumptests.DumpTarget1', foo='FOO')
+        source = om.Object('dumptests.DumpTarget2',
+                           nested=nested, another=nested, ref=nested)
+        result = self._runner.testDump(source)
+        res = self._get_body(result)
+        string_keys = [k for k in res.keys()
+                       if isinstance(res[k], six.string_types)]
+        obj_keys = [k for k in res.keys()
+                    if isinstance(res[k], dict)]
+        self.assertEqual(2, len(string_keys))
+        self.assertEqual(1, len(obj_keys))
+        obj = self._get_body(res[obj_keys[0]])
+        self.assertEqual('FOO', obj['foo'])
+        for ref_id in string_keys:
+            self.assertEqual(res[obj_keys[0]]['id'], res[ref_id])
+
+    def test_dump_with_meta_attributes(self):
+        n1 = om.Object('dumptests.DumpTarget1', foo='FOO')
+        n2 = om.Object('dumptests.DumpTarget1', foo='Bar')
+        source = om.Object('dumptests.DumpTarget3', a=n1, b=n2)
+        result = self._runner.testDump(source)
+        res = self._get_body(result)
+        self._get_body(res['a'])
+        self.assertIsInstance(res['b'], six.string_types)
+
+    def test_dump_with_inheritance(self):
+        source = om.Object('dumptests.DumpTarget4', foo='FOO', qux='QUX')
+        result = self._runner.testDump(source)
+        res = self._get_body(result)
+        self.assertEqual('FOO', res['foo'])
+        self.assertEqual('QUX', res['qux'])
+
+    def test_dump_with_inheritance_upcast_ignored(self):
+        source = om.Object('dumptests.DumpTarget4', foo='FOO', qux='QUX')
+        result = self._runner.testDumpWithUpcast(source, True, True)
+        res = self._get_body(result)
+        self.assertEqual('FOO', res['foo'])
+        self.assertEqual('QUX', res['qux'])
+
+    def test_dump_with_inheritance_upcast_allowed(self):
+        source = om.Object('dumptests.DumpTarget4', foo='FOO', qux='QUX')
+        result = self._runner.testDumpWithUpcast(source, True, False)
+        res = self._get_body(result)
+        self.assertEqual('FOO', res['foo'])
+        self.assertNotIn('qux', res)
+
+    def _get_body(self, obj):
+        body_key = [k for k in obj.keys() if k not in ('id', 'name')][0]
+        self.assertIsInstance(body_key, dsl_types.MuranoType)
+        return obj[body_key]