From 79b2a2b93540f267d5561ecc5e8b126d5ec19a97 Mon Sep 17 00:00:00 2001 From: Alexander Tivelkov Date: Tue, 19 Jul 2016 14:15:38 +0300 Subject: [PATCH] dump() function added to DSL Added a dsl-level yaql function 'dump' capable to serialize any given MuranoPL object into one of three different formats identified by its `serialization_type` argument: * `Serializable` - a json-compliant notation with '?'-sections describing type metadata including type name, package and package name. * 'Inline' - a MuranoPL-compliant notation with dict keys being instances of `MuranoType` class. * `Mixed` - similar to `Serializable` but type information is not stringified and is present in '?'-sections as objects of MuranoType class. Function arguments also control whether object upcasting should be honored or ignored. Change-Id: Id36bb5daf9ebbdc42b09ad7bb956f51cfbf3c465 --- .../Classes/metadata/engine/Serialize.yaml | 25 ++++ meta/io.murano/manifest.yaml | 1 + murano/dsl/murano_object.py | 59 ++++++--- murano/dsl/serializer.py | 84 +++++++++--- murano/dsl/yaql_functions.py | 22 +++- murano/tests/unit/dsl/meta/TestDump.yaml | 107 +++++++++++++++ murano/tests/unit/dsl/test_dump.py | 122 ++++++++++++++++++ 7 files changed, 381 insertions(+), 39 deletions(-) create mode 100644 meta/io.murano/Classes/metadata/engine/Serialize.yaml create mode 100644 murano/tests/unit/dsl/meta/TestDump.yaml create mode 100644 murano/tests/unit/dsl/test_dump.py diff --git a/meta/io.murano/Classes/metadata/engine/Serialize.yaml b/meta/io.murano/Classes/metadata/engine/Serialize.yaml new file mode 100644 index 00000000..e1d2b20c --- /dev/null +++ b/meta/io.murano/Classes/metadata/engine/Serialize.yaml @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +Namespaces: + =: io.murano.metadata.engine + +Name: Serialize +Usage: Meta +Cardinality: One +Inherited: true +Applies: + - Property + +Properties: + as: + Contract: $.check($ in ['reference', 'copy']) diff --git a/meta/io.murano/manifest.yaml b/meta/io.murano/manifest.yaml index fba4d322..840a989c 100644 --- a/meta/io.murano/manifest.yaml +++ b/meta/io.murano/manifest.yaml @@ -77,5 +77,6 @@ Classes: io.murano.metadata.Title: metadata/Title.yaml io.murano.metadata.forms.Hidden: metadata/forms/Hidden.yaml io.murano.metadata.forms.Section: metadata/forms/Section.yaml + io.murano.metadata.engine.Serialize: metadata/engine/Serialize.yaml io.murano.test.TestFixture: test/TestFixture.yaml diff --git a/murano/dsl/murano_object.py b/murano/dsl/murano_object.py index d19830dd..97b847db 100644 --- a/murano/dsl/murano_object.py +++ b/murano/dsl/murano_object.py @@ -19,6 +19,7 @@ from murano.dsl import dsl from murano.dsl import dsl_types from murano.dsl import exceptions from murano.dsl import helpers +from murano.dsl import serializer from murano.dsl import yaql_integration @@ -244,24 +245,48 @@ class MuranoObject(dsl_types.MuranoObject): return '<{0}/{1} {2} ({3})>'.format( self.type.name, self.type.version, self.object_id, id(self)) - def to_dictionary(self, include_hidden=False): + def to_dictionary(self, include_hidden=False, + serialization_type=serializer.DumpTypes.Serializable, + allow_refs=False): + context = helpers.get_context() result = {} for parent in self.__parents.values(): - result.update(parent.to_dictionary(include_hidden)) - result.update({'?': { - 'type': self.type.name, - 'id': self.object_id, - 'name': self.name, - 'classVersion': str(self.type.version), - 'package': self.type.package.name - }}) - if include_hidden: - result.update(self.__properties) + result.update(parent.to_dictionary( + include_hidden, serializer.DumpTypes.Serializable, + allow_refs)) + for property_name in self.type.properties: + if property_name in self.__properties: + spec = self.type.properties[property_name] + if (spec.usage != dsl_types.PropertyUsages.Runtime or + include_hidden): + prop_value = self.__properties[property_name] + if isinstance(prop_value, MuranoObject) and allow_refs: + meta = [m for m in spec.get_meta(context) + if m.type.name == ('io.murano.metadata.' + 'engine.Serialize')] + if meta and meta[0].get_property( + 'as', context) == 'reference': + prop_value = prop_value.object_id + result[property_name] = prop_value + if serialization_type == serializer.DumpTypes.Inline: + result.pop('?') + result = { + self.type: result, + 'id': self.object_id, + 'name': self.name + } + elif serialization_type == serializer.DumpTypes.Mixed: + result.update({'?': { + 'type': self.type, + 'id': self.object_id, + 'name': self.name, + }}) else: - for property_name in self.type.properties: - if property_name in self.__properties: - spec = self.type.properties[property_name] - if spec.usage != dsl_types.PropertyUsages.Runtime: - result[property_name] = self.__properties[ - property_name] + result.update({'?': { + 'type': self.type.name, + 'id': self.object_id, + 'name': self.name, + 'classVersion': str(self.type.version), + 'package': self.type.package.name + }}) return result diff --git a/murano/dsl/serializer.py b/murano/dsl/serializer.py index 324e856b..82cadd01 100644 --- a/murano/dsl/serializer.py +++ b/murano/dsl/serializer.py @@ -21,24 +21,38 @@ from murano.dsl import dsl_types from murano.dsl import helpers +class DumpTypes(object): + Serializable = 'Serializable' + Inline = 'Inline' + Mixed = 'Mixed' + All = {Serializable, Inline, Mixed} + + class ObjRef(object): def __init__(self, obj): self.ref_obj = obj -def serialize(obj, executor): +def serialize(obj, executor, serialization_type=DumpTypes.Serializable): with helpers.with_object_store(executor.object_store): - return serialize_model(obj, executor, True)[0]['Objects'] + return serialize_model( + obj, executor, True, + make_copy=False, + serialize_attributes=False, + serialize_actions=False, + serialization_type=serialization_type)[0]['Objects'] def _serialize_object(root_object, designer_attributes, allow_refs, - executor): + executor, serialize_actions=True, + serialization_type=DumpTypes.Serializable): serialized_objects = set() obj = root_object while True: obj, need_another_pass = _pass12_serialize( - obj, None, serialized_objects, designer_attributes, executor) + obj, None, serialized_objects, designer_attributes, executor, + serialize_actions, serialization_type, allow_refs) if not need_another_pass: break tree = [obj] @@ -46,7 +60,12 @@ def _serialize_object(root_object, designer_attributes, allow_refs, return tree[0], serialized_objects -def serialize_model(root_object, executor, allow_refs=False): +def serialize_model(root_object, executor, + allow_refs=False, + make_copy=True, + serialize_attributes=True, + serialize_actions=True, + serialization_type=DumpTypes.Serializable): designer_attributes = executor.object_store.designer_attributes if root_object is None: @@ -57,10 +76,15 @@ def serialize_model(root_object, executor, allow_refs=False): else: with helpers.with_object_store(executor.object_store): tree, serialized_objects = _serialize_object( - root_object, designer_attributes, allow_refs, executor) - tree_copy, _ = _serialize_object(root_object, None, allow_refs, - executor) - attributes = executor.attribute_store.serialize(serialized_objects) + root_object, designer_attributes, allow_refs, executor, + serialize_actions, serialization_type) + + tree_copy = _serialize_object( + root_object, None, allow_refs, executor, serialize_actions, + serialization_type)[0] if make_copy else None + + attributes = executor.attribute_store.serialize( + serialized_objects) if serialize_attributes else None return { 'Objects': tree, @@ -95,7 +119,8 @@ def _serialize_available_action(obj, current_actions, executor): def _pass12_serialize(value, parent, serialized_objects, - designer_attributes_getter, executor): + designer_attributes_getter, executor, + serialize_actions, serialization_type, allow_refs): if isinstance(value, dsl.MuranoObjectInterface): value = value.object if isinstance(value, (six.string_types, @@ -111,25 +136,42 @@ def _pass12_serialize(value, parent, serialized_objects, else: return value, False if isinstance(value, dsl_types.MuranoObject): - result = value.to_dictionary() + + result = value.to_dictionary( + serialization_type=serialization_type, allow_refs=allow_refs) if designer_attributes_getter is not None: - result['?'].update(designer_attributes_getter(value.object_id)) - # deserialize and merge list of actions - result['?']['_actions'] = _serialize_available_action( - value, result['?'].get('_actions', {}), executor) + if serialization_type == DumpTypes.Inline: + system_data = result + else: + system_data = result['?'] + system_data.update(designer_attributes_getter(value.object_id)) + if serialize_actions: + # deserialize and merge list of actions + system_data['_actions'] = _serialize_available_action( + value, system_data.get('_actions', {}), executor) serialized_objects.add(value.object_id) return _pass12_serialize( result, value, serialized_objects, designer_attributes_getter, - executor) + executor, serialize_actions, serialization_type, allow_refs) elif isinstance(value, utils.MappingType): result = {} need_another_pass = False for d_key, d_value in six.iteritems(value): - result_key = str(d_key) - result_value = _pass12_serialize( - d_value, parent, serialized_objects, - designer_attributes_getter, executor) + if (isinstance(d_key, dsl_types.MuranoType) and + serialization_type == DumpTypes.Serializable): + result_key = str(d_key) + else: + result_key = d_key + if (result_key == 'type' and + isinstance(d_value, dsl_types.MuranoType) and + serialization_type == DumpTypes.Mixed): + result_value = d_value, False + else: + result_value = _pass12_serialize( + d_value, parent, serialized_objects, + designer_attributes_getter, executor, serialize_actions, + serialization_type, allow_refs) result[result_key] = result_value[0] if result_value[1]: need_another_pass = True @@ -140,7 +182,7 @@ def _pass12_serialize(value, parent, serialized_objects, for t in value: v, nmp = _pass12_serialize( t, parent, serialized_objects, designer_attributes_getter, - executor) + executor, serialize_actions, serialization_type, allow_refs) if nmp: need_another_pass = True result.append(v) diff --git a/murano/dsl/yaql_functions.py b/murano/dsl/yaql_functions.py index 0ac8aaef..4b9c880f 100644 --- a/murano/dsl/yaql_functions.py +++ b/murano/dsl/yaql_functions.py @@ -24,6 +24,7 @@ from murano.dsl import dsl from murano.dsl import dsl_types from murano.dsl import helpers from murano.dsl import reflection +from murano.dsl import serializer @specs.parameter('object_', dsl.MuranoObjectParameter()) @@ -216,6 +217,11 @@ def is_instance_of(obj, type_): return type_.type.is_compatible(obj) +def is_object(value): + return isinstance(value, (dsl_types.MuranoObject, + dsl_types.MuranoTypeReference)) + + @specs.name('call') @specs.parameter('name', yaqltypes.String()) @specs.parameter('args', yaqltypes.Sequence()) @@ -237,6 +243,19 @@ def call_func(context, op_dot, base, name, args, kwargs, return base(context, name, args, kwargs, receiver) +@specs.parameter('obj', dsl.MuranoObjectParameter(decorate=False)) +@specs.parameter('serialization_type', yaqltypes.String()) +@specs.parameter('ignore_upcasts', bool) +def dump(obj, serialization_type=serializer.DumpTypes.Serializable, + ignore_upcasts=True): + if serialization_type not in serializer.DumpTypes.All: + raise ValueError('Invalid Serialization Type') + executor = helpers.get_executor() + if ignore_upcasts: + obj = obj.real_this + return serializer.serialize(obj, executor, serialization_type) + + def register(context, runtime_version): context.register_function(cast) context.register_function(new) @@ -272,5 +291,6 @@ def register(context, runtime_version): context.register_function(spec) context.register_function(type_from_name) - + context.register_function(is_object) + context.register_function(dump) return context diff --git a/murano/tests/unit/dsl/meta/TestDump.yaml b/murano/tests/unit/dsl/meta/TestDump.yaml new file mode 100644 index 00000000..3dd85bf2 --- /dev/null +++ b/murano/tests/unit/dsl/meta/TestDump.yaml @@ -0,0 +1,107 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +Namespaces: + =: dumptests + std: io.murano + m: io.murano.metadata.engine +--- # ------------------------------------------------------------------ # --- + +Name: DumpTarget1 + +Properties: + foo: + Contract: $.string() + + bar: + Contract: + - $.int() + + baz: + Contract: + $.string(): $.int() + + +Methods: + getOwner: + Body: + - Return: $.find(DumpTarget2).require() + +--- # ------------------------------------------------------------------ # --- + +Name: DumpTarget2 + +Properties: + nested: + Usage: InOut + Contract: $.class(std:Object) + + another: + Contract: $.class(DumpTarget1) + + ref: + Usage: InOut + Contract: $.class(std:Object) + +--- # ------------------------------------------------------------------ # --- + +Name: DumpTarget3 + +Properties: + a: + Meta: + - m:Serialize: + as: copy + Contract: $.class(DumpTarget1) + b: + Meta: + - m:Serialize: + as: reference + Contract: $.class(DumpTarget1) + +--- # ------------------------------------------------------------------ # --- +Name: DumpTarget4 +Extends: DumpTarget1 +Properties: + qux: + Contract: $.string().notNull() + +--- # ------------------------------------------------------------------ # --- + + +Name: TestDump + +Methods: + testDump: + Arguments: + - object: + Contract: $.class(std:Object).notNull() + - serializationType: + Contract: $.string().check($ in [Serializable, Mixed, Inline]) + Default: 'Inline' + Body: + - Return: dump($object, $serializationType, true) + + testDumpWithUpcast: + Arguments: + - object: + Contract: $.class(std:Object).notNull() + - doUpcast: + Contract: $.bool().notNull() + - passIgnoreUpcast: + Contract: $.bool().notNull() + Body: + - If: $doUpcast + Then: + - $object: $object.cast(DumpTarget1) + - Return: dump($object, Inline, $passIgnoreUpcast) diff --git a/murano/tests/unit/dsl/test_dump.py b/murano/tests/unit/dsl/test_dump.py new file mode 100644 index 00000000..95e5f9e5 --- /dev/null +++ b/murano/tests/unit/dsl/test_dump.py @@ -0,0 +1,122 @@ +# Copyright (c) 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import six + +from murano.dsl import dsl_types +from murano.tests.unit.dsl.foundation import object_model as om +from murano.tests.unit.dsl.foundation import test_case + + +class TestDump(test_case.DslTestCase): + def setUp(self): + super(TestDump, self).setUp() + self._runner = self.new_runner(om.Object('dumptests.TestDump')) + + def test_dump_simple_inline(self): + source = om.Object('dumptests.DumpTarget1', + foo='FOO', bar=[40, 41, 42], baz={'BAZ': 99}) + result = self._runner.testDump(source, 'Inline') + self.assertIn('id', result) + res = self._get_body(result) + self.assertEqual('FOO', res['foo']) + self.assertEqual([40, 41, 42], res['bar']) + self.assertEqual({'BAZ': 99}, res['baz']) + + def test_dump_simple_serializable(self): + source = om.Object('dumptests.DumpTarget1', + foo='FOO', bar=[40, 41, 42], baz={'BAZ': 99}) + result = self._runner.testDump(source, 'Serializable') + self.assertIn('?', result) + self.assertIn('classVersion', result['?']) + self.assertIn('package', result['?']) + self.assertEqual('dumptests.DumpTarget1', result['?']['type']) + + def test_dump_simple_full_mixed(self): + source = om.Object('dumptests.DumpTarget1', + foo='FOO', bar=[40, 41, 42], baz={'BAZ': 99}) + + result = self._runner.testDump(source, 'Mixed') + self.assertIn('?', result) + self.assertNotIn('classVersion', result['?']) + self.assertNotIn('package', result['?']) + self.assertIsInstance(result['?']['type'], dsl_types.MuranoType) + self.assertEqual('dumptests.DumpTarget1', result['?']['type'].name) + + def test_nested(self): + n1 = om.Object('dumptests.DumpTarget1', foo='FOO') + n2 = om.Object('dumptests.DumpTarget1', foo='BAR') + n3 = om.Object('dumptests.DumpTarget1', foo='BAZ') + source = om.Object('dumptests.DumpTarget2', + nested=n1, another=n2, ref=n3) + result = self._runner.testDump(source) + res = self._get_body(result) + self.assertIsNotNone(res['ref']) + self.assertIsNotNone(res['another']) + self.assertIsNotNone(res['nested']) + self.assertEqual('FOO', self._get_body(res['nested'])['foo']) + self.assertEqual('BAR', self._get_body(res['another'])['foo']) + self.assertEqual('BAZ', self._get_body(res['ref'])['foo']) + + def test_same_ref_dump(self): + nested = om.Object('dumptests.DumpTarget1', foo='FOO') + source = om.Object('dumptests.DumpTarget2', + nested=nested, another=nested, ref=nested) + result = self._runner.testDump(source) + res = self._get_body(result) + string_keys = [k for k in res.keys() + if isinstance(res[k], six.string_types)] + obj_keys = [k for k in res.keys() + if isinstance(res[k], dict)] + self.assertEqual(2, len(string_keys)) + self.assertEqual(1, len(obj_keys)) + obj = self._get_body(res[obj_keys[0]]) + self.assertEqual('FOO', obj['foo']) + for ref_id in string_keys: + self.assertEqual(res[obj_keys[0]]['id'], res[ref_id]) + + def test_dump_with_meta_attributes(self): + n1 = om.Object('dumptests.DumpTarget1', foo='FOO') + n2 = om.Object('dumptests.DumpTarget1', foo='Bar') + source = om.Object('dumptests.DumpTarget3', a=n1, b=n2) + result = self._runner.testDump(source) + res = self._get_body(result) + self._get_body(res['a']) + self.assertIsInstance(res['b'], six.string_types) + + def test_dump_with_inheritance(self): + source = om.Object('dumptests.DumpTarget4', foo='FOO', qux='QUX') + result = self._runner.testDump(source) + res = self._get_body(result) + self.assertEqual('FOO', res['foo']) + self.assertEqual('QUX', res['qux']) + + def test_dump_with_inheritance_upcast_ignored(self): + source = om.Object('dumptests.DumpTarget4', foo='FOO', qux='QUX') + result = self._runner.testDumpWithUpcast(source, True, True) + res = self._get_body(result) + self.assertEqual('FOO', res['foo']) + self.assertEqual('QUX', res['qux']) + + def test_dump_with_inheritance_upcast_allowed(self): + source = om.Object('dumptests.DumpTarget4', foo='FOO', qux='QUX') + result = self._runner.testDumpWithUpcast(source, True, False) + res = self._get_body(result) + self.assertEqual('FOO', res['foo']) + self.assertNotIn('qux', res) + + def _get_body(self, obj): + body_key = [k for k in obj.keys() if k not in ('id', 'name')][0] + self.assertIsInstance(body_key, dsl_types.MuranoType) + return obj[body_key]