diff --git a/murano/dsl/constants.py b/murano/dsl/constants.py index f9927bdd..438ec0e3 100644 --- a/murano/dsl/constants.py +++ b/murano/dsl/constants.py @@ -14,6 +14,7 @@ import semantic_version + EXPRESSION_MEMORY_QUOTA = 512 * 1024 ITERATORS_LIMIT = 2000 @@ -49,6 +50,9 @@ TL_CONTEXT = '__murano_context' TL_ID = '__thread_id' TL_OBJECT_STORE = '__murano_object_store' TL_SESSION = '__murano_execution_session' +TL_CONTRACT_PASSKEY = '__murano_contract_passkey' +TL_OBJECTS_DRY_RUN = '__murano_objects_dry_run' + RUNTIME_VERSION_1_0 = semantic_version.Version('1.0.0') RUNTIME_VERSION_1_1 = semantic_version.Version('1.1.0') diff --git a/murano/dsl/dsl_types.py b/murano/dsl/dsl_types.py index 8182712c..333a4d69 100644 --- a/murano/dsl/dsl_types.py +++ b/murano/dsl/dsl_types.py @@ -71,6 +71,13 @@ class MethodArgumentUsages(object): All = {Standard, VarArgs, KwArgs} +class DumpTypes(object): + Serializable = 'Serializable' + Inline = 'Inline' + Mixed = 'Mixed' + All = {Serializable, Inline, Mixed} + + class MuranoType(object): pass diff --git a/murano/dsl/helpers.py b/murano/dsl/helpers.py index 46c53f34..7eed7509 100644 --- a/murano/dsl/helpers.py +++ b/murano/dsl/helpers.py @@ -223,6 +223,16 @@ def get_class(name, context=None): return murano_type.package.find_class(name) +def get_contract_passkey(): + current_thread = eventlet.greenthread.getcurrent() + return getattr(current_thread, constants.TL_CONTRACT_PASSKEY, None) + + +def is_objects_dry_run_mode(): + current_thread = eventlet.greenthread.getcurrent() + return bool(getattr(current_thread, constants.TL_OBJECTS_DRY_RUN, False)) + + def get_current_thread_id(): global _threads_sequencer @@ -562,6 +572,34 @@ def parse_object_definition(spec, scope_type, context): } +def assemble_object_definition(parsed, model_format=dsl_types.DumpTypes.Mixed): + if model_format == dsl_types.DumpTypes.Inline: + result = { + parsed['type']: parsed['properties'], + 'id': parsed['id'], + 'name': parsed['name'] + } + result.update(parsed['extra']) + return result + result = parsed['properties'] + header = { + 'id': parsed['id'], + 'name': parsed['name'] + } + header.update(parsed['extra']) + result['?'] = header + if model_format == dsl_types.DumpTypes.Mixed: + header['type'] = parsed['type'] + return result + elif model_format == dsl_types.DumpTypes.Serializable: + cls = parsed['type'] + if cls: + header['type'] = format_type_string(cls) + return result + else: + raise ValueError('Invalid Serialization Type') + + def function(c): if hasattr(c, 'im_func'): return c.im_func @@ -606,3 +644,17 @@ def format_type_string(type_obj): type_obj.name, type_obj.version, type_obj.package.name) else: raise ValueError('Invalid argument') + + +def patch_dict(dct, path, value): + parts = path.split('.') + for i in range(len(parts) - 1): + if not isinstance(dct, dict): + dct = None + break + dct = dct.get(parts[i]) + if isinstance(dct, dict): + if value is yaqlutils.NO_VALUE: + dct.pop(parts[-1]) + else: + dct[parts[-1]] = value diff --git a/murano/dsl/murano_object.py b/murano/dsl/murano_object.py index 7ac2e5d0..fa8a6330 100644 --- a/murano/dsl/murano_object.py +++ b/murano/dsl/murano_object.py @@ -19,7 +19,6 @@ from murano.dsl import dsl from murano.dsl import dsl_types from murano.dsl import exceptions from murano.dsl import helpers -from murano.dsl import serializer from murano.dsl import yaql_integration @@ -143,7 +142,8 @@ class MuranoObject(dsl_types.MuranoObject): init.invoke(self.real_this, (), init_args, context.create_child_context()) - if not object_store.initializing and init: + if (not object_store.initializing and init + and not helpers.is_objects_dry_run_mode()): yield run_init @property @@ -254,13 +254,13 @@ class MuranoObject(dsl_types.MuranoObject): self.type.name, self.type.version, self.object_id, id(self)) def to_dictionary(self, include_hidden=False, - serialization_type=serializer.DumpTypes.Serializable, + serialization_type=dsl_types.DumpTypes.Serializable, allow_refs=False): context = helpers.get_context() result = {} for parent in self.__parents.values(): result.update(parent.to_dictionary( - include_hidden, serializer.DumpTypes.Serializable, + include_hidden, dsl_types.DumpTypes.Serializable, allow_refs)) for property_name in self.type.properties: if property_name in self.__properties: @@ -276,14 +276,14 @@ class MuranoObject(dsl_types.MuranoObject): 'as', context) == 'reference': prop_value = prop_value.object_id result[property_name] = prop_value - if serialization_type == serializer.DumpTypes.Inline: + if serialization_type == dsl_types.DumpTypes.Inline: result.pop('?') result = { self.type: result, 'id': self.object_id, 'name': self.name } - elif serialization_type == serializer.DumpTypes.Mixed: + elif serialization_type == dsl_types.DumpTypes.Mixed: result.update({'?': { 'type': self.type, 'id': self.object_id, diff --git a/murano/dsl/object_store.py b/murano/dsl/object_store.py index 1c1f517f..a89f35c6 100644 --- a/murano/dsl/object_store.py +++ b/murano/dsl/object_store.py @@ -58,7 +58,8 @@ class ObjectStore(object): scope_type=None, context=None, keep_ids=False): # do the object model load in a temporary object store and copy # loaded objects here after that - model_store = InitializationObjectStore(owner, self, keep_ids) + model_store = InitializationObjectStore( + owner, self, keep_ids) with helpers.with_object_store(model_store): result = model_store.load( value, owner, scope_type=scope_type, @@ -111,45 +112,44 @@ class InitializationObjectStore(ObjectStore): if not parsed: raise ValueError('Invalid object representation format') - try: - if owner is self._root_owner: - self._initializing = True + if owner is self._root_owner: + self._initializing = True - class_obj = parsed['type'] or default_type - if not class_obj: - raise ValueError( - 'Invalid object representation: ' - 'no type information was provided') - if isinstance(class_obj, dsl_types.MuranoTypeReference): - class_obj = class_obj.type - object_id = parsed['id'] - obj = None if object_id is None else self._store.get(object_id) - if not obj: - obj = murano_object.MuranoObject( - class_obj, helpers.weak_proxy(owner), - name=parsed['name'], - object_id=object_id if self._keep_ids else None) - self.put(obj, object_id or obj.object_id) + class_obj = parsed['type'] or default_type + if not class_obj: + raise ValueError( + 'Invalid object representation: ' + 'no type information was provided') + if isinstance(class_obj, dsl_types.MuranoTypeReference): + class_obj = class_obj.type + object_id = parsed['id'] + obj = None if object_id is None else self._store.get(object_id) + if not obj: + obj = murano_object.MuranoObject( + class_obj, helpers.weak_proxy(owner), + name=parsed['name'], + object_id=object_id if self._keep_ids else None) + self.put(obj, object_id or obj.object_id) - system_value = ObjectStore._get_designer_attributes( - parsed['extra']) - self._designer_attributes_store[object_id] = system_value + system_value = ObjectStore._get_designer_attributes( + parsed['extra']) + self._designer_attributes_store[object_id] = system_value - if context is None: - context = self.executor.create_object_context(obj) + if context is None: + context = self.executor.create_object_context(obj) - def run_initialize(): - self._initializers.extend( - obj.initialize(context, parsed['properties'])) + def run_initialize(): + self._initializers.extend( + obj.initialize(context, parsed['properties'])) + run_initialize() + if owner is self._root_owner: + self._initializing = False run_initialize() - if owner is self._root_owner: - self._initializing = False - run_initialize() - finally: - if owner is self._root_owner: - with helpers.with_object_store(self.parent_store): - for fn in self._initializers: - fn() + + if owner is self._root_owner: + with helpers.with_object_store(self.parent_store): + for fn in self._initializers: + fn() return obj diff --git a/murano/dsl/schema_generator.py b/murano/dsl/schema_generator.py index c444fdde..c727c52f 100644 --- a/murano/dsl/schema_generator.py +++ b/murano/dsl/schema_generator.py @@ -128,10 +128,11 @@ def prepare_context(exc, cls): context.register_function(bool_) context.register_function(not_null) context.register_function(check) - context.register_function(class_factory(context)) context.register_function(owned) context.register_function(not_owned) context.register_function(finalize) + for fn in class_factory(context): + context.register_function(fn) return context @@ -295,7 +296,24 @@ def class_factory(context): 'muranoType': name.type.name }) - return class_ + @specs.parameter('schema', Schema) + @specs.parameter('type_', dsl.MuranoTypeParameter( + nullable=False, context=context)) + @specs.parameter('default_type', dsl.MuranoTypeParameter( + nullable=True, context=context)) + @specs.parameter('version_spec', yaqltypes.String(True)) + @specs.parameter( + 'exclude_properties', yaqltypes.Sequence(nullable=True)) + @specs.method + def template(schema, type_, exclude_properties=None, + default_type=None, version_spec=None): + result = class_(schema, type_, default_type, version_spec) + result.data['owned'] = True + if exclude_properties: + result.data['excludedProperties'] = exclude_properties + return result + + return class_, template @specs.parameter('schema', Schema) diff --git a/murano/dsl/serializer.py b/murano/dsl/serializer.py index ecee673e..1409afd4 100644 --- a/murano/dsl/serializer.py +++ b/murano/dsl/serializer.py @@ -21,19 +21,13 @@ from murano.dsl import dsl_types from murano.dsl import helpers -class DumpTypes(object): - Serializable = 'Serializable' - Inline = 'Inline' - Mixed = 'Mixed' - All = {Serializable, Inline, Mixed} - - class ObjRef(object): def __init__(self, obj): self.ref_obj = obj -def serialize(obj, executor, serialization_type=DumpTypes.Serializable): +def serialize(obj, executor, + serialization_type=dsl_types.DumpTypes.Serializable): with helpers.with_object_store(executor.object_store): return serialize_model( obj, executor, True, @@ -45,7 +39,7 @@ def serialize(obj, executor, serialization_type=DumpTypes.Serializable): def _serialize_object(root_object, designer_attributes, allow_refs, executor, serialize_actions=True, - serialization_type=DumpTypes.Serializable): + serialization_type=dsl_types.DumpTypes.Serializable): serialized_objects = set() obj = root_object @@ -65,7 +59,7 @@ def serialize_model(root_object, executor, make_copy=True, serialize_attributes=True, serialize_actions=True, - serialization_type=DumpTypes.Serializable): + serialization_type=dsl_types.DumpTypes.Serializable): designer_attributes = executor.object_store.designer_attributes if root_object is None: @@ -138,11 +132,13 @@ def _pass12_serialize(value, parent, serialized_objects, if isinstance(value, (dsl_types.MuranoType, dsl_types.MuranoTypeReference)): return helpers.format_type_string(value), False + if value is helpers.get_contract_passkey(): + return value, False if isinstance(value, dsl_types.MuranoObject): result = value.to_dictionary( serialization_type=serialization_type, allow_refs=allow_refs) if designer_attributes_getter is not None: - if serialization_type == DumpTypes.Inline: + if serialization_type == dsl_types.DumpTypes.Inline: system_data = result else: system_data = result['?'] @@ -161,13 +157,13 @@ def _pass12_serialize(value, parent, serialized_objects, for d_key, d_value in six.iteritems(value): if (isinstance(d_key, dsl_types.MuranoType) and - serialization_type == DumpTypes.Serializable): + serialization_type == dsl_types.DumpTypes.Serializable): result_key = str(d_key) else: result_key = d_key if (result_key == 'type' and isinstance(d_value, dsl_types.MuranoType) and - serialization_type == DumpTypes.Mixed): + serialization_type == dsl_types.DumpTypes.Mixed): result_value = d_value, False else: result_value = _pass12_serialize( diff --git a/murano/dsl/type_scheme.py b/murano/dsl/type_scheme.py index 621a8ff3..dcaccd61 100644 --- a/murano/dsl/type_scheme.py +++ b/murano/dsl/type_scheme.py @@ -17,10 +17,12 @@ from yaql.language import specs from yaql.language import utils from yaql.language import yaqltypes +from murano.dsl import constants from murano.dsl import dsl from murano.dsl import dsl_types from murano.dsl import exceptions from murano.dsl import helpers +from murano.dsl import serializer class TypeScheme(object): @@ -187,9 +189,69 @@ class TypeScheme(object): version_spec or helpers.get_type(root_context)): raise exceptions.ContractViolationException( 'Object of type {0} is not compatible with ' - 'requested type {1}'.format(obj.type.name, name)) + 'requested type {1}'.format(obj.type.name, murano_class)) return obj + @specs.parameter('type_', dsl.MuranoTypeParameter( + nullable=False, context=root_context)) + @specs.parameter('default_type', dsl.MuranoTypeParameter( + nullable=True, context=root_context)) + @specs.parameter('value', nullable=True) + @specs.parameter('version_spec', yaqltypes.String(True)) + @specs.parameter( + 'exclude_properties', yaqltypes.Sequence(nullable=True)) + @specs.method + def template(engine, value, type_, exclude_properties=None, + default_type=None, version_spec=None): + object_store = helpers.get_object_store() + passkey = None + if not default_type: + default_type = type_ + murano_class = type_.type + if value is None: + return None + if isinstance(value, dsl_types.MuranoObject): + obj = value + elif isinstance(value, dsl_types.MuranoObjectInterface): + obj = value.object + elif isinstance(value, utils.MappingType): + passkey = utils.create_marker('') + if exclude_properties: + parsed = helpers.parse_object_definition( + value, calling_type, context) + props = dsl.to_mutable(parsed['properties'], engine) + for p in exclude_properties: + helpers.patch_dict(props, p, passkey) + parsed['properties'] = props + value = helpers.assemble_object_definition(parsed) + with helpers.thread_local_attribute( + constants.TL_CONTRACT_PASSKEY, passkey): + with helpers.thread_local_attribute( + constants.TL_OBJECTS_DRY_RUN, True): + obj = object_store.load( + value, owner, context=context, + default_type=default_type, scope_type=calling_type) + else: + raise exceptions.ContractViolationException( + 'Value {0} cannot be represented as class {1}'.format( + format_scalar(value), type_)) + if not helpers.is_instance_of( + obj, murano_class.name, + version_spec or helpers.get_type(root_context)): + raise exceptions.ContractViolationException( + 'Object of type {0} is not compatible with ' + 'requested type {1}'.format(obj.type.name, type_)) + + with helpers.thread_local_attribute( + constants.TL_CONTRACT_PASSKEY, passkey): + result = serializer.serialize( + obj.real_this, object_store.executor, + dsl_types.DumpTypes.Mixed) + if exclude_properties: + for p in exclude_properties: + helpers.patch_dict(result, p, utils.NO_VALUE) + return result + context = root_context.create_child_context() context.register_function(int_) context.register_function(string) @@ -198,6 +260,7 @@ class TypeScheme(object): context.register_function(not_null) context.register_function(error) context.register_function(class_) + context.register_function(template) context.register_function(owned_ref) context.register_function(owned) context.register_function(not_owned_ref) @@ -249,12 +312,33 @@ class TypeScheme(object): @specs.parameter('version_spec', yaqltypes.String(True)) @specs.method def class_(value, type, version_spec=None): - if helpers.is_instance_of( + if value is None or helpers.is_instance_of( value, type.type.name, version_spec or helpers.get_names_scope(root_context)): return value raise exceptions.ContractViolationException() + @specs.parameter('type_', dsl.MuranoTypeParameter( + nullable=False, context=root_context)) + @specs.parameter('default_type', dsl.MuranoTypeParameter( + nullable=True, context=root_context)) + @specs.parameter('value', nullable=True) + @specs.parameter('version_spec', yaqltypes.String(True)) + @specs.parameter( + 'exclude_properties', yaqltypes.Sequence(nullable=True)) + @specs.method + def template(value, type_, exclude_properties=None, + default_type=None, version_spec=None): + + if value is None or isinstance(value, utils.MappingType): + return value + + if helpers.is_instance_of( + value, type_.type.name, + version_spec or helpers.get_names_scope(root_context)): + return value + raise exceptions.ContractViolationException() + context = root_context.create_child_context() context.register_function(int_) context.register_function(string) @@ -262,6 +346,7 @@ class TypeScheme(object): context.register_function(check) context.register_function(not_null) context.register_function(class_) + context.register_function(template) return context def _map_dict(self, data, spec, context, path): @@ -351,6 +436,8 @@ class TypeScheme(object): return data def _map(self, data, spec, context, path): + if is_passkey(data): + return data child_context = context.create_child_context() if isinstance(spec, dsl_types.YaqlExpression): child_context[''] = data @@ -375,6 +462,9 @@ class TypeScheme(object): if data is dsl.NO_VALUE: data = helpers.evaluate(default, context) + if is_passkey(data): + return data + context = self.prepare_transform_context( context, this, owner, default, calling_type) return self._map(data, self._spec, context, '') @@ -383,6 +473,9 @@ class TypeScheme(object): if data is dsl.NO_VALUE: data = helpers.evaluate(default, context) + if is_passkey(data): + return True + context = self.prepare_validate_context(context) try: self._map(data, self._spec, context, '') @@ -395,3 +488,8 @@ def format_scalar(value): if isinstance(value, six.string_types): return "'{0}'".format(value) return six.text_type(value) + + +def is_passkey(value): + passkey = helpers.get_contract_passkey() + return passkey is not None and value is passkey diff --git a/murano/dsl/yaql_functions.py b/murano/dsl/yaql_functions.py index e907e544..766b0416 100644 --- a/murano/dsl/yaql_functions.py +++ b/murano/dsl/yaql_functions.py @@ -245,9 +245,9 @@ def call_func(context, op_dot, base, name, args, kwargs, @specs.parameter('obj', dsl.MuranoObjectParameter(decorate=False)) @specs.parameter('serialization_type', yaqltypes.String()) @specs.parameter('ignore_upcasts', bool) -def dump(obj, serialization_type=serializer.DumpTypes.Serializable, +def dump(obj, serialization_type=dsl_types.DumpTypes.Serializable, ignore_upcasts=True): - if serialization_type not in serializer.DumpTypes.All: + if serialization_type not in dsl_types.DumpTypes.All: raise ValueError('Invalid Serialization Type') executor = helpers.get_executor() if ignore_upcasts: diff --git a/murano/tests/unit/dsl/meta/ContractExamples.yaml b/murano/tests/unit/dsl/meta/ContractExamples.yaml index 5f3e93c3..cce74fb1 100644 --- a/murano/tests/unit/dsl/meta/ContractExamples.yaml +++ b/murano/tests/unit/dsl/meta/ContractExamples.yaml @@ -38,6 +38,28 @@ Methods: Body: Return: $arg + testTemplateContract: + Arguments: + arg: + Contract: $.template(CreatedClass2) + Body: + Return: $arg + + testTemplateContractExcludePropertyFromMpl: + Body: + - $model: + :CreatedClass2: + property1: qwerty + property2: 'not integer' + - Return: $.testTemplateContractExcludeProperty($model) + + testTemplateContractExcludeProperty: + Arguments: + arg: + Contract: $.template(CreatedClass2, excludeProperties => [property2]) + Body: + Return: $arg + testClassFromIdContract: Arguments: arg: diff --git a/murano/tests/unit/dsl/meta/TestSchema.yaml b/murano/tests/unit/dsl/meta/TestSchema.yaml index 0c453256..bcf786f1 100644 --- a/murano/tests/unit/dsl/meta/TestSchema.yaml +++ b/murano/tests/unit/dsl/meta/TestSchema.yaml @@ -42,6 +42,9 @@ Properties: classProperty: Contract: $.class(SampleClass1) + templateProperty: + Contract: $.template(SampleClass1, excludeProperties => [stringProperty]) + defaultProperty: Contract: $.int() Default: 999 diff --git a/murano/tests/unit/dsl/test_contracts.py b/murano/tests/unit/dsl/test_contracts.py index 6e7be7e8..26f50e51 100644 --- a/murano/tests/unit/dsl/test_contracts.py +++ b/murano/tests/unit/dsl/test_contracts.py @@ -126,6 +126,30 @@ class TestContracts(test_case.DslTestCase): self.assertIsInstance(result, dsl.MuranoObjectInterface) self.assertEqual(object_id, result.id) + def test_template_contract(self): + arg = om.Object('CreatedClass2', property1='qwerty', property2=123) + result = self._runner.testTemplateContract(arg) + self.assertIsInstance(result, dict) + self.assertItemsEqual(['?', 'property1', 'property2'], result.keys()) + + def test_template_contract_fail_on_type(self): + arg = om.Object('SampleClass2', class2Property='qwerty') + self.assertRaises( + exceptions.ContractViolationException, + self._runner.testTemplateContract, arg) + + def test_template_contract_with_property_exclusion(self): + arg = om.Object('CreatedClass2', property1='qwerty', + property2='INVALID') + result = self._runner.testTemplateContractExcludeProperty(arg) + self.assertIsInstance(result, dict) + self.assertItemsEqual(['?', 'property1'], result.keys()) + + def test_template_contract_with_property_exclusion_from_mpl(self): + result = self._runner.testTemplateContractExcludePropertyFromMpl() + self.assertIsInstance(result, dict) + self.assertItemsEqual(['?', 'property1'], result.keys()) + def test_check_contract(self): arg = om.Object('SampleClass2', class2Property='qwerty') self.assertIsNone(self._runner.testCheckContract(arg, 100)) diff --git a/murano/tests/unit/dsl/test_schema_generation.py b/murano/tests/unit/dsl/test_schema_generation.py index df54e1ea..eedc235f 100644 --- a/murano/tests/unit/dsl/test_schema_generation.py +++ b/murano/tests/unit/dsl/test_schema_generation.py @@ -75,6 +75,15 @@ class TestSchemaGeneration(test_case.DslTestCase): 'classProperty', ['null', 'muranoObject']) self.assertEqual('SampleClass1', schema.get('muranoType')) + def test_template_property(self): + schema = self._test_simple_property( + 'templateProperty', ['null', 'muranoObject']) + self.assertEqual('SampleClass1', schema.get('muranoType')) + self.assertTrue(schema.get('owned')) + self.assertItemsEqual( + ['stringProperty'], + schema.get('excludedProperties')) + def test_default_property(self): schema = self._test_simple_property( 'defaultProperty', ['null', 'integer']) diff --git a/releasenotes/notes/template-contract-b71840cbc35eb478.yaml b/releasenotes/notes/template-contract-b71840cbc35eb478.yaml new file mode 100644 index 00000000..b32fbd6d --- /dev/null +++ b/releasenotes/notes/template-contract-b71840cbc35eb478.yaml @@ -0,0 +1,11 @@ +--- +features: + - New contract function ``template`` was introduced. ``template`` works + similar to the ``class`` in regards to the data validation but does not + instantiate objects. Instead the data is left in the object model + in dictionary format so that it could be instantiated later with the new() + function. In addition it allows to exclude specified properties from + validation and resulting template so that they could be provided later. + Objects that are assigned to the property or argument with ``template`` + contract will be automatically converted to their object model + representation.