Break cyclic references in DSL

Because of several bugs/code design issues in the
DSL a cyclic references between objects were created.
Thus the object model objects were not automatically
deleted upon deployment finish even if there were no
cross-links between the objects in object model.

This commit both breaks the links and increases engine
performance due to
1) For LHS expressions there is no more need to parse
 yaql function definitions upon each variable modification
 because now the base LHS context is fixed and static
2) In most cases the objects now are reclaimed immediately
 after deployment finish thus python GC doesn't have to
 traverse large graphs

Targets-blueprint: dependency-driven-resource-deallocation

Change-Id: I4b1e0038bf7c08ced357fa20c4b1e3d612c93ae9

murano/dsl/dsl.py

@@ -163,17 +163,18 @@ class MuranoObjectInterface(dsl_types.MuranoObjectInterface):
             oi[key] = value
 
     class CallInterface(object):
-        def __init__(self, mpl_object, object_store):
-            self.__object = mpl_object
-            self.__object_store = object_store
+        def __init__(self, object_interface):
+            self.__object_interface = object_interface
 
         def __getattr__(self, item):
             def func(*args, **kwargs):
                 self._insert_instruction()
-                with helpers.with_object_store(self.__object_store):
+                with helpers.with_object_store(
+                        self.__object_interface.object_store):
                     context = helpers.get_context()
-                    return to_mutable(self.__object.type.invoke(
-                        item, self.__object, args, kwargs,
+                    obj = self.__object_interface.object
+                    return to_mutable(obj.type.invoke(
+                        item, obj, args, kwargs,
                         context), helpers.get_yaql_engine(context))
             return func
 
@@ -188,27 +189,31 @@ class MuranoObjectInterface(dsl_types.MuranoObjectInterface):
                 context[constants.CTX_CURRENT_INSTRUCTION] = NativeInstruction(
                     frame[4][0].strip(), location)
 
-    def __init__(self, mpl_object, object_store=None):
+    def __init__(self, mpl_object):
         self.__object = mpl_object
-        self.__object_store = object_store or helpers.get_object_store()
+        self.__object_store = helpers.get_object_store()
 
     @staticmethod
-    def create(mpl_object, object_store=None):
+    def create(mpl_object):
         if mpl_object is None or isinstance(mpl_object, MuranoObjectInterface):
             return mpl_object
-        return MuranoObjectInterface(mpl_object, object_store)
+        return MuranoObjectInterface(mpl_object)
 
     @property
     def object(self):
         return self.__object
 
+    @property
+    def object_store(self):
+        return self.__object_store
+
     @property
     def id(self):
-        return self.__object.object_id
+        return self.object.object_id
 
     @property
     def owner(self):
-        owner = self.__object.owner
+        owner = self.object.owner
         return MuranoObjectInterface.create(owner)
 
     def find_owner(self, type, optional=False):
@@ -216,7 +221,7 @@ class MuranoObjectInterface(dsl_types.MuranoObjectInterface):
             type = helpers.get_class(type)
         elif isinstance(type, dsl_types.MuranoTypeReference):
             type = type.type
-        p = self.__object.owner
+        p = self.object.owner
         while p is not None:
             if type.is_compatible(p):
                 return MuranoObjectInterface(p)
@@ -228,7 +233,7 @@ class MuranoObjectInterface(dsl_types.MuranoObjectInterface):
 
     @property
     def type(self):
-        return self.__object.type
+        return self.object.type
 
     @property
     def package(self):
@@ -244,17 +249,17 @@ class MuranoObjectInterface(dsl_types.MuranoObjectInterface):
 
     @property
     def extension(self):
-        return self.__object.extension
+        return self.object.extension
 
     def cast(self, murano_class, version_spec=None):
         return MuranoObjectInterface.create(
             helpers.cast(
-                self.__object, murano_class,
+                self.object, murano_class,
                 version_spec or helpers.get_type()))
 
     def is_instance_of(self, murano_class, version_spec=None):
         return helpers.is_instance_of(
-            self.__object, murano_class,
+            self.object, murano_class,
             version_spec or helpers.get_type())
 
     def ancestors(self):
@@ -263,17 +268,16 @@ class MuranoObjectInterface(dsl_types.MuranoObjectInterface):
     def __getitem__(self, item):
         context = helpers.get_context()
         return to_mutable(
-            self.__object.get_property(item, context),
+            self.object.get_property(item, context),
             helpers.get_yaql_engine(context))
 
     def __setitem__(self, key, value):
         context = helpers.get_context()
         value = helpers.evaluate(value, context)
-        self.__object.set_property(key, value, context)
+        self.object.set_property(key, value, context)
 
     def __call__(self):
-        return MuranoObjectInterface.CallInterface(
-            self.object, self.__object_store)
+        return MuranoObjectInterface.CallInterface(self)
 
     def __repr__(self):
         return '<{0}>'.format(repr(self.object))

murano/dsl/dsl_types.py

@@ -12,6 +12,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import weakref
+
 
 class ClassUsages(object):
     Class = 'Class'
@@ -112,11 +114,11 @@ class MuranoProperty(object):
 
 class MuranoTypeReference(object):
     def __init__(self, murano_type):
-        self.__murano_type = murano_type
+        self.__murano_type = weakref.ref(murano_type)
 
     @property
     def type(self):
-        return self.__murano_type
+        return self.__murano_type()
 
     def __repr__(self):
         return '*' + repr(self.type)

murano/dsl/helpers.py

@@ -620,6 +620,12 @@ def weak_proxy(obj):
     return weakref.proxy(obj)
 
 
+def weak_ref(obj):
+    if obj is None or isinstance(obj, weakref.ReferenceType):
+        return obj
+    return weakref.ref(obj)
+
+
 def parse_type_string(type_str, default_version, default_package):
     res = TYPE_RE.match(type_str)
     if res is None:

murano/dsl/lhs_expression.py

@@ -27,153 +27,156 @@ from murano.dsl import yaql_functions
 from murano.dsl import yaql_integration
 
 
+def _prepare_context():
+    @specs.parameter('name', yaqltypes.StringConstant())
+    def get_context_data(context, name):
+        root_context = context['#root_context']
+
+        def set_data(value):
+            if not name or name == '$' or name == '$this':
+                raise ValueError('Cannot assign to {0}'.format(name))
+            ctx = root_context
+            while constants.CTX_VARIABLE_SCOPE not in ctx:
+                ctx = ctx.parent
+            ctx[name] = value
+
+        return _Property(lambda: root_context[name], set_data)
+
+    @specs.parameter('this', _Property)
+    @specs.parameter('key', yaqltypes.Keyword())
+    def attribution(context, this, key):
+        def setter(src_property, value):
+            src = src_property.get()
+            if isinstance(src, utils.MappingType):
+                src_property.set(
+                    utils.FrozenDict(
+                        itertools.chain(
+                            six.iteritems(src),
+                            ((key, value),))))
+            elif isinstance(src, dsl_types.MuranoObject):
+                src.set_property(key, value, context['#root_context'])
+            elif isinstance(src, (
+                    dsl_types.MuranoTypeReference,
+                    dsl_types.MuranoType)):
+                if isinstance(src, dsl_types.MuranoTypeReference):
+                    mc = src.type
+                else:
+                    mc = src
+                mc.set_property(key, value, context['#root_context'])
+            else:
+                raise ValueError(
+                    'attribution may only be applied to '
+                    'objects and dictionaries')
+
+        def getter(src):
+            if isinstance(src, utils.MappingType):
+                return src.get(key, {})
+            elif isinstance(src, dsl_types.MuranoObject):
+                try:
+                    return src.get_property(key, context['#root_context'])
+                except exceptions.UninitializedPropertyAccessError:
+                    return {}
+
+            else:
+                raise ValueError(
+                    'attribution may only be applied to '
+                    'objects and dictionaries')
+
+        return _Property(
+            lambda: getter(this.get()),
+            lambda value: setter(this, value))
+
+    @specs.parameter('this', _Property)
+    @specs.parameter('index', yaqltypes.Lambda(with_context=True))
+    def indexation(context, this, index):
+        index = index(context['#root_context'])
+
+        def getter(src):
+            if utils.is_sequence(src):
+                return src[index]
+            else:
+                raise ValueError('indexation may only be applied to lists')
+
+        def setter(src_property, value):
+            src = src_property.get()
+            if utils.is_sequence(src):
+                src_property.set(src[:index] + (value,) + src[index + 1:])
+            elif isinstance(src, utils.MappingType):
+                attribution(src_property, index).set(value)
+
+        if isinstance(index, int):
+            return _Property(
+                lambda: getter(this.get()),
+                lambda value: setter(this, value))
+        else:
+            return attribution(context, this, index)
+
+    def _wrap_type_reference(tr, context):
+        return _Property(
+            lambda: tr, context['#self']._invalid_target)
+
+    @specs.parameter('prefix', yaqltypes.Keyword())
+    @specs.parameter('name', yaqltypes.Keyword())
+    @specs.name('#operator_:')
+    def ns_resolve(context, prefix, name):
+        return _wrap_type_reference(
+            yaql_functions.ns_resolve(context, prefix, name), context)
+
+    @specs.parameter('name', yaqltypes.Keyword())
+    @specs.name('#unary_operator_:')
+    def ns_resolve_unary(context, name):
+        return _wrap_type_reference(
+            yaql_functions.ns_resolve_unary(context, name), context)
+
+    @specs.parameter('object_', dsl_types.MuranoObject)
+    def type_(context, object_):
+        return _wrap_type_reference(yaql_functions.type_(object_), context)
+
+    @specs.name('type')
+    @specs.parameter('cls', dsl.MuranoTypeParameter())
+    def type_from_name(context, cls):
+        return _wrap_type_reference(cls, context)
+
+    res_context = yaql_integration.create_empty_context()
+    res_context.register_function(get_context_data, '#get_context_data')
+    res_context.register_function(attribution, '#operator_.')
+    res_context.register_function(indexation, '#indexer')
+    res_context.register_function(ns_resolve)
+    res_context.register_function(ns_resolve_unary)
+    res_context.register_function(type_)
+    res_context.register_function(type_from_name)
+    return res_context
+
+
+class _Property(object):
+    def __init__(self, getter, setter):
+        self._getter = getter
+        self._setter = setter
+
+    def get(self):
+        return self._getter()
+
+    def set(self, value):
+        self._setter(value)
+
+
 class LhsExpression(object):
-    class Property(object):
-        def __init__(self, getter, setter):
-            self._getter = getter
-            self._setter = setter
-
-        def get(self):
-            return self._getter()
-
-        def set(self, value):
-            self._setter(value)
+    lhs_context = _prepare_context()
 
     def __init__(self, expression):
         self._expression = expression
 
-    def _create_context(self, root_context):
-        @specs.parameter('name', yaqltypes.StringConstant())
-        def get_context_data(name):
-
-            def set_data(value):
-                if not name or name == '$' or name == '$this':
-                    raise ValueError('Cannot assign to {0}'.format(name))
-                ctx = root_context
-                while constants.CTX_VARIABLE_SCOPE not in ctx:
-                    ctx = ctx.parent
-                ctx[name] = value
-
-            return LhsExpression.Property(
-                lambda: root_context[name], set_data)
-
-        @specs.parameter('this', LhsExpression.Property)
-        @specs.parameter('key', yaqltypes.Keyword())
-        def attribution(this, key):
-            def setter(src_property, value):
-                src = src_property.get()
-                if isinstance(src, utils.MappingType):
-                    src_property.set(
-                        utils.FrozenDict(
-                            itertools.chain(
-                                six.iteritems(src),
-                                ((key, value),))))
-                elif isinstance(src, dsl_types.MuranoObject):
-                    src.set_property(key, value, root_context)
-                elif isinstance(src, (
-                        dsl_types.MuranoTypeReference,
-                        dsl_types.MuranoType)):
-                    if isinstance(src, dsl_types.MuranoTypeReference):
-                        mc = src.type
-                    else:
-                        mc = src
-                    mc.set_property(key, value, root_context)
-                else:
-                    raise ValueError(
-                        'attribution may only be applied to '
-                        'objects and dictionaries')
-
-            def getter(src):
-                if isinstance(src, utils.MappingType):
-                    return src.get(key, {})
-                elif isinstance(src, dsl_types.MuranoObject):
-                    self._current_obj = src
-                    self._current_obj_name = key
-                    try:
-                        return src.get_property(key, root_context)
-                    except exceptions.UninitializedPropertyAccessError:
-                        return {}
-
-                else:
-                    raise ValueError(
-                        'attribution may only be applied to '
-                        'objects and dictionaries')
-
-            return LhsExpression.Property(
-                lambda: getter(this.get()),
-                lambda value: setter(this, value))
-
-        @specs.parameter('this', LhsExpression.Property)
-        @specs.parameter('index', yaqltypes.Lambda(with_context=True))
-        def indexation(this, index):
-            index = index(root_context)
-
-            def getter(src):
-                if utils.is_sequence(src):
-                    return src[index]
-                else:
-                    raise ValueError('indexation may only be applied to lists')
-
-            def setter(src_property, value):
-                src = src_property.get()
-                if utils.is_sequence(src):
-                    src_property.set(src[:index] + (value,) + src[index + 1:])
-                elif isinstance(src, utils.MappingType):
-                    attribution(src_property, index).set(value)
-
-            if isinstance(index, int):
-                return LhsExpression.Property(
-                    lambda: getter(this.get()),
-                    lambda value: setter(this, value))
-            else:
-                return attribution(this, index)
-
-        def _wrap_type_reference(tr):
-            return LhsExpression.Property(lambda: tr, self._invalid_target)
-
-        @specs.parameter('prefix', yaqltypes.Keyword())
-        @specs.parameter('name', yaqltypes.Keyword())
-        @specs.name('#operator_:')
-        def ns_resolve(prefix, name):
-            return _wrap_type_reference(
-                yaql_functions.ns_resolve(context, prefix, name))
-
-        @specs.parameter('name', yaqltypes.Keyword())
-        @specs.name('#unary_operator_:')
-        def ns_resolve_unary(context, name):
-            return _wrap_type_reference(
-                yaql_functions.ns_resolve_unary(context, name))
-
-        @specs.parameter('object_', dsl_types.MuranoObject)
-        def type_(object_):
-            return _wrap_type_reference(yaql_functions.type_(object_))
-
-        @specs.name('type')
-        @specs.parameter('cls', dsl.MuranoTypeParameter())
-        def type_from_name(cls):
-            return _wrap_type_reference(cls)
-
-        context = yaql_integration.create_empty_context()
-        context.register_function(get_context_data, '#get_context_data')
-        context.register_function(attribution, '#operator_.')
-        context.register_function(indexation, '#indexer')
-        context.register_function(ns_resolve)
-        context.register_function(ns_resolve_unary)
-        context.register_function(type_)
-        context.register_function(type_from_name)
-        return context
-
     def _invalid_target(self, *args, **kwargs):
         raise exceptions.InvalidLhsTargetError(self._expression)
 
     def __call__(self, value, context):
-        new_context = self._create_context(context)
+        new_context = LhsExpression.lhs_context.create_child_context()
         new_context[''] = context['$']
+        new_context['#root_context'] = context
+        new_context['#self'] = self
         for name in (constants.CTX_NAMES_SCOPE,):
             new_context[name] = context[name]
-        self._current_obj = None
-        self._current_obj_name = None
-        property = self._expression(context=new_context)
-        if not isinstance(property, LhsExpression.Property):
+        prop = self._expression(context=new_context)
+        if not isinstance(prop, _Property):
             self._invalid_target()
-        property.set(value)
+        prop.set(value)

murano/dsl/murano_object.py

@@ -28,7 +28,7 @@ class MuranoObject(dsl_types.MuranoObject):
         self.__initialized = False
         if known_classes is None:
             known_classes = {}
-        self.__owner = owner.real_this if owner else None
+        self.__owner = helpers.weak_ref(owner.real_this if owner else None)
         self.__object_id = object_id or helpers.generate_id()
         self.__type = murano_class
         self.__properties = {}
@@ -129,8 +129,10 @@ class MuranoObject(dsl_types.MuranoObject):
                 raise exceptions.CircularExpressionDependenciesError()
             last_errors = errors
 
-        if (not object_store.initializing and self.__extension is None and
-                not self.__initialized):
+        if (not object_store.initializing and
+                self.__extension is None and
+                not self.__initialized and
+                not helpers.is_objects_dry_run_mode()):
             method = self.type.methods.get('__init__')
             if method:
                 filtered_params = yaql_integration.filter_parameters(
@@ -164,7 +166,9 @@ class MuranoObject(dsl_types.MuranoObject):
 
     @property
     def owner(self):
-        return self.__owner
+        if self.__owner is None:
+            return None
+        return self.__owner()
 
     @property
     def real_this(self):

murano/dsl/object_store.py

@@ -128,7 +128,7 @@ class InitializationObjectStore(ObjectStore):
             object_id, self._keep_ids)
         if not obj:
             obj = murano_object.MuranoObject(
-                class_obj, helpers.weak_proxy(owner),
+                class_obj, owner,
                 name=parsed['name'],
                 object_id=object_id if self._keep_ids else None)
             self.put(obj, object_id or obj.object_id)

murano/engine/system/net_explorer.py

@@ -194,6 +194,5 @@ class NetworkExplorer(object):
     def list_ports(self):
         return self._client.list_ports()['ports']
 
-    @session_local_storage.execution_session_memoize
     def list_neutron_extensions(self):
         return self._client.list_extensions()['extensions']