Ability to have several MuranoPL classes in single YAML file

Classes are separated using YAML document separator (3 dashes).
Empty documents are skipped. If the class doesn't have Namespace
section corresponding section from the previous class in the same
file is used. Thus it is possible to declare namespace prefixes once
at the file header. Even if there are several classes in one file all of
them are still required to be declared in manifest file (except for
the DSL unit tests that don't have manifests)

Change-Id: Ic8df487d8a7c327f085f9723c251e99fa998c481

murano/dsl/murano_package.py

@@ -17,6 +17,8 @@ import weakref
 
 import semantic_version
 import six
+from yaql.language import specs
+from yaql.language import utils
 
 from murano.dsl import constants
 from murano.dsl import dsl_types
@@ -24,12 +26,12 @@ from murano.dsl import exceptions
 from murano.dsl import helpers
 from murano.dsl import murano_object
 from murano.dsl import murano_type
+from murano.dsl import namespace_resolver
 from murano.dsl import principal_objects
 from murano.dsl import yaql_integration
 
 
 class MuranoPackage(dsl_types.MuranoPackage):
-
     def __init__(self, package_loader, name, version=None,
                  runtime_version=None, requirements=None):
         super(MuranoPackage, self).__init__()
@@ -85,11 +87,37 @@ class MuranoPackage(dsl_types.MuranoPackage):
     def get_class_config(self, name):
         return {}
 
-    def _register_mpl_class(self, data, name=None):
+    def _register_mpl_classes(self, data, name):
         type_obj = self._classes.get(name)
-        if not type_obj:
+        if type_obj is not None:
-            type_obj = murano_type.create(data, self, name)
+            return type_obj
+        if callable(data):
+            data = data()
+        if not utils.is_sequence(data):
+            data = [data]
+        unnamed_class = None
+        last_ns = {}
+        for cls_data in data:
+            last_ns = cls_data.setdefault('Namespaces', last_ns)
+            if len(cls_data) == 1:
+                continue
+            cls_name = cls_data.get('Name')
+            if not cls_name:
+                if unnamed_class:
+                    raise exceptions.AmbiguousClassName(name)
+                unnamed_class = cls_data
+            else:
+                ns_resolver = namespace_resolver.NamespaceResolver(last_ns)
+                cls_name = ns_resolver.resolve_name(cls_name)
+                if cls_name == name:
+                    type_obj = murano_type.create(
+                        cls_data, self, cls_name, ns_resolver)
                     self._classes[name] = type_obj
+                else:
+                    self._load_queue.setdefault(cls_name, cls_data)
+        if type_obj is None and unnamed_class:
+            unnamed_class['Name'] = name
+            return self._register_mpl_classes(unnamed_class, name)
         return type_obj
 
     def _register_native_class(self, cls, name):
@@ -99,7 +127,7 @@ class MuranoPackage(dsl_types.MuranoPackage):
         try:
             m_class = self.find_class(name, False)
         except exceptions.NoClassFound:
-            m_class = self._register_mpl_class({'Name': name}, name)
+            m_class = self._register_mpl_classes({'Name': name}, name)
 
         m_class.extension_class = cls
 
@@ -112,14 +140,10 @@ class MuranoPackage(dsl_types.MuranoPackage):
                     helpers.inspect_is_static(cls, method_name),
                     helpers.inspect_is_classmethod(cls, method_name))):
                 continue
-            # TODO(slagun): update the code below to use yaql native
-            # method for this when https://review.openstack.org/#/c/220748/
-            # will get merged and Murano requirements bump to corresponding
-            # yaql version
             method_name_alias = (getattr(
                 method, '__murano_name', None) or
-                yaql_integration.CONVENTION.convert_function_name(
+                specs.convert_function_name(
-                    method_name.rstrip('_')))
+                    method_name, yaql_integration.CONVENTION))
             m_class.add_method(method_name_alias, method, method_name)
         self._imported_types.add(cls)
         return m_class
@@ -130,10 +154,10 @@ class MuranoPackage(dsl_types.MuranoPackage):
             if name in self._classes:
                 self._register_native_class(cls, name)
             else:
-                self._native_load_queue[name] = cls
+                self._native_load_queue.setdefault(name, cls)
         elif isinstance(cls, dsl_types.MuranoType):
             self._classes[cls.name] = cls
-        else:
+        elif name not in self._classes:
             self._load_queue[name] = cls
 
     def find_class(self, name, search_requirements=True):
@@ -143,9 +167,9 @@ class MuranoPackage(dsl_types.MuranoPackage):
 
         payload = self._load_queue.pop(name, None)
         if payload is not None:
-            if callable(payload):
+            result = self._register_mpl_classes(payload, name)
-                payload = payload()
+            if result:
-            return self._register_mpl_class(payload, name)
+                return result
 
         result = self._classes.get(name)
         if result:

murano/dsl/murano_type.py

@@ -27,7 +27,6 @@ from murano.dsl import helpers
 from murano.dsl import murano_method
 from murano.dsl import murano_object
 from murano.dsl import murano_property
-from murano.dsl import namespace_resolver
 from murano.dsl import yaql_integration
 
 
@@ -365,13 +364,7 @@ class MuranoClass(dsl_types.MuranoClass, MuranoType):
         return dsl_types.MuranoTypeReference(self)
 
 
-def create(data, package, name=None):
+def create(data, package, name, ns_resolver):
-    namespaces = data.get('Namespaces') or {}
-    ns_resolver = namespace_resolver.NamespaceResolver(namespaces)
-
-    if not name:
-        name = ns_resolver.resolve_name(str(data['Name']))
-
     parent_class_names = data.get('Extends')
     parent_classes = []
     if parent_class_names:

murano/engine/yaql_yaml_loader.py

@@ -72,6 +72,9 @@ def get_loader(version):
 
         YaqlYamlLoader.add_constructor(u'!yaql', yaql_constructor)
         YaqlYamlLoader.add_implicit_resolver(u'!yaql', YaqlExpression, None)
-        return yaml.load(contents, Loader=YaqlYamlLoader)
+        return list(filter(
+            lambda t: t,
+            yaml.load_all(contents, Loader=YaqlYamlLoader))
+        )
 
     return load

murano/tests/functional/engine/io.murano.apps.test.ApacheHttpServerCustom/Classes/ApacheHttpServer.yaml

@@ -11,14 +11,14 @@
 #  under the License.
 
 Namespaces:
-  =: io.murano.apps.apache
+  =: io.murano.apps.test
   std: io.murano
   res: io.murano.resources
   sys: io.murano.system
   conf: io.murano.configuration
 
 
-Name: ApacheHttpServer
+Name: ApacheHttpServerCustom
 
 Extends: std:Application
 

murano/tests/functional/engine/io.murano.conflang.chef.ExampleChef/Classes/ExampleChef.yaml

@@ -1,5 +1,5 @@
 Namespaces:
-  =: io.murano.conflang.chef.ExampleChef
+  =: io.murano.conflang.chef
   std: io.murano
   res: io.murano.resources
   sys: io.murano.system

murano/tests/functional/engine/io.murano.conflang.puppet.ExamplePuppet/Classes/ExamplePuppet.yaml

@@ -1,5 +1,5 @@
 Namespaces:
-  =: io.murano.conflang.puppet.ExamplePuppet
+  =: io.murano.conflang.puppet
   std: io.murano
   res: io.murano.resources
   sys: io.murano.system

murano/tests/unit/dsl/foundation/test_package_loader.py

@@ -78,29 +78,33 @@ class TestPackageLoader(package_loader.MuranoPackageLoader):
             raise KeyError(class_name)
 
     def _build_index(self, directory):
-        yamls = [os.path.join(dirpath, f)
+        yamls = [
+            os.path.join(dirpath, f)
             for dirpath, _, files in os.walk(directory)
             for f in fnmatch.filter(files, '*.yaml')
             if f != 'manifest.yaml'
         ]
         for class_def_file in yamls:
-            self._load_class(class_def_file)
+            self._load_classes(class_def_file)
 
-    def _load_class(self, class_def_file):
+    def _load_classes(self, class_def_file):
         with open(class_def_file) as stream:
-            data = self._yaml_loader(stream.read(), class_def_file)
+            data_lst = self._yaml_loader(stream.read(), class_def_file)
 
+        last_ns = {}
+        for data in data_lst:
+            last_ns = data.get('Namespaces', last_ns)
             if 'Name' not in data:
-            return
+                continue
 
             for name, method in six.iteritems(data.get('Methods') or data.get(
                     'Workflow') or {}):
                 if name.startswith('test'):
                     method['Usage'] = 'Action'
 
-        ns = namespace_resolver.NamespaceResolver(data.get('Namespaces', {}))
+            ns = namespace_resolver.NamespaceResolver(last_ns)
             class_name = ns.resolve_name(data['Name'])
-        self._classes[class_name] = data
+            self._classes[class_name] = data_lst
 
     def set_config_value(self, class_name, property_name, value):
         if isinstance(class_name, object_model.Object):

murano/tests/unit/dsl/meta/TestStatics.yaml

@@ -3,6 +3,26 @@ Namespaces:
   =: test
   e: ''
 
+--- # ---------------------------------------------------------------------
+# TestStaticsBase class - base class for TestStatics to test how static
+# entities work in respect to class inheritance
+--- # ---------------------------------------------------------------------
+Name: TestStaticsBase
+
+Properties:
+  baseStaticProperty:
+    Contract: $.string()
+    Default: baseStaticProperty
+    Usage: Static
+
+  conflictingStaticProperty:
+    Contract: $.string()
+    Default: 'conflictingStaticProperty-base'
+    Usage: Static
+
+--- # ---------------------------------------------------------------------
+# TestStatics class - main class for the static tests
+--- # ---------------------------------------------------------------------
 Name: TestStatics
 
 Extends: TestStaticsBase

murano/tests/unit/dsl/meta/TestStaticsBase.yaml

@@ -1,15 +0,0 @@
-Namespaces:
-  =: test
-
-Name: TestStaticsBase
-
-Properties:
-  baseStaticProperty:
-    Contract: $.string()
-    Default: baseStaticProperty
-    Usage: Static
-
-  conflictingStaticProperty:
-    Contract: $.string()
-    Default: 'conflictingStaticProperty-base'
-    Usage: Static

murano_tempest_tests/extras/MockApp/Classes/mock_muranopl.yaml

@@ -2,7 +2,7 @@ Namespaces:
   =: io.murano.apps
   std: io.murano
 
-Name: MockApp
+# Name: MockApp # use name from the manifest
 
 Extends: std:Application
 

releasenotes/notes/multi-class-yamls-cbb3ef1d8578f41a.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - Now it is possible to have several classes in one YAML file.
+    Classes are separated using YAML document separator (3 dashes).
+    Empty documents are skipped. If the class doesn't have Namespace
+    section corresponding section from the previous class in the same
+    file is used. Thus it is possible to declare namespace prefixes once
+    at the file header. Even if there are several classes in one file all of
+    them are still required to be declared in manifest file.
+