Introduces combined class loader

This package class loader combines two types of
class loaders: loading packages from API and from local directory.

If folders to look packages in are specified,
packages would be loaded from their.
Otherwise, standart loader by API will operate as usual.

Implements blueprint change-murano-class-loader

Change-Id: Ifd8f40a755dc580703a44edc2b32cdd17691669d

murano/common/config.py

@@ -195,7 +195,10 @@ engine_opts = [
                 help=_('Enable model policy enforcer using Congress')),
     cfg.IntOpt('agent_timeout', default=3600,
                help=_('Time for waiting for a response from murano agent '
-                      'during the deployment'))
+                      'during the deployment')),
+    cfg.ListOpt('load_packages_from', default=[],
+                help=_('List of directories to load local packages from. '
+                       'If not provided, packages will be loaded by API'))
 ]
 
 # TODO(sjmc7): move into engine opts?

murano/common/engine.py

@@ -131,16 +131,9 @@ class TaskExecutor(object):
         self._create_trust()
 
         try:
-            # !!! please do not delete 2 commented lines of code below.
-            # Uncomment to make engine load packages from
-            # local folder rather than from API !!!
-
-            # pkg_loader = package_loader.DirectoryPackageLoader('./meta')
-            # return self._execute(pkg_loader)
-
             murano_client_factory = lambda: \
                 self._environment.clients.get_murano_client(self._environment)
-            with package_loader.ApiPackageLoader(
+            with package_loader.CombinedPackageLoader(
                     murano_client_factory,
                     self._environment.tenant_id) as pkg_loader:
                 return self._execute(pkg_loader)

murano/engine/package_loader.py

@@ -25,7 +25,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import six
 
-from murano.common.i18n import _LE
+from murano.common.i18n import _LE, _LI
 from murano.dsl import exceptions
 from murano.engine import yaql_yaml_loader
 from murano.packages import exceptions as pkg_exc
@@ -92,7 +92,7 @@ class ApiPackageLoader(PackageLoader):
                           'more then 1 package found for query "{0}", '
                           'will resolve based on the ownership'.
                           format(filter_opts))
-                return get_best_package_match(packages, self.tenant_id)
+                return self._get_best_package_match(packages, self.tenant_id)
             elif len(packages) == 1:
                 return packages[0]
             else:
@@ -148,6 +148,21 @@ class ApiPackageLoader(PackageLoader):
             except OSError:
                 pass
 
+    def _get_best_package_match(self, packages):
+        public = None
+        other = []
+        for package in packages:
+            if package.owner_id == self.tenant_id:
+                return package
+            elif package.is_public:
+                public = package
+            else:
+                other.append(package)
+        if public is not None:
+            return public
+        elif other:
+            return other[0]
+
     def cleanup(self):
         shutil.rmtree(self._cache_directory, ignore_errors=True)
 
@@ -185,10 +200,11 @@ class DirectoryPackageLoader(PackageLoader):
                     folder, preload=True,
                     loader=yaql_yaml_loader.YaqlYamlLoader)
             except pkg_exc.PackageLoadError:
-                LOG.exception(_LE('Unable to load package from path: '
+                LOG.info(_LI('Unable to load package from path: {0}').format(
-                                  '{0}').format(entry))
+                    os.path.join(self._base_path, entry)))
                 continue
-
+            LOG.info(_LI('Loaded package from path {0}').format(
+                os.path.join(self._base_path, entry)))
             for c in package.classes:
                 self._packages_by_class[c] = package
             self._packages_by_name[package.full_name] = package
@@ -196,17 +212,37 @@ class DirectoryPackageLoader(PackageLoader):
             self._processed_entries.add(entry)
 
 
-def get_best_package_match(packages, tenant_id):
+class CombinedPackageLoader(PackageLoader):
-    public = None
+    def __init__(self, murano_client_factory, tenant_id):
-    other = []
+        self.murano_client_factory = murano_client_factory
-    for package in packages:
+        self.tenant_id = tenant_id
-        if package.owner_id == tenant_id:
+        self.loader_from_api = ApiPackageLoader(self.murano_client_factory,
-            return package
+                                                self.tenant_id)
-        elif package.is_public:
+        self.loaders_from_dir = []
-            public = package
+
-        else:
+        for directory in CONF.engine.load_packages_from:
-            other.append(package)
+            if os.path.exists(directory):
-    if public is not None:
+                self.loaders_from_dir.append(DirectoryPackageLoader(directory))
-        return public
+
-    elif other:
+    def get_package_by_class(self, name):
-        return other[0]
+        for loader in self.loaders_from_dir:
+            pkg = loader.get_package_by_class(name)
+            if pkg:
+                return pkg
+        return self.loader_from_api.get_package_by_class(name)
+
+    def get_package(self, name):
+        # Try to load from local directory first
+        for loader in self.loaders_from_dir:
+            pkg = loader.get_package(name)
+            if pkg:
+                return pkg
+        # If no package found, load package by API
+        return self.loader_from_api.get_package(name)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.loader_from_api.cleanup()
+        return False

murano/tests/unit/engine/meta/Classes/Mytest.yaml

@@ -0,0 +1,8 @@
+Namespaces:
+    =: io.murano.test
+
+Extends: sys:TestFixture
+
+Name: MyTest
+
+Methods:

murano/tests/unit/engine/meta/manifest.yaml

@@ -0,0 +1,10 @@
+Format: 1.0
+Type: Application
+FullName: io.murano.test.MyTest
+Name: Test case Example
+Description: |
+ Example of simple package
+Author: 'Mirantis, Inc'
+Tags: []
+Classes:
+ io.murano.test.MyTest: Mytest.yaml

murano/tests/unit/engine/test_package_loader.py

@@ -0,0 +1,67 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+import os
+
+import mock
+from oslo_config import cfg
+
+from murano.engine import package_loader
+from murano.packages import mpl_package
+from murano.tests.unit import base
+
+CONF = cfg.CONF
+
+
+class TestCombinedPackageLoader(base.MuranoTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestCombinedPackageLoader, cls).setUpClass()
+
+        location = os.path.dirname(__file__)
+        CONF.set_override('load_packages_from', [location], 'engine')
+        cls.murano_client_factory = mock.MagicMock()
+        cls.loader = package_loader.CombinedPackageLoader(
+            cls.murano_client_factory, 'test_tenant_id')
+        cls.api_loader = mock.MagicMock()
+        cls.loader.loader_from_api = cls.api_loader
+
+        cls.local_pkg_name = 'io.murano.test.MyTest'
+        cls.api_pkg_name = 'test.mpl.v1.app.Thing'
+
+    def test_loaders_initialized(self):
+        self.assertEqual(1, len(self.loader.loaders_from_dir),
+                         'One directory class loader should be initialized'
+                         ' since there is one valid murano pl package in the'
+                         ' provided directory in config.')
+        self.assertIsInstance(self.loader.loaders_from_dir[0],
+                              package_loader.DirectoryPackageLoader)
+
+    def test_get_package_by_class_directory_loader(self):
+        result = self.loader.get_package_by_class(self.local_pkg_name)
+        self.assertIsInstance(result, mpl_package.MuranoPlPackage)
+
+    def test_get_package_by_name_directory_loader(self):
+        result = self.loader.get_package(self.local_pkg_name)
+        self.assertIsInstance(result, mpl_package.MuranoPlPackage)
+
+    def test_get_package_by_class_api_loader(self):
+        self.loader.get_package(self.api_pkg_name)
+
+        self.api_loader.get_package.assert_called_with(self.api_pkg_name)
+
+    def test_get_package_api_loader(self):
+        self.loader.get_package_by_class(self.api_pkg_name)
+
+        self.api_loader.get_package_by_class.assert_called_with(
+            self.api_pkg_name)