From 1dd1c245294e60eba35649ffda9f02c28da63b2b Mon Sep 17 00:00:00 2001 From: Ekaterina Chernova Date: Wed, 8 Jul 2015 19:23:00 +0300 Subject: [PATCH] Introduces combined class loader This package class loader combines two types of class loaders: loading packages from API and from local directory. If folders to look packages in are specified, packages would be loaded from their. Otherwise, standart loader by API will operate as usual. Implements blueprint change-murano-class-loader Change-Id: Ifd8f40a755dc580703a44edc2b32cdd17691669d --- murano/common/config.py | 7 +- murano/common/engine.py | 9 +-- murano/engine/package_loader.py | 74 ++++++++++++++----- .../unit/engine/meta/Classes/Mytest.yaml | 8 ++ murano/tests/unit/engine/meta/manifest.yaml | 10 +++ .../tests/unit/engine/test_package_loader.py | 67 +++++++++++++++++ 6 files changed, 146 insertions(+), 29 deletions(-) create mode 100644 murano/tests/unit/engine/meta/Classes/Mytest.yaml create mode 100644 murano/tests/unit/engine/meta/manifest.yaml create mode 100644 murano/tests/unit/engine/test_package_loader.py diff --git a/murano/common/config.py b/murano/common/config.py index 405f2d6f..9ca3b582 100644 --- a/murano/common/config.py +++ b/murano/common/config.py @@ -194,8 +194,11 @@ engine_opts = [ cfg.BoolOpt('enable_model_policy_enforcer', default=False, help=_('Enable model policy enforcer using Congress')), cfg.IntOpt('agent_timeout', default=3600, - help=_('Time for waiting for a response from murano agent' - 'during the deployment')) + help=_('Time for waiting for a response from murano agent ' + 'during the deployment')), + cfg.ListOpt('load_packages_from', default=[], + help=_('List of directories to load local packages from. ' + 'If not provided, packages will be loaded by API')) ] # TODO(sjmc7): move into engine opts? diff --git a/murano/common/engine.py b/murano/common/engine.py index e56f16ae..19c27ed0 100755 --- a/murano/common/engine.py +++ b/murano/common/engine.py @@ -131,16 +131,9 @@ class TaskExecutor(object): self._create_trust() try: - # !!! please do not delete 2 commented lines of code below. - # Uncomment to make engine load packages from - # local folder rather than from API !!! - - # pkg_loader = package_loader.DirectoryPackageLoader('./meta') - # return self._execute(pkg_loader) - murano_client_factory = lambda: \ self._environment.clients.get_murano_client(self._environment) - with package_loader.ApiPackageLoader( + with package_loader.CombinedPackageLoader( murano_client_factory, self._environment.tenant_id) as pkg_loader: return self._execute(pkg_loader) diff --git a/murano/engine/package_loader.py b/murano/engine/package_loader.py index f0af64e3..c7a67dd8 100644 --- a/murano/engine/package_loader.py +++ b/murano/engine/package_loader.py @@ -25,7 +25,7 @@ from oslo_config import cfg from oslo_log import log as logging import six -from murano.common.i18n import _LE +from murano.common.i18n import _LE, _LI from murano.dsl import exceptions from murano.engine import yaql_yaml_loader from murano.packages import exceptions as pkg_exc @@ -92,7 +92,7 @@ class ApiPackageLoader(PackageLoader): 'more then 1 package found for query "{0}", ' 'will resolve based on the ownership'. format(filter_opts)) - return get_best_package_match(packages, self.tenant_id) + return self._get_best_package_match(packages, self.tenant_id) elif len(packages) == 1: return packages[0] else: @@ -148,6 +148,21 @@ class ApiPackageLoader(PackageLoader): except OSError: pass + def _get_best_package_match(self, packages): + public = None + other = [] + for package in packages: + if package.owner_id == self.tenant_id: + return package + elif package.is_public: + public = package + else: + other.append(package) + if public is not None: + return public + elif other: + return other[0] + def cleanup(self): shutil.rmtree(self._cache_directory, ignore_errors=True) @@ -185,10 +200,11 @@ class DirectoryPackageLoader(PackageLoader): folder, preload=True, loader=yaql_yaml_loader.YaqlYamlLoader) except pkg_exc.PackageLoadError: - LOG.exception(_LE('Unable to load package from path: ' - '{0}').format(entry)) + LOG.info(_LI('Unable to load package from path: {0}').format( + os.path.join(self._base_path, entry))) continue - + LOG.info(_LI('Loaded package from path {0}').format( + os.path.join(self._base_path, entry))) for c in package.classes: self._packages_by_class[c] = package self._packages_by_name[package.full_name] = package @@ -196,17 +212,37 @@ class DirectoryPackageLoader(PackageLoader): self._processed_entries.add(entry) -def get_best_package_match(packages, tenant_id): - public = None - other = [] - for package in packages: - if package.owner_id == tenant_id: - return package - elif package.is_public: - public = package - else: - other.append(package) - if public is not None: - return public - elif other: - return other[0] +class CombinedPackageLoader(PackageLoader): + def __init__(self, murano_client_factory, tenant_id): + self.murano_client_factory = murano_client_factory + self.tenant_id = tenant_id + self.loader_from_api = ApiPackageLoader(self.murano_client_factory, + self.tenant_id) + self.loaders_from_dir = [] + + for directory in CONF.engine.load_packages_from: + if os.path.exists(directory): + self.loaders_from_dir.append(DirectoryPackageLoader(directory)) + + def get_package_by_class(self, name): + for loader in self.loaders_from_dir: + pkg = loader.get_package_by_class(name) + if pkg: + return pkg + return self.loader_from_api.get_package_by_class(name) + + def get_package(self, name): + # Try to load from local directory first + for loader in self.loaders_from_dir: + pkg = loader.get_package(name) + if pkg: + return pkg + # If no package found, load package by API + return self.loader_from_api.get_package(name) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.loader_from_api.cleanup() + return False diff --git a/murano/tests/unit/engine/meta/Classes/Mytest.yaml b/murano/tests/unit/engine/meta/Classes/Mytest.yaml new file mode 100644 index 00000000..ae18eb63 --- /dev/null +++ b/murano/tests/unit/engine/meta/Classes/Mytest.yaml @@ -0,0 +1,8 @@ +Namespaces: + =: io.murano.test + +Extends: sys:TestFixture + +Name: MyTest + +Methods: diff --git a/murano/tests/unit/engine/meta/manifest.yaml b/murano/tests/unit/engine/meta/manifest.yaml new file mode 100644 index 00000000..aba9477a --- /dev/null +++ b/murano/tests/unit/engine/meta/manifest.yaml @@ -0,0 +1,10 @@ +Format: 1.0 +Type: Application +FullName: io.murano.test.MyTest +Name: Test case Example +Description: | + Example of simple package +Author: 'Mirantis, Inc' +Tags: [] +Classes: + io.murano.test.MyTest: Mytest.yaml diff --git a/murano/tests/unit/engine/test_package_loader.py b/murano/tests/unit/engine/test_package_loader.py new file mode 100644 index 00000000..3a4ab4a3 --- /dev/null +++ b/murano/tests/unit/engine/test_package_loader.py @@ -0,0 +1,67 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import mock +from oslo_config import cfg + +from murano.engine import package_loader +from murano.packages import mpl_package +from murano.tests.unit import base + +CONF = cfg.CONF + + +class TestCombinedPackageLoader(base.MuranoTestCase): + + @classmethod + def setUpClass(cls): + super(TestCombinedPackageLoader, cls).setUpClass() + + location = os.path.dirname(__file__) + CONF.set_override('load_packages_from', [location], 'engine') + cls.murano_client_factory = mock.MagicMock() + cls.loader = package_loader.CombinedPackageLoader( + cls.murano_client_factory, 'test_tenant_id') + cls.api_loader = mock.MagicMock() + cls.loader.loader_from_api = cls.api_loader + + cls.local_pkg_name = 'io.murano.test.MyTest' + cls.api_pkg_name = 'test.mpl.v1.app.Thing' + + def test_loaders_initialized(self): + self.assertEqual(1, len(self.loader.loaders_from_dir), + 'One directory class loader should be initialized' + ' since there is one valid murano pl package in the' + ' provided directory in config.') + self.assertIsInstance(self.loader.loaders_from_dir[0], + package_loader.DirectoryPackageLoader) + + def test_get_package_by_class_directory_loader(self): + result = self.loader.get_package_by_class(self.local_pkg_name) + self.assertIsInstance(result, mpl_package.MuranoPlPackage) + + def test_get_package_by_name_directory_loader(self): + result = self.loader.get_package(self.local_pkg_name) + self.assertIsInstance(result, mpl_package.MuranoPlPackage) + + def test_get_package_by_class_api_loader(self): + self.loader.get_package(self.api_pkg_name) + + self.api_loader.get_package.assert_called_with(self.api_pkg_name) + + def test_get_package_api_loader(self): + self.loader.get_package_by_class(self.api_pkg_name) + + self.api_loader.get_package_by_class.assert_called_with( + self.api_pkg_name)