openstack/deb-murano
Code Issues Proposed changes

Version-aware YAML loading

Browse Source 
* Refactoring of "packages" Python package
* Now MuranoPL YAML classes can be parsed by different engines
depending on Format string in package manifest

With this change MuranoPL classes will no longer
be read and parsed (including YAQL expressions parsing)
upon package load but only on first class access and
with ability to peak different parsers depending on
format version specified in manifest. As a consequence
it is now possible to use different YAQL engines for
different package format versions. Also startup
time was greatly improved as unneeded classes are no more
parsed and logos and UI forms are not loaded.

Partially implements: blueprint murano-versioning

Change-Id: I23fc5da1a43b405d526438329dc04aed589dee13
This commit is contained in:
Stan Lagun 2015-08-27 04:59:50 +03:00 committed by Victor Ryzhenkin
parent 068831ccd8
commit 9a9f3436e1
19 changed files with 445 additions and 438 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
murano/api/v1/__init__.py
View File

@ -24,7 +24,7 @@ LIST_PARAMS = ('category', 'tag', 'class', 'order_by')
ORDER_VALUES = ('fqn', 'name', 'created')
PKG_PARAMS_MAP = {'display_name': 'name',
'full_name': 'fully_qualified_name',
'raw_ui': 'ui_definition',
'ui': 'ui_definition',
'logo': 'logo',
'package_type': 'type',
'description': 'description',

31
murano/api/v1/catalog.py
View File

@ -226,8 +226,22 @@ class Controller(object):
tempf.write(content)
package_meta['archive'] = content
try:
pkg_to_upload = load_utils.load_from_file(
tempf.name, target_dir=None, drop_dir=True)
with load_utils.load_from_file(
tempf.name, target_dir=None,
drop_dir=True) as pkg_to_upload:
# extend dictionary for update db
for k, v in PKG_PARAMS_MAP.iteritems():
if hasattr(pkg_to_upload, k):
package_meta[v] = getattr(pkg_to_upload, k)
try:
package = db_api.package_upload(
package_meta, req.context.tenant)
except db_exc.DBDuplicateEntry:
msg = _('Package with specified full '
'name is already registered')
LOG.exception(msg)
raise exc.HTTPConflict(msg)
return package.to_dict()
except pkg_exc.PackageLoadError as e:
msg = _("Couldn't load package from file: {0}").format(e)
LOG.exception(msg)
@ -236,19 +250,6 @@ class Controller(object):
LOG.debug("Deleting package archive temporary file")
os.remove(tempf.name)
# extend dictionary for update db
for k, v in PKG_PARAMS_MAP.iteritems():
if hasattr(pkg_to_upload, k):
package_meta[v] = getattr(pkg_to_upload, k)
try:
package = db_api.package_upload(package_meta, req.context.tenant)
except db_exc.DBDuplicateEntry:
msg = _('Package with specified full name is already registered')
LOG.exception(msg)
raise exc.HTTPConflict(msg)
return package.to_dict()
def get_ui(self, req, package_id):
target = {'package_id': package_id}
policy.check("get_package", req.context, target)

2
murano/cmd/manage.py
View File

@ -73,7 +73,7 @@ def _do_import_package(_dir, categories, update=False):
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.raw_ui,
'ui_definition': pkg.ui,
'class_definitions': pkg.classes,
'archive': pkg.blob,
'categories': categories or []

8
murano/dsl/helpers.py
View File

@ -244,6 +244,14 @@ def parse_version_spec(version_spec):
return version_spec
def parse_version(version):
if isinstance(version, semantic_version.Version):
return version
if not version:
version = '0'
return semantic_version.Version.coerce(str(version))
def traverse(seed, producer=None, track_visited=True):
if not yaqlutils.is_iterable(seed):
seed = [seed]

5
murano/dsl/macros.py
View File

@ -95,10 +95,7 @@ class ContinueMacro(expressions.DslExpression):
class ParallelMacro(CodeBlock):
def __init__(self, Parallel, Limit=None):
super(ParallelMacro, self).__init__(Parallel)
if Limit:
self._limit = yaql_expression.YaqlExpression(str(Limit))
else:
self._limit = len(self.code_block)
self._limit = Limit or len(self.code_block)
def execute(self, context):
if not self.code_block:

12
murano/dsl/murano_package.py
View File

@ -36,8 +36,8 @@ class MuranoPackage(dsl_types.MuranoPackage):
super(MuranoPackage, self).__init__()
self._package_loader = weakref.proxy(package_loader)
self._name = name
self._version = self._parse_version(version)
self._runtime_version = self._parse_version(runtime_version)
self._version = helpers.parse_version(version)
self._runtime_version = helpers.parse_version(runtime_version)
self._requirements = {
name: semantic_version.Spec('==' + str(self._version.major))
}
@ -182,11 +182,3 @@ class MuranoPackage(dsl_types.MuranoPackage):
except exceptions.NoClassFound:
continue
raise exceptions.NoClassFound(name)
@staticmethod
def _parse_version(version):
if isinstance(version, semantic_version.Version):
return version
if not version:
version = '0'
return semantic_version.Version.coerce(str(version))

18
murano/dsl/yaql_expression.py
View File

@ -25,10 +25,12 @@ from murano.dsl import yaql_integration
class YaqlExpression(dsl_types.YaqlExpression):
def __init__(self, expression):
def __init__(self, expression, version):
self._version = version
if isinstance(expression, types.StringTypes):
self._expression = encodeutils.safe_encode(expression)
self._parsed_expression = yaql_integration.parse(self._expression)
self._parsed_expression = yaql_integration.parse(
self._expression, version)
self._file_position = None
elif isinstance(expression, YaqlExpression):
self._expression = expression._expression
@ -45,6 +47,10 @@ class YaqlExpression(dsl_types.YaqlExpression):
def expression(self):
return self._expression
@property
def version(self):
return self._version
@property
def source_file_position(self):
return self._file_position
@ -60,13 +66,13 @@ class YaqlExpression(dsl_types.YaqlExpression):
return self._expression
@staticmethod
def match(expr):
if not isinstance(expr, types.StringTypes):
def is_expression(expression, version):
if not isinstance(expression, types.StringTypes):
return False
if re.match('^[\s\w\d.:]*$', expr):
if re.match('^[\s\w\d.:]*$', expression):
return False
try:
yaql_integration.parse(expr)
yaql_integration.parse(expression, version)
return True
except yaql_exceptions.YaqlParsingException:
return False

8
murano/dsl/yaql_integration.py
View File

@ -80,8 +80,12 @@ def create_context():
return ROOT_CONTEXT.create_child_context()
def parse(expression):
return ENGINE(expression)
def choose_yaql_engine(version):
return ENGINE
def parse(expression, version):
return choose_yaql_engine(version)(expression)
def call_func(__context, __name, *args, **kwargs):

31
murano/engine/package_loader.py
View File

@ -125,7 +125,7 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
self._root_loader, app_package)
for name in app_package.classes:
dsl_package.register_class(
(lambda cls: lambda: app_package.get_class(cls))(name),
(lambda cls: lambda: get_class(app_package, cls))(name),
name)
if app_package.full_name == constants.CORE_LIBRARY:
system_objects.register(dsl_package)
@ -138,9 +138,7 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory, preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.exception(_LE(
'Unable to load package from cache. Clean-up...'))
@ -159,13 +157,11 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader,
preload=False
)
with load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False) as app_package:
return app_package
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
@ -215,15 +211,13 @@ class DirectoryPackageLoader(package_loader.MuranoPackageLoader):
def _build_index(self):
for folder in self.search_package_folders(self._base_path):
try:
package = load_utils.load_from_dir(
folder, preload=False,
loader=yaql_yaml_loader.YaqlYamlLoader)
package = load_utils.load_from_dir(folder)
dsl_package = murano_package.MuranoPackage(
self._root_loader, package)
for class_name in package.classes:
dsl_package.register_class(
(lambda pkg, cls:
lambda: pkg.get_class(cls))(package, class_name),
lambda: get_class(pkg, cls))(package, class_name),
class_name
)
if dsl_package.name == constants.CORE_LIBRARY:
@ -328,3 +322,10 @@ class CombinedPackageLoader(package_loader.MuranoPackageLoader):
def __exit__(self, exc_type, exc_val, exc_tb):
self.api_loader.cleanup()
return False
def get_class(package, name):
version = package.runtime_version
loader = yaql_yaml_loader.get_loader(version)
contents, file_id = package.get_class(name)
return loader(contents, file_id)

79
murano/engine/yaql_yaml_loader.py
View File

@ -18,52 +18,59 @@ import yaml.composer
import yaml.constructor
from murano.dsl import dsl_types
from murano.dsl import helpers
from murano.dsl import yaql_expression
class MuranoPlDict(dict):
pass
def get_loader(version):
version = helpers.parse_version(version)
class MuranoPlDict(dict):
pass
class MuranoPlYamlConstructor(yaml.constructor.Constructor):
def construct_yaml_map(self, node):
data = MuranoPlDict()
data.source_file_position = build_position(node)
yield data
value = self.construct_mapping(node)
data.update(value)
class YaqlExpression(yaql_expression.YaqlExpression):
@staticmethod
def match(expr):
return yaql_expression.YaqlExpression.is_expression(expr, version)
def load(contents, file_id):
def build_position(node):
return dsl_types.ExpressionFilePosition(
file_id,
node.start_mark.line + 1,
node.start_mark.column + 1,
node.end_mark.line + 1,
node.end_mark.column + 1)
class YaqlYamlLoader(yaml.Loader, MuranoPlYamlConstructor):
pass
class MuranoPlYamlConstructor(yaml.constructor.Constructor):
def construct_yaml_map(self, node):
data = MuranoPlDict()
data.source_file_position = build_position(node)
yield data
value = self.construct_mapping(node)
data.update(value)
class YaqlYamlLoader(yaml.Loader, MuranoPlYamlConstructor):
pass
YaqlYamlLoader.add_constructor(u'tag:yaml.org,2002:map',
MuranoPlYamlConstructor.construct_yaml_map)
YaqlYamlLoader.add_constructor(
u'tag:yaml.org,2002:map',
MuranoPlYamlConstructor.construct_yaml_map)
# workaround for PyYAML bug: http://pyyaml.org/ticket/221
resolvers = {}
for k, v in yaml.Loader.yaml_implicit_resolvers.items():
resolvers[k] = v[:]
YaqlYamlLoader.yaml_implicit_resolvers = resolvers
# workaround for PyYAML bug: http://pyyaml.org/ticket/221
resolvers = {}
for k, v in yaml.Loader.yaml_implicit_resolvers.items():
resolvers[k] = v[:]
YaqlYamlLoader.yaml_implicit_resolvers = resolvers
def yaql_constructor(loader, node):
value = loader.construct_scalar(node)
result = yaql_expression.YaqlExpression(value, version)
result.source_file_position = build_position(node)
return result
YaqlYamlLoader.add_constructor(u'!yaql', yaql_constructor)
YaqlYamlLoader.add_implicit_resolver(u'!yaql', YaqlExpression, None)
return yaml.load(contents, Loader=YaqlYamlLoader)
def build_position(node):
return dsl_types.ExpressionFilePosition(
node.start_mark.name,
node.start_mark.line + 1,
node.start_mark.column + 1,
node.end_mark.line + 1,
node.end_mark.column + 1)
def yaql_constructor(loader, node):
value = loader.construct_scalar(node)
result = yaql_expression.YaqlExpression(value)
result.source_file_position = build_position(node)
return result
YaqlYamlLoader.add_constructor(u'!yaql', yaql_constructor)
YaqlYamlLoader.add_implicit_resolver(u'!yaql', yaql_expression.YaqlExpression,
None)
return load

191
murano/packages/application_package.py
View File

@ -1,191 +0,0 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import imghdr
import io
import os
import re
import sys
import zipfile
from murano.packages import exceptions
class PackageTypes(object):
Library = 'Library'
Application = 'Application'
ALL = [Library, Application]
class ApplicationPackage(object):
def __init__(self, source_directory, manifest, loader):
self.yaml_loader = loader
self._source_directory = source_directory
self._full_name = None
self._package_type = None
self._display_name = None
self._description = None
self._author = None
self._supplier = {}
self._tags = None
self._logo = None
self._format = manifest.get('Format')
self._logo_cache = None
self._supplier_logo_cache = None
self._blob_cache = None
self._version = None
self._runtime_version = None
self._requirements = {}
@property
def full_name(self):
return self._full_name
@property
def version(self):
return self._version
@property
def classes(self):
return tuple()
@property
def runtime_version(self):
return self._runtime_version
@property
def requirements(self):
return self._requirements
@property
def package_type(self):
return self._package_type
@property
def display_name(self):
return self._display_name
@property
def description(self):
return self._description
@property
def author(self):
return self._author
@property
def supplier(self):
return self._supplier
@property
def tags(self):
return list(self._tags)
@property
def logo(self):
if not self._logo_cache:
self._load_logo(False)
return self._logo_cache
@property
def supplier_logo(self):
if not self._supplier_logo_cache:
self._load_supplier_logo(False)
return self._supplier_logo_cache
@property
def blob(self):
if not self._blob_cache:
self._blob_cache = _pack_dir(self._source_directory)
return self._blob_cache
def get_class(self, name):
raise exceptions.PackageClassLoadError(name)
def get_resource(self, name):
resources_dir = os.path.join(self._source_directory, 'Resources')
if not os.path.exists(resources_dir):
os.makedirs(resources_dir)
return os.path.join(resources_dir, name)
def load(self):
self._load_logo(True)
self._load_supplier_logo(True)
def _load_logo(self, validate=False):
logo_file = self._logo or 'logo.png'
full_path = os.path.join(self._source_directory, logo_file)
if not os.path.isfile(full_path) and logo_file == 'logo.png':
self._logo_cache = None
return
try:
if validate:
if imghdr.what(full_path) != 'png':
raise exceptions.PackageLoadError(
'Logo is not in PNG format')
with open(full_path) as stream:
self._logo_cache = stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
raise exceptions.PackageLoadError(
'Unable to load logo: ' + str(ex)), None, trace
def _load_supplier_logo(self, validate=False):
if 'Logo' not in self._supplier:
self._supplier['Logo'] = None
logo_file = self._supplier['Logo'] or 'supplier_logo.png'
full_path = os.path.join(self._source_directory, logo_file)
if not os.path.isfile(full_path) and logo_file == 'supplier_logo.png':
del self._supplier['Logo']
return
try:
if validate:
if imghdr.what(full_path) != 'png':
raise exceptions.PackageLoadError(
'Supplier Logo is not in PNG format')
with open(full_path) as stream:
self._supplier_logo_cache = stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
raise exceptions.PackageLoadError(
'Unable to load supplier logo: ' + str(ex)), None, trace
@staticmethod
def _check_full_name(full_name):
error = exceptions.PackageFormatError('Invalid FullName')
if re.match(r'^[\w\.]+$', full_name):
if full_name.startswith('.') or full_name.endswith('.'):
raise error
if '..' in full_name:
raise error
else:
raise error
def _zipdir(path, zipf):
for root, dirs, files in os.walk(path):
for f in files:
abspath = os.path.join(root, f)
relpath = os.path.relpath(abspath, path)
zipf.write(abspath, relpath)
def _pack_dir(source_directory):
blob = io.BytesIO()
zipf = zipfile.ZipFile(blob, mode='w')
_zipdir(source_directory, zipf)
zipf.close()
return blob.getvalue()

61
murano/packages/hot_package.py
View File

@ -17,84 +17,63 @@ import shutil
import sys
import types
import semantic_version
import yaml
from murano.dsl import yaql_expression
from murano.packages import application_package
from murano.packages import exceptions
from murano.packages import package_base
YAQL = yaql_expression.YaqlExpression
RESOURCES_DIR_NAME = 'Resources/'
HOT_FILES_DIR_NAME = 'HotFiles/'
HOT_ENV_DIR_NAME = 'HotEnvironments/'
class YAQL(object):
def __init__(self, expr):
self.expr = expr
class Dumper(yaml.Dumper):
pass
def yaql_representer(dumper, data):
return dumper.represent_scalar(u'!yaql', str(data))
return dumper.represent_scalar(u'!yaql', data.expr)
Dumper.add_representer(YAQL, yaql_representer)
class HotPackage(application_package.ApplicationPackage):
def __init__(self, source_directory, manifest, loader, runtime_version):
super(HotPackage, self).__init__(source_directory, manifest, loader)
class HotPackage(package_base.PackageBase):
def __init__(self, source_directory, manifest,
package_format, runtime_version):
super(HotPackage, self).__init__(
source_directory, manifest, package_format, runtime_version)
self._translated_class = None
self._source_directory = source_directory
self._translated_ui = None
self._full_name = manifest.get('FullName')
if not self._full_name:
raise exceptions.PackageFormatError(
'FullName not specified')
self._check_full_name(self._full_name)
self._package_type = manifest.get('Type')
if self._package_type not in application_package.PackageTypes.ALL:
raise exceptions.PackageFormatError('Invalid Package Type')
self._display_name = manifest.get('Name', self._full_name)
self._description = manifest.get('Description')
self._author = manifest.get('Author')
self._supplier = manifest.get('Supplier') or {}
self._logo = manifest.get('Logo')
self._tags = manifest.get('Tags')
self._version = semantic_version.Version.coerce(str(manifest.get(
'Version', '0.0.0')))
self._runtime_version = runtime_version
@property
def classes(self):
return self.full_name,
@property
def requirements(self):
return {}
@property
def ui(self):
if not self._translated_ui:
self._translated_ui = self._translate_ui()
return self._translated_ui
@property
def raw_ui(self):
ui_obj = self.ui
result = yaml.dump(ui_obj, Dumper=Dumper, default_style='"')
return result
def get_class(self, name):
if name != self.full_name:
raise exceptions.PackageClassLoadError(
name, 'Class not defined in this package')
if not self._translated_class:
self._translate_class()
return self._translated_class
def load(self):
self.get_class(self.full_name)
if not self._translated_ui:
self._translated_ui = self._translate_ui()
super(HotPackage, self).load()
return self._translated_class, '<generated code>'
def _translate_class(self):
template_file = os.path.join(self._source_directory, 'template.yaml')
@ -128,7 +107,7 @@ class HotPackage(application_package.ApplicationPackage):
files = HotPackage._translate_files(self._source_directory)
translated.update(HotPackage._generate_workflow(hot, files))
self._translated_class = translated
self._translated_class = yaml.dump(translated, Dumper=Dumper)
@staticmethod
def _build_properties(hot, validate_hot_parameters):
@ -546,4 +525,4 @@ class HotPackage(application_package.ApplicationPackage):
groups, self.full_name),
'Forms': forms
}
return translated
return yaml.dump(translated, Dumper=Dumper)

17
murano/packages/load_utils.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import contextlib
import os
import shutil
import string
@ -22,15 +23,13 @@ import zipfile
import semantic_version
import yaml
from murano.engine import yaql_yaml_loader
import murano.packages.application_package
import murano.packages.exceptions as e
import murano.packages.hot_package
import murano.packages.mpl_package
def load_from_file(archive_path, target_dir=None, drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader, preload=True):
@contextlib.contextmanager
def load_from_file(archive_path, target_dir=None, drop_dir=False):
if not os.path.isfile(archive_path):
raise e.PackageLoadError('Unable to find package file')
created = False
@ -50,7 +49,7 @@ def load_from_file(archive_path, target_dir=None, drop_dir=False,
"zip archive".format(archive_path))
package = zipfile.ZipFile(archive_path)
package.extractall(path=target_dir)
return load_from_dir(target_dir, preload=preload, loader=loader)
yield load_from_dir(target_dir)
except ValueError as err:
raise e.PackageLoadError("Couldn't load package from file: "
"{0}".format(err))
@ -63,8 +62,7 @@ def load_from_file(archive_path, target_dir=None, drop_dir=False,
os.unlink(os.path.join(target_dir, f))
def load_from_dir(source_directory, filename='manifest.yaml', preload=False,
loader=yaql_yaml_loader.YaqlYamlLoader):
def load_from_dir(source_directory, filename='manifest.yaml'):
formats = {
'MuranoPL': {
('1.0.0', '1.0.0'): murano.packages.mpl_package.MuranoPlPackage,
@ -104,9 +102,6 @@ def load_from_dir(source_directory, filename='manifest.yaml', preload=False,
min_version = semantic_version.Version(key[0])
max_version = semantic_version.Version(key[1])
if min_version <= version <= max_version:
package = value(source_directory, content, loader, version)
if preload:
package.load()
return package
return value(source_directory, content, parts[0], version)
raise e.PackageFormatError(
'Unsupported {0} format version {1}'.format(parts[0], version))

97
murano/packages/mpl_package.py
View File

@ -13,89 +13,37 @@
# under the License.
import os
import sys
import semantic_version
import yaml
from murano.packages import application_package
from murano.packages import exceptions
from murano.packages import package_base
class MuranoPlPackage(application_package.ApplicationPackage):
def __init__(self, source_directory, manifest, loader, runtime_version):
class MuranoPlPackage(package_base.PackageBase):
def __init__(self, source_directory, manifest,
package_format, runtime_version):
super(MuranoPlPackage, self).__init__(
source_directory, manifest, loader)
self._classes = None
self._ui = None
self._ui_cache = None
self._raw_ui_cache = None
self._classes_cache = {}
self._full_name = manifest.get('FullName')
if not self._full_name:
raise exceptions.PackageFormatError('FullName not specified')
self._check_full_name(self._full_name)
self._package_type = manifest.get('Type')
if self._package_type not in application_package.PackageTypes.ALL:
raise exceptions.PackageFormatError('Invalid Package Type')
self._display_name = manifest.get('Name', self._full_name)
self._description = manifest.get('Description')
self._author = manifest.get('Author')
self._supplier = manifest.get('Supplier') or {}
source_directory, manifest, package_format, runtime_version)
self._classes = manifest.get('Classes')
self._ui = manifest.get('UI', 'ui.yaml')
self._logo = manifest.get('Logo')
self._tags = manifest.get('Tags')
self._version = semantic_version.Version.coerce(str(manifest.get(
'Version', '0.0.0')))
self._runtime_version = runtime_version
self._ui_file = manifest.get('UI', 'ui.yaml')
self._requirements = manifest.get('Require') or {}
@property
def classes(self):
return tuple(self._classes.keys())
return self._classes.keys()
@property
def ui(self):
if not self._ui_cache:
self._load_ui(True)
return self._ui_cache
full_path = os.path.join(self._source_directory, 'UI', self._ui_file)
if not os.path.isfile(full_path):
return None
with open(full_path) as stream:
return stream.read()
@property
def raw_ui(self):
if not self._raw_ui_cache:
self._load_ui(False)
return self._raw_ui_cache
def requirements(self):
return self._requirements
def get_class(self, name):
if name not in self._classes_cache:
self._load_class(name)
return self._classes_cache[name]
# Private methods
def _load_ui(self, load_yaml=False):
if self._raw_ui_cache and load_yaml:
self._ui_cache = yaml.load(self._raw_ui_cache, self.yaml_loader)
else:
ui_file = self._ui
full_path = os.path.join(self._source_directory, 'UI', ui_file)
if not os.path.isfile(full_path):
self._raw_ui_cache = None
self._ui_cache = None
return
try:
with open(full_path) as stream:
self._raw_ui_cache = stream.read()
if load_yaml:
self._ui_cache = yaml.load(self._raw_ui_cache,
self.yaml_loader)
except Exception as ex:
trace = sys.exc_info()[2]
raise exceptions.PackageUILoadError(str(ex)), None, trace
def _load_class(self, name):
if name not in self._classes:
raise exceptions.PackageClassLoadError(
name, 'Class not defined in package ' + self.full_name)
@ -104,18 +52,5 @@ class MuranoPlPackage(application_package.ApplicationPackage):
if not os.path.isfile(full_path):
raise exceptions.PackageClassLoadError(
name, 'File with class definition not found')
try:
with open(full_path) as stream:
self._classes_cache[name] = yaml.load(stream, self.yaml_loader)
except Exception as ex:
trace = sys.exc_info()[2]
msg = 'Unable to load class definition due to "{0}"'.format(
str(ex))
raise exceptions.PackageClassLoadError(name, msg), None, trace
def load(self):
self._classes_cache.clear()
for class_name in self._classes:
self.get_class(class_name)
self._load_ui(True)
super(MuranoPlPackage, self).load()
with open(full_path) as stream:
return stream.read(), full_path

127
murano/packages/package.py Normal file
View File

@ -0,0 +1,127 @@
# Copyright (c) 2015 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import io
import os
import zipfile
import six
class PackageType(object):
Library = 'Library'
Application = 'Application'
ALL = [Library, Application]
@six.add_metaclass(abc.ABCMeta)
class Package(object):
def __init__(self, source_directory, package_format, runtime_version):
self._source_directory = source_directory
self._format = package_format
self._runtime_version = runtime_version
self._blob_cache = None
@property
def format(self):
return self._format
@abc.abstractproperty
def full_name(self):
raise NotImplementedError()
@abc.abstractproperty
def version(self):
raise NotImplementedError()
@abc.abstractproperty
def classes(self):
raise NotImplementedError()
@property
def runtime_version(self):
return self._runtime_version
@abc.abstractproperty
def requirements(self):
raise NotImplementedError()
@abc.abstractproperty
def package_type(self):
raise NotImplementedError()
@abc.abstractproperty
def display_name(self):
raise NotImplementedError()
@abc.abstractproperty
def description(self):
raise NotImplementedError()
@abc.abstractproperty
def author(self):
raise NotImplementedError()
@abc.abstractproperty
def supplier(self):
raise NotImplementedError()
@abc.abstractproperty
def tags(self):
raise NotImplementedError()
@abc.abstractproperty
def logo(self):
raise NotImplementedError()
@abc.abstractproperty
def supplier_logo(self):
raise NotImplementedError()
@property
def blob(self):
if not self._blob_cache:
self._blob_cache = _pack_dir(self._source_directory)
return self._blob_cache
@abc.abstractmethod
def get_class(self, name):
raise NotImplementedError()
@abc.abstractmethod
def get_resource(self, name):
raise NotImplementedError()
@abc.abstractproperty
def ui(self):
raise NotImplementedError()
def _zip_dir(path, zip_file):
for root, dirs, files in os.walk(path):
for f in files:
abs_path = os.path.join(root, f)
relative_path = os.path.relpath(abs_path, path)
zip_file.write(abs_path, relative_path)
def _pack_dir(source_directory):
blob = io.BytesIO()
zip_file = zipfile.ZipFile(blob, mode='w')
_zip_dir(source_directory, zip_file)
zip_file.close()
return blob.getvalue()

141
murano/packages/package_base.py Normal file
View File

@ -0,0 +1,141 @@
# Copyright (c) 2015 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import imghdr
import os
import re
import sys
import semantic_version
from murano.packages import exceptions
from murano.packages import package
class PackageBase(package.Package):
def __init__(self, source_directory, manifest,
package_format, runtime_version):
super(PackageBase, self).__init__(
source_directory, package_format, runtime_version)
self._full_name = manifest.get('FullName')
if not self._full_name:
raise exceptions.PackageFormatError('FullName is not specified')
self._check_full_name(self._full_name)
self._version = semantic_version.Version.coerce(str(manifest.get(
'Version', '0.0.0')))
self._package_type = manifest.get('Type')
if self._package_type not in package.PackageType.ALL:
raise exceptions.PackageFormatError(
'Invalid package Type {0}'.format(self._package_type))
self._display_name = manifest.get('Name', self._full_name)
self._description = manifest.get('Description')
self._author = manifest.get('Author')
self._supplier = manifest.get('Supplier') or {}
self._logo = manifest.get('Logo')
self._tags = manifest.get('Tags')
self._logo_cache = None
self._supplier_logo_cache = None
@abc.abstractproperty
def requirements(self):
raise NotImplementedError()
@abc.abstractproperty
def classes(self):
raise NotImplementedError()
@abc.abstractmethod
def get_class(self, name):
raise NotImplementedError()
@abc.abstractproperty
def ui(self):
raise NotImplementedError()
@property
def full_name(self):
return self._full_name
@property
def version(self):
return self._version
@property
def package_type(self):
return self._package_type
@property
def display_name(self):
return self._display_name
@property
def description(self):
return self._description
@property
def author(self):
return self._author
@property
def supplier(self):
return self._supplier
@property
def tags(self):
return list(self._tags)
@property
def logo(self):
return self._load_image(self._logo, 'logo.png', 'logo')
@property
def supplier_logo(self):
return self._load_image(
self._supplier.get('Logo'), 'supplier_logo.png', 'supplier logo')
def get_resource(self, name):
resources_dir = os.path.join(self._source_directory, 'Resources')
if not os.path.exists(resources_dir):
os.makedirs(resources_dir)
return os.path.join(resources_dir, name)
def _load_image(self, file_name, default_name, what_image):
full_path = os.path.join(
self._source_directory, file_name or default_name)
if not os.path.isfile(full_path) and not file_name:
return
try:
if imghdr.what(full_path) != 'png':
raise exceptions.PackageLoadError(
'{0} is not in PNG format'.format(what_image))
with open(full_path) as stream:
return stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
raise exceptions.PackageLoadError(
'Unable to load {0}: {1}'.format(what_image, ex)), None, trace
@staticmethod
def _check_full_name(full_name):
error = exceptions.PackageFormatError('Invalid FullName ' + full_name)
if re.match(r'^[\w\.]+$', full_name):
if full_name.startswith('.') or full_name.endswith('.'):
raise error
if '..' in full_name:
raise error
else:
raise error

9
murano/tests/unit/api/v1/test_catalog.py
View File

@ -227,8 +227,8 @@ class TestCatalogApi(test_base.ControllerTest, test_base.MuranoApiTestCase):
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.raw_ui,
'class_definitions': pkg.classes,
'ui_definition': pkg.ui,
'class_definitions': tuple(pkg.classes),
'archive': pkg.blob,
'categories': [],
}
@ -322,7 +322,10 @@ This is a fake zip archive
--BOUNDARY--'''
with mock.patch('murano.packages.load_utils.load_from_file') as lff:
lff.return_value = package_from_dir
ctxmgr = mock.Mock()
ctxmgr.__enter__ = mock.Mock(return_value=package_from_dir)
ctxmgr.__exit__ = mock.Mock(return_value=False)
lff.return_value = ctxmgr
# Uploading a non-public package
req = self._post(

12
murano/tests/unit/dsl/foundation/test_package_loader.py
View File

@ -15,8 +15,6 @@
import fnmatch
import os.path
import yaml
from murano.dsl import murano_package
from murano.dsl import namespace_resolver
from murano.dsl import package_loader
@ -25,22 +23,26 @@ from murano.tests.unit.dsl.foundation import object_model
class TestPackage(murano_package.MuranoPackage):
def __init__(self, package_loader, name, version,
def __init__(self, pkg_loader, name, version,
runtime_version, requirements, configs):
self.__configs = configs
super(TestPackage, self).__init__(
package_loader, name, version,
pkg_loader, name, version,
runtime_version, requirements)
def get_class_config(self, name):
return self.__configs.get(name, {})
def get_resource(self, name):
pass
class TestPackageLoader(package_loader.MuranoPackageLoader):
_classes_cache = {}
def __init__(self, directory, package_name, parent_loader=None):
self._package_name = package_name
self._yaml_loader = yaql_yaml_loader.get_loader('1.0')
if directory in TestPackageLoader._classes_cache:
self._classes = TestPackageLoader._classes_cache[directory]
else:
@ -80,7 +82,7 @@ class TestPackageLoader(package_loader.MuranoPackageLoader):
def _load_class(self, class_def_file):
with open(class_def_file) as stream:
data = yaml.load(stream, yaql_yaml_loader.YaqlYamlLoader)
data = self._yaml_loader(stream.read(), class_def_file)
if 'Name' not in data:
return

32
murano/tests/unit/test_engine.py
View File

@ -17,6 +17,7 @@
import re
import mock
import semantic_version
import yaql
from yaql.language import exceptions
from yaql.language import utils
@ -30,7 +31,6 @@ ROOT_CLASS = 'io.murano.Object'
class TestNamespaceResolving(base.MuranoTestCase):
def setUp(self):
super(TestNamespaceResolving, self).setUp()
@ -114,7 +114,6 @@ class Bunch(object):
class TestHelperFunctions(base.MuranoTestCase):
def setUp(self):
super(TestHelperFunctions, self).setUp()
@ -132,7 +131,6 @@ class TestHelperFunctions(base.MuranoTestCase):
'atom': ('some', (1, 'atom'), 'hi!'),
'sample': ('atom', (0, 1, 2, 3, 4))
})
print yaql_value(1)
context = yaql.create_context()
evaluated_value = helpers.evaluate(yaql_value, context)
evaluated_complex_value = helpers.evaluate(complex_value, context)
@ -142,50 +140,52 @@ class TestHelperFunctions(base.MuranoTestCase):
class TestYaqlExpression(base.MuranoTestCase):
def setUp(self):
self._version = semantic_version.Version.coerce('1.0')
super(TestYaqlExpression, self).setUp()
def test_expression(self):
yaql_expr = yaql_expression.YaqlExpression('string')
yaql_expr = yaql_expression.YaqlExpression('string', self._version)
self.assertEqual('string', yaql_expr.expression)
def test_unicode_expression(self):
yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'")
yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'",
self._version)
self.assertEqual(u"'yaql ♥ unicode'".encode('utf-8'),
yaql_expr.expression)
def test_unicode_expression_expression(self):
yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'")
yaql_expr2 = yaql_expression.YaqlExpression(yaql_expr)
yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'",
self._version)
yaql_expr2 = yaql_expression.YaqlExpression(yaql_expr, self._version)
self.assertEqual(u"'yaql ♥ unicode'".encode('utf-8'),
yaql_expr2.expression)
def test_evaluate_calls(self):
string = 'string'
expected_calls = [mock.call(string),
expected_calls = [mock.call(string, self._version),
mock.call().evaluate(context=None)]
with mock.patch('murano.dsl.yaql_integration.parse') as mock_parse:
yaql_expr = yaql_expression.YaqlExpression(string)
yaql_expr = yaql_expression.YaqlExpression(string, self._version)
yaql_expr(None)
self.assertEqual(expected_calls, mock_parse.mock_calls)
def test_match_returns(self):
expr = yaql_expression.YaqlExpression('string')
def test_is_expression_returns(self):
expr = yaql_expression.YaqlExpression('string', self._version)
with mock.patch('murano.dsl.yaql_integration.parse'):
self.assertTrue(expr.match('$some'))
self.assertTrue(expr.match('$.someMore'))
self.assertTrue(expr.is_expression('$some', self._version))
self.assertTrue(expr.is_expression('$.someMore', self._version))
with mock.patch('murano.dsl.yaql_integration.parse') as parse_mock:
parse_mock.side_effect = exceptions.YaqlGrammarException
self.assertFalse(expr.match(''))
self.assertFalse(expr.is_expression('', self._version))
with mock.patch('murano.dsl.yaql_integration.parse') as parse_mock:
parse_mock.side_effect = exceptions.YaqlLexicalException
self.assertFalse(expr.match(''))
self.assertFalse(expr.is_expression('', self._version))