openstack/deb-murano
Code Issues Proposed changes

Merge "Version-aware YAML loading"

Browse Source
This commit is contained in:
Jenkins 2015-09-03 17:22:22 +00:00 committed by Gerrit Code Review
commit 29dd968da5
19 changed files with 445 additions and 438 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
murano/api/v1/__init__.py
View File

@ -24,7 +24,7 @@ LIST_PARAMS = ('category', 'tag', 'class', 'order_by')
ORDER_VALUES = ('fqn', 'name', 'created')
PKG_PARAMS_MAP = {'display_name': 'name',
'full_name': 'fully_qualified_name',
'raw_ui': 'ui_definition',
'ui': 'ui_definition',
'logo': 'logo',
'package_type': 'type',
'description': 'description',

31
murano/api/v1/catalog.py
View File

@ -226,8 +226,22 @@ class Controller(object):
tempf.write(content)
package_meta['archive'] = content
try:
pkg_to_upload = load_utils.load_from_file(
tempf.name, target_dir=None, drop_dir=True)
with load_utils.load_from_file(
tempf.name, target_dir=None,
drop_dir=True) as pkg_to_upload:
# extend dictionary for update db
for k, v in PKG_PARAMS_MAP.iteritems():
if hasattr(pkg_to_upload, k):
package_meta[v] = getattr(pkg_to_upload, k)
try:
package = db_api.package_upload(
package_meta, req.context.tenant)
except db_exc.DBDuplicateEntry:
msg = _('Package with specified full '
'name is already registered')
LOG.exception(msg)
raise exc.HTTPConflict(msg)
return package.to_dict()
except pkg_exc.PackageLoadError as e:
msg = _("Couldn't load package from file: {0}").format(e)
LOG.exception(msg)
@ -236,19 +250,6 @@ class Controller(object):
LOG.debug("Deleting package archive temporary file")
os.remove(tempf.name)
# extend dictionary for update db
for k, v in PKG_PARAMS_MAP.iteritems():
if hasattr(pkg_to_upload, k):
package_meta[v] = getattr(pkg_to_upload, k)
try:
package = db_api.package_upload(package_meta, req.context.tenant)
except db_exc.DBDuplicateEntry:
msg = _('Package with specified full name is already registered')
LOG.exception(msg)
raise exc.HTTPConflict(msg)
return package.to_dict()
def get_ui(self, req, package_id):
target = {'package_id': package_id}
policy.check("get_package", req.context, target)

2
murano/cmd/manage.py
View File

@ -73,7 +73,7 @@ def _do_import_package(_dir, categories, update=False):
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.raw_ui,
'ui_definition': pkg.ui,
'class_definitions': pkg.classes,
'archive': pkg.blob,
'categories': categories or []

8
murano/dsl/helpers.py
View File

@ -244,6 +244,14 @@ def parse_version_spec(version_spec):
return version_spec
def parse_version(version):
if isinstance(version, semantic_version.Version):
return version
if not version:
version = '0'
return semantic_version.Version.coerce(str(version))
def traverse(seed, producer=None, track_visited=True):
if not yaqlutils.is_iterable(seed):
seed = [seed]

5
murano/dsl/macros.py
View File

@ -95,10 +95,7 @@ class ContinueMacro(expressions.DslExpression):
class ParallelMacro(CodeBlock):
def __init__(self, Parallel, Limit=None):
super(ParallelMacro, self).__init__(Parallel)
if Limit:
self._limit = yaql_expression.YaqlExpression(str(Limit))
else:
self._limit = len(self.code_block)
self._limit = Limit or len(self.code_block)
def execute(self, context):
if not self.code_block:

12
murano/dsl/murano_package.py
View File

@ -36,8 +36,8 @@ class MuranoPackage(dsl_types.MuranoPackage):
super(MuranoPackage, self).__init__()
self._package_loader = weakref.proxy(package_loader)
self._name = name
self._version = self._parse_version(version)
self._runtime_version = self._parse_version(runtime_version)
self._version = helpers.parse_version(version)
self._runtime_version = helpers.parse_version(runtime_version)
self._requirements = {
name: semantic_version.Spec('==' + str(self._version.major))
}
@ -182,11 +182,3 @@ class MuranoPackage(dsl_types.MuranoPackage):
except exceptions.NoClassFound:
continue
raise exceptions.NoClassFound(name)
@staticmethod
def _parse_version(version):
if isinstance(version, semantic_version.Version):
return version
if not version:
version = '0'
return semantic_version.Version.coerce(str(version))

18
murano/dsl/yaql_expression.py
View File

@ -25,10 +25,12 @@ from murano.dsl import yaql_integration
class YaqlExpression(dsl_types.YaqlExpression):
def __init__(self, expression):
def __init__(self, expression, version):
self._version = version
if isinstance(expression, types.StringTypes):
self._expression = encodeutils.safe_encode(expression)
self._parsed_expression = yaql_integration.parse(self._expression)
self._parsed_expression = yaql_integration.parse(
self._expression, version)
self._file_position = None
elif isinstance(expression, YaqlExpression):
self._expression = expression._expression
@ -45,6 +47,10 @@ class YaqlExpression(dsl_types.YaqlExpression):
def expression(self):
return self._expression
@property
def version(self):
return self._version
@property
def source_file_position(self):
return self._file_position
@ -60,13 +66,13 @@ class YaqlExpression(dsl_types.YaqlExpression):
return self._expression
@staticmethod
def match(expr):
if not isinstance(expr, types.StringTypes):
def is_expression(expression, version):
if not isinstance(expression, types.StringTypes):
return False
if re.match('^[\s\w\d.:]*$', expr):
if re.match('^[\s\w\d.:]*$', expression):
return False
try:
yaql_integration.parse(expr)
yaql_integration.parse(expression, version)
return True
except yaql_exceptions.YaqlParsingException:
return False

8
murano/dsl/yaql_integration.py
View File

@ -80,8 +80,12 @@ def create_context():
return ROOT_CONTEXT.create_child_context()
def parse(expression):
return ENGINE(expression)
def choose_yaql_engine(version):
return ENGINE
def parse(expression, version):
return choose_yaql_engine(version)(expression)
def call_func(__context, __name, *args, **kwargs):

31
murano/engine/package_loader.py
View File

@ -125,7 +125,7 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
self._root_loader, app_package)
for name in app_package.classes:
dsl_package.register_class(
(lambda cls: lambda: app_package.get_class(cls))(name),
(lambda cls: lambda: get_class(app_package, cls))(name),
name)
if app_package.full_name == constants.CORE_LIBRARY:
system_objects.register(dsl_package)
@ -138,9 +138,7 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory, preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.exception(_LE(
'Unable to load package from cache. Clean-up...'))
@ -159,13 +157,11 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader,
preload=False
)
with load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False) as app_package:
return app_package
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
@ -215,15 +211,13 @@ class DirectoryPackageLoader(package_loader.MuranoPackageLoader):
def _build_index(self):
for folder in self.search_package_folders(self._base_path):
try:
package = load_utils.load_from_dir(
folder, preload=False,
loader=yaql_yaml_loader.YaqlYamlLoader)
package = load_utils.load_from_dir(folder)
dsl_package = murano_package.MuranoPackage(
self._root_loader, package)
for class_name in package.classes:
dsl_package.register_class(
(lambda pkg, cls:
lambda: pkg.get_class(cls))(package, class_name),
lambda: get_class(pkg, cls))(package, class_name),
class_name
)
if dsl_package.name == constants.CORE_LIBRARY:
@ -328,3 +322,10 @@ class CombinedPackageLoader(package_loader.MuranoPackageLoader):
def __exit__(self, exc_type, exc_val, exc_tb):
self.api_loader.cleanup()
return False
def get_class(package, name):
version = package.runtime_version
loader = yaql_yaml_loader.get_loader(version)
contents, file_id = package.get_class(name)
return loader(contents, file_id)

79
murano/engine/yaql_yaml_loader.py
View File

@ -18,52 +18,59 @@ import yaml.composer
import yaml.constructor
from murano.dsl import dsl_types
from murano.dsl import helpers
from murano.dsl import yaql_expression
class MuranoPlDict(dict):
pass
def get_loader(version):
version = helpers.parse_version(version)
class MuranoPlDict(dict):
pass
class MuranoPlYamlConstructor(yaml.constructor.Constructor):
def construct_yaml_map(self, node):
data = MuranoPlDict()
data.source_file_position = build_position(node)
yield data
value = self.construct_mapping(node)
data.update(value)
class YaqlExpression(yaql_expression.YaqlExpression):
@staticmethod
def match(expr):
return yaql_expression.YaqlExpression.is_expression(expr, version)
def load(contents, file_id):
def build_position(node):
return dsl_types.ExpressionFilePosition(
file_id,
node.start_mark.line + 1,
node.start_mark.column + 1,
node.end_mark.line + 1,
node.end_mark.column + 1)
class YaqlYamlLoader(yaml.Loader, MuranoPlYamlConstructor):
pass
class MuranoPlYamlConstructor(yaml.constructor.Constructor):
def construct_yaml_map(self, node):
data = MuranoPlDict()
data.source_file_position = build_position(node)
yield data
value = self.construct_mapping(node)
data.update(value)
class YaqlYamlLoader(yaml.Loader, MuranoPlYamlConstructor):
pass
YaqlYamlLoader.add_constructor(u'tag:yaml.org,2002:map',
MuranoPlYamlConstructor.construct_yaml_map)
YaqlYamlLoader.add_constructor(
u'tag:yaml.org,2002:map',
MuranoPlYamlConstructor.construct_yaml_map)
# workaround for PyYAML bug: http://pyyaml.org/ticket/221
resolvers = {}
for k, v in yaml.Loader.yaml_implicit_resolvers.items():
resolvers[k] = v[:]
YaqlYamlLoader.yaml_implicit_resolvers = resolvers
# workaround for PyYAML bug: http://pyyaml.org/ticket/221
resolvers = {}
for k, v in yaml.Loader.yaml_implicit_resolvers.items():
resolvers[k] = v[:]
YaqlYamlLoader.yaml_implicit_resolvers = resolvers
def yaql_constructor(loader, node):
value = loader.construct_scalar(node)
result = yaql_expression.YaqlExpression(value, version)
result.source_file_position = build_position(node)
return result
YaqlYamlLoader.add_constructor(u'!yaql', yaql_constructor)
YaqlYamlLoader.add_implicit_resolver(u'!yaql', YaqlExpression, None)
return yaml.load(contents, Loader=YaqlYamlLoader)
def build_position(node):
return dsl_types.ExpressionFilePosition(
node.start_mark.name,
node.start_mark.line + 1,
node.start_mark.column + 1,
node.end_mark.line + 1,
node.end_mark.column + 1)
def yaql_constructor(loader, node):
value = loader.construct_scalar(node)
result = yaql_expression.YaqlExpression(value)
result.source_file_position = build_position(node)
return result
YaqlYamlLoader.add_constructor(u'!yaql', yaql_constructor)
YaqlYamlLoader.add_implicit_resolver(u'!yaql', yaql_expression.YaqlExpression,
None)
return load

191
murano/packages/application_package.py
View File

@ -1,191 +0,0 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import imghdr
import io
import os
import re
import sys
import zipfile
from murano.packages import exceptions
class PackageTypes(object):
Library = 'Library'
Application = 'Application'
ALL = [Library, Application]
class ApplicationPackage(object):
def __init__(self, source_directory, manifest, loader):
self.yaml_loader = loader
self._source_directory = source_directory
self._full_name = None
self._package_type = None
self._display_name = None
self._description = None
self._author = None
self._supplier = {}
self._tags = None
self._logo = None
self._format = manifest.get('Format')
self._logo_cache = None
self._supplier_logo_cache = None
self._blob_cache = None
self._version = None
self._runtime_version = None
self._requirements = {}
@property
def full_name(self):
return self._full_name
@property
def version(self):
return self._version
@property
def classes(self):
return tuple()
@property
def runtime_version(self):
return self._runtime_version
@property
def requirements(self):
return self._requirements
@property
def package_type(self):
return self._package_type
@property
def display_name(self):
return self._display_name
@property
def description(self):
return self._description
@property
def author(self):
return self._author
@property
def supplier(self):
return self._supplier
@property
def tags(self):
return list(self._tags)
@property
def logo(self):
if not self._logo_cache:
self._load_logo(False)
return self._logo_cache
@property
def supplier_logo(self):
if not self._supplier_logo_cache:
self._load_supplier_logo(False)
return self._supplier_logo_cache
@property
def blob(self):
if not self._blob_cache:
self._blob_cache = _pack_dir(self._source_directory)
return self._blob_cache
def get_class(self, name):
raise exceptions.PackageClassLoadError(name)
def get_resource(self, name):
resources_dir = os.path.join(self._source_directory, 'Resources')
if not os.path.exists(resources_dir):
os.makedirs(resources_dir)
return os.path.join(resources_dir, name)
def load(self):
self._load_logo(True)
self._load_supplier_logo(True)
def _load_logo(self, validate=False):
logo_file = self._logo or 'logo.png'
full_path = os.path.join(self._source_directory, logo_file)
if not os.path.isfile(full_path) and logo_file == 'logo.png':
self._logo_cache = None
return
try:
if validate:
if imghdr.what(full_path) != 'png':
raise exceptions.PackageLoadError(
'Logo is not in PNG format')
with open(full_path) as stream:
self._logo_cache = stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
raise exceptions.PackageLoadError(
'Unable to load logo: ' + str(ex)), None, trace
def _load_supplier_logo(self, validate=False):
if 'Logo' not in self._supplier:
self._supplier['Logo'] = None
logo_file = self._supplier['Logo'] or 'supplier_logo.png'
full_path = os.path.join(self._source_directory, logo_file)
if not os.path.isfile(full_path) and logo_file == 'supplier_logo.png':
del self._supplier['Logo']
return
try:
if validate:
if imghdr.what(full_path) != 'png':
raise exceptions.PackageLoadError(
'Supplier Logo is not in PNG format')
with open(full_path) as stream:
self._supplier_logo_cache = stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
raise exceptions.PackageLoadError(
'Unable to load supplier logo: ' + str(ex)), None, trace
@staticmethod
def _check_full_name(full_name):
error = exceptions.PackageFormatError('Invalid FullName')
if re.match(r'^[\w\.]+$', full_name):
if full_name.startswith('.') or full_name.endswith('.'):
raise error
if '..' in full_name:
raise error
else:
raise error
def _zipdir(path, zipf):
for root, dirs, files in os.walk(path):
for f in files:
abspath = os.path.join(root, f)
relpath = os.path.relpath(abspath, path)
zipf.write(abspath, relpath)
def _pack_dir(source_directory):
blob = io.BytesIO()
zipf = zipfile.ZipFile(blob, mode='w')
_zipdir(source_directory, zipf)
zipf.close()
return blob.getvalue()

61
murano/packages/hot_package.py
View File

@ -17,84 +17,63 @@ import shutil
import sys
import types
import semantic_version
import yaml
from murano.dsl import yaql_expression
from murano.packages import application_package
from murano.packages import exceptions
from murano.packages import package_base
YAQL = yaql_expression.YaqlExpression
RESOURCES_DIR_NAME = 'Resources/'
HOT_FILES_DIR_NAME = 'HotFiles/'
HOT_ENV_DIR_NAME = 'HotEnvironments/'
class YAQL(object):
def __init__(self, expr):
self.expr = expr
class Dumper(yaml.Dumper):
pass
def yaql_representer(dumper, data):
return dumper.represent_scalar(u'!yaql', str(data))
return dumper.represent_scalar(u'!yaql', data.expr)
Dumper.add_representer(YAQL, yaql_representer)
class HotPackage(application_package.ApplicationPackage):
def __init__(self, source_directory, manifest, loader, runtime_version):
super(HotPackage, self).__init__(source_directory, manifest, loader)
class HotPackage(package_base.PackageBase):
def __init__(self, source_directory, manifest,
package_format, runtime_version):
super(HotPackage, self).__init__(
source_directory, manifest, package_format, runtime_version)
self._translated_class = None
self._source_directory = source_directory
self._translated_ui = None
self._full_name = manifest.get('FullName')
if not self._full_name:
raise exceptions.PackageFormatError(
'FullName not specified')
self._check_full_name(self._full_name)
self._package_type = manifest.get('Type')
if self._package_type not in application_package.PackageTypes.ALL:
raise exceptions.PackageFormatError('Invalid Package Type')
self._display_name = manifest.get('Name', self._full_name)
self._description = manifest.get('Description')
self._author = manifest.get('Author')
self._supplier = manifest.get('Supplier') or {}
self._logo = manifest.get('Logo')
self._tags = manifest.get('Tags')
self._version = semantic_version.Version.coerce(str(manifest.get(
'Version', '0.0.0')))
self._runtime_version = runtime_version
@property
def classes(self):
return self.full_name,
@property
def requirements(self):
return {}
@property
def ui(self):
if not self._translated_ui:
self._translated_ui = self._translate_ui()
return self._translated_ui
@property
def raw_ui(self):
ui_obj = self.ui
result = yaml.dump(ui_obj, Dumper=Dumper, default_style='"')
return result
def get_class(self, name):
if name != self.full_name:
raise exceptions.PackageClassLoadError(
name, 'Class not defined in this package')
if not self._translated_class:
self._translate_class()
return self._translated_class
def load(self):
self.get_class(self.full_name)
if not self._translated_ui:
self._translated_ui = self._translate_ui()
super(HotPackage, self).load()
return self._translated_class, '<generated code>'
def _translate_class(self):
template_file = os.path.join(self._source_directory, 'template.yaml')
@ -128,7 +107,7 @@ class HotPackage(application_package.ApplicationPackage):
files = HotPackage._translate_files(self._source_directory)
translated.update(HotPackage._generate_workflow(hot, files))
self._translated_class = translated
self._translated_class = yaml.dump(translated, Dumper=Dumper)
@staticmethod
def _build_properties(hot, validate_hot_parameters):
@ -546,4 +525,4 @@ class HotPackage(application_package.ApplicationPackage):
groups, self.full_name),
'Forms': forms
}
return translated
return yaml.dump(translated, Dumper=Dumper)

17
murano/packages/load_utils.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import contextlib
import os
import shutil
import string
@ -22,15 +23,13 @@ import zipfile
import semantic_version
import yaml
from murano.engine import yaql_yaml_loader
import murano.packages.application_package
import murano.packages.exceptions as e
import murano.packages.hot_package
import murano.packages.mpl_package
def load_from_file(archive_path, target_dir=None, drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader, preload=True):
@contextlib.contextmanager
def load_from_file(archive_path, target_dir=None, drop_dir=False):
if not os.path.isfile(archive_path):
raise e.PackageLoadError('Unable to find package file')
created = False
@ -50,7 +49,7 @@ def load_from_file(archive_path, target_dir=None, drop_dir=False,
"zip archive".format(archive_path))
package = zipfile.ZipFile(archive_path)
package.extractall(path=target_dir)
return load_from_dir(target_dir, preload=preload, loader=loader)
yield load_from_dir(target_dir)
except ValueError as err:
raise e.PackageLoadError("Couldn't load package from file: "
"{0}".format(err))
@ -63,8 +62,7 @@ def load_from_file(archive_path, target_dir=None, drop_dir=False,
os.unlink(os.path.join(target_dir, f))
def load_from_dir(source_directory, filename='manifest.yaml', preload=False,
loader=yaql_yaml_loader.YaqlYamlLoader):
def load_from_dir(source_directory, filename='manifest.yaml'):
formats = {
'MuranoPL': {
('1.0.0', '1.0.0'): murano.packages.mpl_package.MuranoPlPackage,
@ -104,9 +102,6 @@ def load_from_dir(source_directory, filename='manifest.yaml', preload=False,
min_version = semantic_version.Version(key[0])
max_version = semantic_version.Version(key[1])
if min_version <= version <= max_version:
package = value(source_directory, content, loader, version)
if preload:
package.load()
return package
return value(source_directory, content, parts[0], version)
raise e.PackageFormatError(
'Unsupported {0} format version {1}'.format(parts[0], version))

97
murano/packages/mpl_package.py
View File

@ -13,89 +13,37 @@
# under the License.
import os
import sys
import semantic_version
import yaml
from murano.packages import application_package
from murano.packages import exceptions
from murano.packages import package_base
class MuranoPlPackage(application_package.ApplicationPackage):
def __init__(self, source_directory, manifest, loader, runtime_version):
class MuranoPlPackage(package_base.PackageBase):
def __init__(self, source_directory, manifest,
package_format, runtime_version):
super(MuranoPlPackage, self).__init__(
source_directory, manifest, loader)
self._classes = None
self._ui = None
self._ui_cache = None
self._raw_ui_cache = None
self._classes_cache = {}
self._full_name = manifest.get('FullName')
if not self._full_name:
raise exceptions.PackageFormatError('FullName not specified')
self._check_full_name(self._full_name)
self._package_type = manifest.get('Type')
if self._package_type not in application_package.PackageTypes.ALL:
raise exceptions.PackageFormatError('Invalid Package Type')
self._display_name = manifest.get('Name', self._full_name)
self._description = manifest.get('Description')
self._author = manifest.get('Author')
self._supplier = manifest.get('Supplier') or {}
source_directory, manifest, package_format, runtime_version)
self._classes = manifest.get('Classes')
self._ui = manifest.get('UI', 'ui.yaml')
self._logo = manifest.get('Logo')
self._tags = manifest.get('Tags')
self._version = semantic_version.Version.coerce(str(manifest.get(
'Version', '0.0.0')))
self._runtime_version = runtime_version
self._ui_file = manifest.get('UI', 'ui.yaml')
self._requirements = manifest.get('Require') or {}
@property
def classes(self):
return tuple(self._classes.keys())
return self._classes.keys()
@property
def ui(self):
if not self._ui_cache:
self._load_ui(True)
return self._ui_cache
full_path = os.path.join(self._source_directory, 'UI', self._ui_file)
if not os.path.isfile(full_path):
return None
with open(full_path) as stream:
return stream.read()
@property
def raw_ui(self):
if not self._raw_ui_cache:
self._load_ui(False)
return self._raw_ui_cache
def requirements(self):
return self._requirements
def get_class(self, name):
if name not in self._classes_cache:
self._load_class(name)
return self._classes_cache[name]
# Private methods
def _load_ui(self, load_yaml=False):
if self._raw_ui_cache and load_yaml:
self._ui_cache = yaml.load(self._raw_ui_cache, self.yaml_loader)
else:
ui_file = self._ui
full_path = os.path.join(self._source_directory, 'UI', ui_file)
if not os.path.isfile(full_path):
self._raw_ui_cache = None
self._ui_cache = None
return
try:
with open(full_path) as stream:
self._raw_ui_cache = stream.read()
if load_yaml:
self._ui_cache = yaml.load(self._raw_ui_cache,
self.yaml_loader)
except Exception as ex:
trace = sys.exc_info()[2]
raise exceptions.PackageUILoadError(str(ex)), None, trace
def _load_class(self, name):
if name not in self._classes:
raise exceptions.PackageClassLoadError(
name, 'Class not defined in package ' + self.full_name)
@ -104,18 +52,5 @@ class MuranoPlPackage(application_package.ApplicationPackage):
if not os.path.isfile(full_path):
raise exceptions.PackageClassLoadError(
name, 'File with class definition not found')
try:
with open(full_path) as stream:
self._classes_cache[name] = yaml.load(stream, self.yaml_loader)
except Exception as ex:
trace = sys.exc_info()[2]
msg = 'Unable to load class definition due to "{0}"'.format(
str(ex))
raise exceptions.PackageClassLoadError(name, msg), None, trace
def load(self):
self._classes_cache.clear()
for class_name in self._classes:
self.get_class(class_name)
self._load_ui(True)
super(MuranoPlPackage, self).load()
with open(full_path) as stream:
return stream.read(), full_path

127
murano/packages/package.py Normal file
View File

@ -0,0 +1,127 @@
# Copyright (c) 2015 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import io
import os
import zipfile
import six
class PackageType(object):
Library = 'Library'
Application = 'Application'
ALL = [Library, Application]
@six.add_metaclass(abc.ABCMeta)
class Package(object):
def __init__(self, source_directory, package_format, runtime_version):
self._source_directory = source_directory
self._format = package_format
self._runtime_version = runtime_version
self._blob_cache = None
@property
def format(self):
return self._format
@abc.abstractproperty
def full_name(self):
raise NotImplementedError()
@abc.abstractproperty
def version(self):
raise NotImplementedError()
@abc.abstractproperty
def classes(self):
raise NotImplementedError()
@property
def runtime_version(self):
return self._runtime_version
@abc.abstractproperty
def requirements(self):
raise NotImplementedError()
@abc.abstractproperty
def package_type(self):
raise NotImplementedError()
@abc.abstractproperty
def display_name(self):
raise NotImplementedError()
@abc.abstractproperty
def description(self):
raise NotImplementedError()
@abc.abstractproperty
def author(self):
raise NotImplementedError()
@abc.abstractproperty
def supplier(self):
raise NotImplementedError()
@abc.abstractproperty
def tags(self):
raise NotImplementedError()
@abc.abstractproperty
def logo(self):
raise NotImplementedError()
@abc.abstractproperty
def supplier_logo(self):
raise NotImplementedError()
@property
def blob(self):
if not self._blob_cache:
self._blob_cache = _pack_dir(self._source_directory)
return self._blob_cache
@abc.abstractmethod
def get_class(self, name):
raise NotImplementedError()
@abc.abstractmethod
def get_resource(self, name):
raise NotImplementedError()
@abc.abstractproperty
def ui(self):
raise NotImplementedError()
def _zip_dir(path, zip_file):
for root, dirs, files in os.walk(path):
for f in files:
abs_path = os.path.join(root, f)
relative_path = os.path.relpath(abs_path, path)
zip_file.write(abs_path, relative_path)
def _pack_dir(source_directory):
blob = io.BytesIO()
zip_file = zipfile.ZipFile(blob, mode='w')
_zip_dir(source_directory, zip_file)
zip_file.close()
return blob.getvalue()

141
murano/packages/package_base.py Normal file
View File

@ -0,0 +1,141 @@
# Copyright (c) 2015 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import imghdr
import os
import re
import sys
import semantic_version
from murano.packages import exceptions
from murano.packages import package
class PackageBase(package.Package):
def __init__(self, source_directory, manifest,
package_format, runtime_version):
super(PackageBase, self).__init__(
source_directory, package_format, runtime_version)
self._full_name = manifest.get('FullName')
if not self._full_name:
raise exceptions.PackageFormatError('FullName is not specified')
self._check_full_name(self._full_name)
self._version = semantic_version.Version.coerce(str(manifest.get(
'Version', '0.0.0')))
self._package_type = manifest.get('Type')
if self._package_type not in package.PackageType.ALL:
raise exceptions.PackageFormatError(
'Invalid package Type {0}'.format(self._package_type))
self._display_name = manifest.get('Name', self._full_name)
self._description = manifest.get('Description')
self._author = manifest.get('Author')
self._supplier = manifest.get('Supplier') or {}
self._logo = manifest.get('Logo')
self._tags = manifest.get('Tags')
self._logo_cache = None
self._supplier_logo_cache = None
@abc.abstractproperty
def requirements(self):
raise NotImplementedError()
@abc.abstractproperty
def classes(self):
raise NotImplementedError()
@abc.abstractmethod
def get_class(self, name):
raise NotImplementedError()
@abc.abstractproperty
def ui(self):
raise NotImplementedError()
@property
def full_name(self):
return self._full_name
@property
def version(self):
return self._version
@property
def package_type(self):
return self._package_type
@property
def display_name(self):
return self._display_name
@property
def description(self):
return self._description
@property
def author(self):
return self._author
@property
def supplier(self):
return self._supplier
@property
def tags(self):
return list(self._tags)
@property
def logo(self):
return self._load_image(self._logo, 'logo.png', 'logo')
@property
def supplier_logo(self):
return self._load_image(
self._supplier.get('Logo'), 'supplier_logo.png', 'supplier logo')
def get_resource(self, name):
resources_dir = os.path.join(self._source_directory, 'Resources')
if not os.path.exists(resources_dir):
os.makedirs(resources_dir)
return os.path.join(resources_dir, name)
def _load_image(self, file_name, default_name, what_image):
full_path = os.path.join(
self._source_directory, file_name or default_name)
if not os.path.isfile(full_path) and not file_name:
return
try:
if imghdr.what(full_path) != 'png':
raise exceptions.PackageLoadError(
'{0} is not in PNG format'.format(what_image))
with open(full_path) as stream:
return stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
raise exceptions.PackageLoadError(
'Unable to load {0}: {1}'.format(what_image, ex)), None, trace
@staticmethod
def _check_full_name(full_name):
error = exceptions.PackageFormatError('Invalid FullName ' + full_name)
if re.match(r'^[\w\.]+$', full_name):
if full_name.startswith('.') or full_name.endswith('.'):
raise error
if '..' in full_name:
raise error
else:
raise error

9
murano/tests/unit/api/v1/test_catalog.py
View File

@ -227,8 +227,8 @@ class TestCatalogApi(test_base.ControllerTest, test_base.MuranoApiTestCase):
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.raw_ui,
'class_definitions': pkg.classes,
'ui_definition': pkg.ui,
'class_definitions': tuple(pkg.classes),
'archive': pkg.blob,
'categories': [],
}
@ -390,7 +390,10 @@ This is a fake zip archive
--BOUNDARY--'''
with mock.patch('murano.packages.load_utils.load_from_file') as lff:
lff.return_value = package_from_dir
ctxmgr = mock.Mock()
ctxmgr.__enter__ = mock.Mock(return_value=package_from_dir)
ctxmgr.__exit__ = mock.Mock(return_value=False)
lff.return_value = ctxmgr
# Uploading a non-public package
req = self._post(

12
murano/tests/unit/dsl/foundation/test_package_loader.py
View File

@ -15,8 +15,6 @@
import fnmatch
import os.path
import yaml
from murano.dsl import murano_package
from murano.dsl import namespace_resolver
from murano.dsl import package_loader
@ -25,22 +23,26 @@ from murano.tests.unit.dsl.foundation import object_model
class TestPackage(murano_package.MuranoPackage):
def __init__(self, package_loader, name, version,
def __init__(self, pkg_loader, name, version,
runtime_version, requirements, configs):
self.__configs = configs
super(TestPackage, self).__init__(
package_loader, name, version,
pkg_loader, name, version,
runtime_version, requirements)
def get_class_config(self, name):
return self.__configs.get(name, {})
def get_resource(self, name):
pass
class TestPackageLoader(package_loader.MuranoPackageLoader):
_classes_cache = {}
def __init__(self, directory, package_name, parent_loader=None):
self._package_name = package_name
self._yaml_loader = yaql_yaml_loader.get_loader('1.0')
if directory in TestPackageLoader._classes_cache:
self._classes = TestPackageLoader._classes_cache[directory]
else:
@ -80,7 +82,7 @@ class TestPackageLoader(package_loader.MuranoPackageLoader):
def _load_class(self, class_def_file):
with open(class_def_file) as stream:
data = yaml.load(stream, yaql_yaml_loader.YaqlYamlLoader)
data = self._yaml_loader(stream.read(), class_def_file)
if 'Name' not in data:
return

32
murano/tests/unit/test_engine.py
View File

@ -17,6 +17,7 @@
import re
import mock
import semantic_version
import yaql
from yaql.language import exceptions
from yaql.language import utils
@ -30,7 +31,6 @@ ROOT_CLASS = 'io.murano.Object'
class TestNamespaceResolving(base.MuranoTestCase):
def setUp(self):
super(TestNamespaceResolving, self).setUp()
@ -114,7 +114,6 @@ class Bunch(object):
class TestHelperFunctions(base.MuranoTestCase):
def setUp(self):
super(TestHelperFunctions, self).setUp()
@ -132,7 +131,6 @@ class TestHelperFunctions(base.MuranoTestCase):
'atom': ('some', (1, 'atom'), 'hi!'),
'sample': ('atom', (0, 1, 2, 3, 4))
})
print yaql_value(1)
context = yaql.create_context()
evaluated_value = helpers.evaluate(yaql_value, context)
evaluated_complex_value = helpers.evaluate(complex_value, context)
@ -142,50 +140,52 @@ class TestHelperFunctions(base.MuranoTestCase):
class TestYaqlExpression(base.MuranoTestCase):
def setUp(self):
self._version = semantic_version.Version.coerce('1.0')
super(TestYaqlExpression, self).setUp()
def test_expression(self):
yaql_expr = yaql_expression.YaqlExpression('string')
yaql_expr = yaql_expression.YaqlExpression('string', self._version)
self.assertEqual('string', yaql_expr.expression)
def test_unicode_expression(self):
yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'")
yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'",
self._version)
self.assertEqual(u"'yaql ♥ unicode'".encode('utf-8'),
yaql_expr.expression)
def test_unicode_expression_expression(self):
yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'")
yaql_expr2 = yaql_expression.YaqlExpression(yaql_expr)
yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'",
self._version)
yaql_expr2 = yaql_expression.YaqlExpression(yaql_expr, self._version)
self.assertEqual(u"'yaql ♥ unicode'".encode('utf-8'),
yaql_expr2.expression)
def test_evaluate_calls(self):
string = 'string'
expected_calls = [mock.call(string),
expected_calls = [mock.call(string, self._version),
mock.call().evaluate(context=None)]
with mock.patch('murano.dsl.yaql_integration.parse') as mock_parse:
yaql_expr = yaql_expression.YaqlExpression(string)
yaql_expr = yaql_expression.YaqlExpression(string, self._version)
yaql_expr(None)
self.assertEqual(expected_calls, mock_parse.mock_calls)
def test_match_returns(self):
expr = yaql_expression.YaqlExpression('string')
def test_is_expression_returns(self):
expr = yaql_expression.YaqlExpression('string', self._version)
with mock.patch('murano.dsl.yaql_integration.parse'):
self.assertTrue(expr.match('$some'))
self.assertTrue(expr.match('$.someMore'))
self.assertTrue(expr.is_expression('$some', self._version))
self.assertTrue(expr.is_expression('$.someMore', self._version))
with mock.patch('murano.dsl.yaql_integration.parse') as parse_mock:
parse_mock.side_effect = exceptions.YaqlGrammarException
self.assertFalse(expr.match(''))
self.assertFalse(expr.is_expression('', self._version))
with mock.patch('murano.dsl.yaql_integration.parse') as parse_mock:
parse_mock.side_effect = exceptions.YaqlLexicalException
self.assertFalse(expr.match(''))
self.assertFalse(expr.is_expression('', self._version))