diff --git a/contrib/plugins/murano_exampleplugin/murano_exampleplugin/__init__.py b/contrib/plugins/murano_exampleplugin/murano_exampleplugin/__init__.py index 17e582b30..63f06355d 100644 --- a/contrib/plugins/murano_exampleplugin/murano_exampleplugin/__init__.py +++ b/contrib/plugins/murano_exampleplugin/murano_exampleplugin/__init__.py @@ -28,11 +28,10 @@ CONF = config.CONF LOG = logging.getLogger(__name__) -# noinspection PyPep8Naming class GlanceClient(object): - def initialize(self, _context): - client_manager = helpers.get_environment(_context).clients - self.client = client_manager.get_client(_context, "glance", True, + def __init__(self, context): + client_manager = helpers.get_environment(context).clients + self.client = client_manager.get_client(context, "glance", True, self.create_glance_client) def list(self): @@ -44,7 +43,7 @@ class GlanceClient(object): except StopIteration: break - def getByName(self, name): + def get_by_name(self, name): images = list(self.client.images.list(filters={"name": name})) if len(images) > 1: raise AmbiguousNameException(name) @@ -53,7 +52,7 @@ class GlanceClient(object): else: return GlanceClient._format(images[0]) - def getById(self, imageId): + def get_by_id(self, imageId): image = self.client.images.get(imageId) return GlanceClient._format(image) diff --git a/meta/io.murano/Classes/Object.yaml b/meta/io.murano/Classes/Object.yaml index 4b0492f1f..1c8bfcb21 100644 --- a/meta/io.murano/Classes/Object.yaml +++ b/meta/io.murano/Classes/Object.yaml @@ -1,7 +1 @@ -Namespaces: - =: io.murano -Name: Object - -Methods: - initialize: - destroy: +Name: io.murano.Object diff --git a/meta/io.murano/Classes/resources/Instance.yaml b/meta/io.murano/Classes/resources/Instance.yaml index b74ecc2be..6a9078e08 100644 --- a/meta/io.murano/Classes/resources/Instance.yaml +++ b/meta/io.murano/Classes/resources/Instance.yaml @@ -131,8 +131,7 @@ Methods: - $.environment.stack.updateTemplate($.instanceTemplate) - $.environment.stack.push() - $outputs: $.environment.stack.output() - # Changing this to use the .networks attribute instead of 'addresses' - - $.ipAddresses: $outputs.get(format('{0}-assigned-ips', $this.name)).values().flatten() + - $.ipAddresses: $outputs.get(format('{0}-assigned-ips', $this.name)).values().flatten().distinct() - $.openstackId: $outputs.get(format('{0}-id', $this.name)) - If: $._floatingIpOutputName != null Then: diff --git a/murano/common/engine.py b/murano/common/engine.py index c593d01b1..e2b9c1c8e 100755 --- a/murano/common/engine.py +++ b/murano/common/engine.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import traceback import uuid @@ -28,15 +29,14 @@ from murano.common.helpers import token_sanitizer from murano.common import plugin_loader from murano.common import rpc from murano.dsl import dsl_exception -from murano.dsl import executor +from murano.dsl import executor as dsl_executor from murano.dsl import serializer -from murano.engine import client_manager from murano.engine import environment from murano.engine import package_class_loader from murano.engine import package_loader from murano.engine.system import status_reporter import murano.engine.system.system_objects as system_objects -from murano.common.i18n import _LI, _LE +from murano.common.i18n import _LI, _LE, _LW from murano.policy import model_policy_enforcer as enforcer CONF = cfg.CONF @@ -49,30 +49,6 @@ LOG = logging.getLogger(__name__) eventlet.debug.hub_exceptions(False) -class TaskProcessingEndpoint(object): - @staticmethod - def handle_task(context, task): - s_task = token_sanitizer.TokenSanitizer().sanitize(task) - LOG.info(_LI('Starting processing task: {task_desc}').format( - task_desc=jsonutils.dumps(s_task))) - - result = {'model': task['model']} - try: - task_executor = TaskExecutor(task) - result = task_executor.execute() - except Exception as e: - LOG.exception(_LE('Error during task execution for tenant %s'), - task['tenant_id']) - result['action'] = TaskExecutor.exception_result( - e, traceback.format_exc()) - msg_env = Environment(task['id']) - reporter = status_reporter.StatusReporter() - reporter.initialize(msg_env) - reporter.report_error(msg_env, str(e)) - finally: - rpc.api().process_result(result, task['id']) - - def _prepare_rpc_service(server_id): endpoints = [TaskProcessingEndpoint()] @@ -97,9 +73,28 @@ def get_plugin_loader(): return PLUGIN_LOADER -class Environment(object): - def __init__(self, object_id): - self.object_id = object_id +class TaskProcessingEndpoint(object): + @classmethod + def handle_task(cls, context, task): + result = cls.execute(task) + rpc.api().process_result(result, task['id']) + + @staticmethod + def execute(task): + s_task = token_sanitizer.TokenSanitizer().sanitize(task) + LOG.info(_LI('Starting processing task: {task_desc}').format( + task_desc=jsonutils.dumps(s_task))) + + result = None + reporter = status_reporter.StatusReporter(task['id']) + + try: + task_executor = TaskExecutor(task, reporter) + result = task_executor.execute() + return result + finally: + LOG.info(_LI('Finished processing task: {task_desc}').format( + task_desc=jsonutils.dumps(result))) class TaskExecutor(object): @@ -115,118 +110,121 @@ class TaskExecutor(object): def model(self): return self._model - def __init__(self, task): + def __init__(self, task, reporter=None): + if reporter is None: + reporter = status_reporter.StatusReporter(task['id']) self._action = task.get('action') self._model = task['model'] self._environment = environment.Environment() self._environment.token = task['token'] self._environment.tenant_id = task['tenant_id'] self._environment.system_attributes = self._model.get('SystemData', {}) - self._environment.clients = client_manager.ClientManager() + self._reporter = reporter self._model_policy_enforcer = enforcer.ModelPolicyEnforcer( self._environment) def execute(self): - self._create_trust() - try: - murano_client_factory = lambda: \ - self._environment.clients.get_murano_client(self._environment) - with package_loader.CombinedPackageLoader( - murano_client_factory, - self._environment.tenant_id) as pkg_loader: - return self._execute(pkg_loader) - finally: - if self._model['Objects'] is None: + self._create_trust() + except Exception as e: + return self.exception_result(e, None, '') + + murano_client_factory = \ + lambda: self._environment.clients.get_murano_client() + with package_loader.CombinedPackageLoader( + murano_client_factory, + self._environment.tenant_id) as pkg_loader: + result = self._execute(pkg_loader) + self._model['SystemData'] = self._environment.system_attributes + result['model'] = self._model + + if (not self._model.get('Objects') + and not self._model.get('ObjectsCopy')): + try: self._delete_trust() + except Exception: + LOG.warn(_LW('Cannot delete trust'), exc_info=True) + + return result def _execute(self, pkg_loader): class_loader = package_class_loader.PackageClassLoader(pkg_loader) system_objects.register(class_loader, pkg_loader) get_plugin_loader().register_in_loader(class_loader) - exc = executor.MuranoDslExecutor(class_loader, self.environment) - obj = exc.load(self.model) + executor = dsl_executor.MuranoDslExecutor( + class_loader, self.environment) + try: + obj = executor.load(self.model) + except Exception as e: + return self.exception_result(e, None, '') - self._validate_model(obj, self.action, class_loader) - action_result = None - exception = None - exception_traceback = None + try: + self._validate_model(obj, self.action, class_loader) + except Exception as e: + return self.exception_result(e, obj, '') try: LOG.info(_LI('Invoking pre-cleanup hooks')) self.environment.start() - exc.cleanup(self._model) + executor.cleanup(self._model) except Exception as e: - exception = e - exception_traceback = TaskExecutor._log_exception(e, obj, '') + return self.exception_result(e, obj, '') finally: LOG.info(_LI('Invoking post-cleanup hooks')) self.environment.finish() + self._model['ObjectsCopy'] = copy.deepcopy(self._model.get('Objects')) - if exception is None and self.action: + action_result = None + if self.action: try: LOG.info(_LI('Invoking pre-execution hooks')) self.environment.start() - action_result = self._invoke(exc) + try: + action_result = self._invoke(executor) + finally: + try: + self._model = serializer.serialize_model(obj, executor) + except Exception as e: + return self.exception_result(e, None, '') except Exception as e: - exception = e - exception_traceback = TaskExecutor._log_exception( - e, obj, self.action['method']) + return self.exception_result(e, obj, self.action['method']) finally: LOG.info(_LI('Invoking post-execution hooks')) self.environment.finish() - model = serializer.serialize_model(obj, exc) - model['SystemData'] = self._environment.system_attributes - result = { - 'model': model, + try: + action_result = serializer.serialize(action_result) + except Exception as e: + return self.exception_result(e, None, '') + + return { 'action': { - 'result': None, + 'result': action_result, 'isException': False } } - if exception is not None: - result['action'] = TaskExecutor.exception_result( - exception, exception_traceback) - # NOTE(kzaitsev): Exception here means that it happened during - # cleanup. ObjectsCopy and Attributes would be empty if obj - # is empty. This would cause failed env to be deleted. - # Therefore restore these attrs from self._model - for attr in ['ObjectsCopy', 'Attributes']: - if not model.get(attr): - model[attr] = self._model[attr] - else: - result['action']['result'] = serializer.serialize_object( - action_result) - return result - - @staticmethod - def _log_exception(e, root, method_name): - if isinstance(e, dsl_exception.MuranoPlException): - LOG.error('\n' + e.format(prefix=' ')) - exception_traceback = e.format() + def exception_result(self, exception, root, method_name): + if isinstance(exception, dsl_exception.MuranoPlException): + LOG.error('\n' + exception.format(prefix=' ')) + exception_traceback = exception.format() else: exception_traceback = traceback.format_exc() LOG.exception( _LE("Exception %(exc)s occurred" " during invocation of %(method)s"), - {'exc': e, 'method': method_name}) - if root is not None: - reporter = status_reporter.StatusReporter() - reporter.initialize(root) - reporter.report_error(root, str(e)) - return exception_traceback + {'exc': exception, 'method': method_name}) + self._reporter.report_error(root, str(exception)) - @staticmethod - def exception_result(exception, exception_traceback): return { - 'isException': True, - 'result': { - 'message': str(exception), - 'details': exception_traceback + 'action': { + 'isException': True, + 'result': { + 'message': str(exception), + 'details': exception_traceback + } } } @@ -239,10 +237,10 @@ class TaskExecutor(object): def _invoke(self, mpl_executor): obj = mpl_executor.object_store.get(self.action['object_id']) - method_name, args = self.action['method'], self.action['args'] + method_name, kwargs = self.action['method'], self.action['args'] if obj is not None: - return obj.type.invoke(method_name, mpl_executor, obj, args) + return obj.type.invoke(method_name, mpl_executor, obj, (), kwargs) def _create_trust(self): if not CONF.engine.use_trusts: diff --git a/murano/dsl/attribute_store.py b/murano/dsl/attribute_store.py index dd5038bde..1cd8f5183 100644 --- a/murano/dsl/attribute_store.py +++ b/murano/dsl/attribute_store.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -import murano.dsl.murano_object as murano_object +from murano.dsl import dsl_types class AttributeStore(object): @@ -20,7 +20,7 @@ class AttributeStore(object): self._attributes = {} def set(self, tagged_object, owner_type, name, value): - if isinstance(value, murano_object.MuranoObject): + if isinstance(value, dsl_types.MuranoObject): value = value.object_id key = (tagged_object.object_id, owner_type.name, name) diff --git a/murano/dsl/class_loader.py b/murano/dsl/class_loader.py index 024f6836f..05264b206 100644 --- a/murano/dsl/class_loader.py +++ b/murano/dsl/class_loader.py @@ -15,22 +15,20 @@ import inspect import types -import yaql -import yaql.context - -import murano.dsl.exceptions as exceptions -import murano.dsl.helpers as helpers -import murano.dsl.murano_class as murano_class -import murano.dsl.murano_object as murano_object -import murano.dsl.namespace_resolver as namespace_resolver -import murano.dsl.principal_objects as principal_objects -import murano.dsl.typespec as typespec +from murano.dsl import exceptions +from murano.dsl import murano_class +from murano.dsl import murano_object +from murano.dsl import namespace_resolver +from murano.dsl import principal_objects +from murano.dsl import typespec +from murano.dsl import yaql_integration class MuranoClassLoader(object): def __init__(self): self._loaded_types = {} self._packages_cache = {} + self._imported_types = {object, murano_object.MuranoObject} principal_objects.register(self) def _get_package_for_class(self, class_name): @@ -56,7 +54,7 @@ class MuranoClassLoader(object): else: raise - namespaces = data.get('Namespaces', {}) + namespaces = data.get('Namespaces') or {} ns_resolver = namespace_resolver.NamespaceResolver(namespaces) parent_class_names = data.get('Extends') @@ -71,14 +69,21 @@ class MuranoClassLoader(object): type_obj = murano_class.MuranoClass(self, ns_resolver, name, package, parent_classes) - properties = data.get('Properties', {}) + properties = data.get('Properties') or {} for property_name, property_spec in properties.iteritems(): - spec = typespec.PropertySpec(property_spec, type_obj) + spec = typespec.PropertySpec(property_spec) type_obj.add_property(property_name, spec) methods = data.get('Methods') or data.get('Workflow') or {} + + method_mappings = { + 'initialize': '.init', + 'destroy': '.destroy' + } + for method_name, payload in methods.iteritems(): - type_obj.add_method(method_name, payload) + type_obj.add_method( + method_mappings.get(method_name, method_name), payload) self._loaded_types[name] = type_obj return type_obj @@ -93,45 +98,30 @@ class MuranoClassLoader(object): raise NotImplementedError() def create_root_context(self): - return yaql.create_context(True) + return yaql_integration.create_context() def get_class_config(self, name): return {} def create_local_context(self, parent_context, murano_class): - return yaql.context.Context(parent_context=parent_context) - - def _fix_parameters(self, kwargs): - result = {} - for key, value in kwargs.iteritems(): - if key in ('class', 'for', 'from', 'is', 'lambda', 'as', - 'exec', 'assert', 'and', 'or', 'break', 'def', - 'del', 'try', 'while', 'yield', 'raise', 'while', - 'pass', 'return', 'not', 'print', 'in', 'import', - 'global', 'if', 'finally', 'except', 'else', 'elif', - 'continue', 'yield'): - key = '_' + key - result[key] = value - return result + return parent_context.create_child_context() def import_class(self, cls, name=None): - if not name: - if inspect.isclass(cls): - name = cls._murano_class_name - else: - name = cls.__class__._murano_class_name + if cls in self._imported_types: + return + name = name or getattr(cls, '__murano_name', None) or cls.__name__ m_class = self.get_class(name, create_missing=True) - if inspect.isclass(cls): - if issubclass(cls, murano_object.MuranoObject): - m_class.object_class = cls - else: - mpc_name = 'mpc' + helpers.generate_id() - bases = (cls, murano_object.MuranoObject) - m_class.object_class = type(mpc_name, bases, {}) + m_class.extend_with_class(cls) - for item in dir(cls): - method = getattr(cls, item) - if ((inspect.isfunction(method) or inspect.ismethod(method)) and - not item.startswith('_')): - m_class.add_method(item, method) + for method_name in dir(cls): + if method_name.startswith('_'): + continue + method = getattr(cls, method_name) + if not inspect.ismethod(method): + continue + m_class.add_method( + yaql_integration.CONVENTION.convert_function_name( + method_name), + method) + self._imported_types.add(cls) diff --git a/murano/dsl/constants.py b/murano/dsl/constants.py new file mode 100644 index 000000000..04180944c --- /dev/null +++ b/murano/dsl/constants.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +EXPRESSION_MEMORY_QUOTA = 512 * 1024 +ITERATORS_LIMIT = 200 + +CTX_ACTIONS_ONLY = '?actionsOnly' +CTX_ALLOW_PROPERTY_WRITES = '$?allowPropertyWrites' +CTX_ARGUMENT_OWNER = '$?argumentOwner' +CTX_ATTRIBUTE_STORE = '$?attributeStore' +CTX_CALLER_CONTEXT = '$?callerContext' +CTX_CLASS_LOADER = '$?classLoader' +CTX_CURRENT_INSTRUCTION = '$?currentInstruction' +CTX_CURRENT_EXCEPTION = '$?currentException' +CTX_CURRENT_METHOD = '$?currentMethod' +CTX_ENVIRONMENT = '$?environment' +CTX_EXECUTOR = '$?executor' +CTX_OBJECT_STORE = '$?objectStore' +CTX_SKIP_FRAME = '$?skipFrame' +CTX_THIS = '$?this' +CTX_TYPE = '$?type' + +DM_OBJECTS = 'Objects' +DM_OBJECTS_COPY = 'ObjectsCopy' +DM_ATTRIBUTES = 'Attributes' + +META_NO_TRACE = '?noTrace' diff --git a/murano/dsl/dsl.py b/murano/dsl/dsl.py new file mode 100644 index 000000000..d19a5dc80 --- /dev/null +++ b/murano/dsl/dsl.py @@ -0,0 +1,309 @@ +# Copyright (c) 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import inspect +import os.path +import types + +from yaql.language import expressions as yaql_expressions +from yaql.language import utils +from yaql.language import yaqltypes + +from murano.dsl import constants +from murano.dsl import dsl_types +from murano.dsl import helpers + + +NO_VALUE = utils.create_marker('NO_VALUE') + + +def name(dsl_name): + def wrapper(cls): + cls.__murano_name = dsl_name + return cls + return wrapper + + +class MuranoObjectType(yaqltypes.PythonType): + def __init__(self, murano_class, nullable=False): + self.murano_class = murano_class + super(MuranoObjectType, self).__init__( + (dsl_types.MuranoObject, MuranoObjectInterface), nullable) + + def check(self, value, context, *args, **kwargs): + if not super(MuranoObjectType, self).check( + value, context, *args, **kwargs): + return False + if isinstance(value, MuranoObjectInterface): + value = value.object + if value is None or isinstance(value, yaql_expressions.Expression): + return True + murano_class = self.murano_class + if isinstance(murano_class, types.StringTypes): + class_loader = helpers.get_class_loader(context) + murano_class = class_loader.get_class(self.murano_class) + return murano_class.is_compatible(value) + + def convert(self, value, sender, context, function_spec, engine, + *args, **kwargs): + result = super(MuranoObjectType, self).convert( + value, sender, context, function_spec, engine, *args, **kwargs) + if isinstance(result, dsl_types.MuranoObject): + return MuranoObjectInterface(result, engine) + return result + + +class ThisParameterType(yaqltypes.HiddenParameterType, yaqltypes.SmartType): + def __init__(self): + super(ThisParameterType, self).__init__(False) + + def convert(self, value, sender, context, function_spec, engine, + *args, **kwargs): + this = helpers.get_this(context) + executor = helpers.get_executor(context) + return MuranoObjectInterface(this, engine, executor) + + +class InterfacesParameterType(yaqltypes.HiddenParameterType, + yaqltypes.SmartType): + def __init__(self): + super(InterfacesParameterType, self).__init__(False) + + def convert(self, value, sender, context, function_spec, engine, + *args, **kwargs): + this = helpers.get_this(context) + return Interfaces(engine, this) + + +class MuranoTypeName(yaqltypes.LazyParameterType, yaqltypes.PythonType): + def __init__(self, nullable=False, context=None): + self._context = context + super(MuranoTypeName, self).__init__( + (dsl_types.MuranoClassReference,) + types.StringTypes, nullable) + + def convert(self, value, sender, context, function_spec, engine, + *args, **kwargs): + context = self._context or context + if isinstance(value, yaql_expressions.Expression): + value = value(utils.NO_VALUE, context, engine) + value = super(MuranoTypeName, self).convert( + value, sender, context, function_spec, engine) + if isinstance(value, types.StringTypes): + class_loader = helpers.get_class_loader(context) + murano_type = helpers.get_type(context) + value = dsl_types.MuranoClassReference( + class_loader.get_class( + murano_type.namespace_resolver.resolve_name(value))) + return value + + +class MuranoObjectInterface(dsl_types.MuranoObjectInterface): + class DataInterface(object): + def __init__(self, object_interface): + object.__setattr__(self, '__object_interface', object_interface) + + def __getattr__(self, item): + oi = getattr(self, '__object_interface') + return oi[item] + + def __setattr__(self, key, value): + oi = getattr(self, '__object_interface') + oi[key] = value + + class CallInterface(object): + def __init__(self, mpl_object, executor): + self.__object = mpl_object + self.__executor = executor + + def __getattr__(self, item): + executor = self.__executor or helpers.get_executor() + + def func(*args, **kwargs): + self._insert_instruction() + return self.__object.type.invoke( + item, executor, self.__object, args, kwargs, + helpers.get_context()) + return func + + @staticmethod + def _insert_instruction(): + context = helpers.get_context() + if context: + frame = inspect.stack()[2] + location = dsl_types.ExpressionFilePosition( + os.path.abspath(frame[1]), frame[2], + -1, frame[2], -1) + context[constants.CTX_CURRENT_INSTRUCTION] = NativeInstruction( + frame[4][0].strip(), location) + + def __init__(self, mpl_object, engine, executor=None): + self.__object = mpl_object + self.__executor = executor + self.__engine = engine + + @property + def object(self): + return self.__object + + @property + def id(self): + return self.__object.object_id + + @property + def owner(self): + return self.__object.owner + + @property + def type(self): + return self.__object.type + + def data(self): + return MuranoObjectInterface.DataInterface(self) + + @property + def extension(self): + return self.__object.extension + + def cast(self, murano_class): + return MuranoObjectInterface( + self.__object.cast(murano_class), self.__engine, self.__executor) + + def __getitem__(self, item): + context = helpers.get_context() + return to_mutable( + self.__object.get_property(item, context), self.__engine) + + def __setitem__(self, key, value): + context = helpers.get_context() + value = helpers.evaluate(value, context) + self.__object.set_property(key, value, context) + + def __call__(self): + return MuranoObjectInterface.CallInterface( + self.object, self.__executor) + + def __repr__(self): + return '<{0}>'.format(repr(self.object)) + + +class YaqlInterface(object): + def __init__(self, engine, sender=utils.NO_VALUE): + self.__engine = engine + self.__sender = sender + + @property + def context(self): + return self.__context + + @property + def engine(self): + return self.__engine + + @property + def sender(self): + return self.__sender + + def on(self, sender): + return YaqlInterface(self.engine, sender) + + def __getattr__(self, item): + def stub(*args, **kwargs): + context = helpers.get_context() + args = tuple(helpers.evaluate(arg, context) for arg in args) + kwargs = dict((key, helpers.evaluate(value, context)) + for key, value in kwargs.iteritems()) + return to_mutable( + context(item, self.engine, self.sender)(*args, **kwargs), + self.engine) + return stub + + def __call__(self, __expression, *args, **kwargs): + context = helpers.get_context().create_child_context() + for i, param in enumerate(args): + context['$' + str(i + 1)] = helpers.evaluate(param, context) + for arg_name, arg_value in kwargs.iteritems(): + context['$' + arg_name] = helpers.evaluate(arg_value, context) + parsed = self.engine(__expression) + res = parsed.evaluate(context=context) + return to_mutable(res, self.engine) + + def __getitem__(self, item): + return helpers.get_context()[item] + + def __setitem__(self, key, value): + helpers.get_context()[key] = value + + +class Interfaces(object): + def __init__(self, engine, mpl_object): + self.__engine = engine + self.__object = mpl_object + + def yaql(self, sender=utils.NO_VALUE): + return YaqlInterface(self.__engine, sender) + + def this(self): + return self.methods(self.__object) + + def methods(self, mpl_object): + if mpl_object is None: + return None + return MuranoObjectInterface(mpl_object, self.__engine) + + @property + def environment(self): + return helpers.get_environment() + + @property + def caller(self): + caller_context = helpers.get_caller_context() + if caller_context is None: + return None + caller = helpers.get_this(caller_context) + if caller is None: + return None + return MuranoObjectInterface(caller, self.__engine) + + @property + def attributes(self): + executor = helpers.get_executor() + return executor.attribute_store + + @property + def class_config(self): + return self.class_loader.get_class_config(self.__object.type.name) + + @property + def class_loader(self): + return helpers.get_class_loader() + + +class NativeInstruction(object): + def __init__(self, instruction, location): + self.instruction = instruction + self.source_file_position = location + + def __str__(self): + return self.instruction + + +def to_mutable(obj, yaql_engine): + def converter(value, limit_func, engine, rec): + if isinstance(value, dsl_types.MuranoObject): + return MuranoObjectInterface(value, engine) + else: + return utils.convert_output_data(value, limit_func, engine, rec) + + limiter = lambda it: utils.limit_iterable(it, constants.ITERATORS_LIMIT) + return converter(obj, limiter, yaql_engine, converter) diff --git a/murano/dsl/dsl_exception.py b/murano/dsl/dsl_exception.py index 256875c3e..7f3d01962 100644 --- a/murano/dsl/dsl_exception.py +++ b/murano/dsl/dsl_exception.py @@ -14,7 +14,7 @@ import sys -import murano.dsl.yaql_functions as yaql_functions +from murano.dsl.principal_objects import stack_trace class MuranoPlException(Exception): @@ -51,7 +51,7 @@ class MuranoPlException(Exception): @staticmethod def from_python_exception(exception, context): - stacktrace = yaql_functions.new('io.murano.StackTrace', context) + stacktrace = stack_trace.create_stack_trace(context) exception_type = type(exception) names = ['{0}.{1}'.format(exception_type.__module__, exception_type.__name__)] @@ -72,10 +72,11 @@ class MuranoPlException(Exception): return self._names def format(self, prefix=''): - text = '{3}{0}: {1}\n' \ - '{3}Traceback (most recent call last):\n' \ - '{2}'.format(self._format_name(), self.message, - self.stacktrace.toString(prefix + ' '), prefix) + text = ('{3}{0}: {1}\n' + '{3}Traceback (most recent call last):\n' + '{2}').format( + self._format_name(), self.message, + self.stacktrace().toString(prefix + ' '), prefix) if self._cause is not None: text += '\n\n{0} Caused by {1}'.format( prefix, self._cause.format(prefix + ' ').lstrip()) diff --git a/murano/dsl/dsl_types.py b/murano/dsl/dsl_types.py new file mode 100644 index 000000000..408094901 --- /dev/null +++ b/murano/dsl/dsl_types.py @@ -0,0 +1,79 @@ +# Copyright (c) 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class MuranoClass(object): + pass + + +class MuranoObject(object): + pass + + +class MuranoMethod(object): + pass + + +class MuranoPackage(object): + pass + + +class MuranoClassReference(object): + def __init__(self, murano_class): + self.__murano_class = murano_class + + @property + def murano_class(self): + return self.__murano_class + + def __str__(self): + return self.__murano_class.name + + +class YaqlExpression(object): + pass + + +class MuranoObjectInterface(object): + pass + + +class ExpressionFilePosition(object): + def __init__(self, file_path, start_line, start_column, + end_line, end_column): + self._file_path = file_path + self._start_line = start_line + self._start_column = start_column + self._end_line = end_line + self._end_column = end_column + + @property + def file_path(self): + return self._file_path + + @property + def start_line(self): + return self._start_line + + @property + def start_column(self): + return self._start_column + + @property + def end_line(self): + return self._end_line + + @property + def end_column(self): + return self._end_column diff --git a/murano/dsl/exceptions.py b/murano/dsl/exceptions.py index 3da7b3698..aac4d6858 100644 --- a/murano/dsl/exceptions.py +++ b/murano/dsl/exceptions.py @@ -129,3 +129,7 @@ class UninitializedPropertyAccessError(PropertyAccessError): super(PropertyAccessError, self).__init__( 'Access to uninitialized property ' '"%s" in class "%s" is forbidden' % (name, murano_class.name)) + + +class CircularExpressionDependenciesError(Exception): + pass diff --git a/murano/dsl/executor.py b/murano/dsl/executor.py index 2078c01f1..143b6cf30 100644 --- a/murano/dsl/executor.py +++ b/murano/dsl/executor.py @@ -13,26 +13,26 @@ # under the License. import collections -import inspect -import sys +import contextlib +import itertools import types -import uuid +import weakref import eventlet import eventlet.event from oslo_log import log as logging -import yaql.context +from yaql.language import specs from murano.common.i18n import _LW -import murano.dsl.attribute_store as attribute_store -import murano.dsl.dsl_exception as dsl_exception -import murano.dsl.expressions as expressions -import murano.dsl.helpers as helpers -import murano.dsl.murano_method as murano_method -import murano.dsl.murano_object as murano_object -import murano.dsl.object_store as object_store -import murano.dsl.principal_objects.stack_trace as trace -import murano.dsl.yaql_functions as yaql_functions +from murano.dsl import attribute_store +from murano.dsl import constants +from murano.dsl import dsl +from murano.dsl import helpers +from murano.dsl import murano_method +from murano.dsl import object_store +from murano.dsl.principal_objects import stack_trace +from murano.dsl import yaql_functions +from murano.dsl import yaql_integration LOG = logging.getLogger(__name__) @@ -40,17 +40,18 @@ LOG = logging.getLogger(__name__) class MuranoDslExecutor(object): def __init__(self, class_loader, environment=None): self._class_loader = class_loader - self._object_store = object_store.ObjectStore(class_loader) self._attribute_store = attribute_store.AttributeStore() - self._root_context = class_loader.create_root_context() - self._root_context.set_data(self, '?executor') - self._root_context.set_data(self._class_loader, '?classLoader') - self._root_context.set_data(environment, '?environment') - self._root_context.set_data(self._object_store, '?objectStore') - self._root_context.set_data(self._attribute_store, '?attributeStore') + self._root_context = \ + class_loader.create_root_context().create_child_context() + self._root_context[constants.CTX_EXECUTOR] = weakref.proxy(self) + self._root_context[ + constants.CTX_CLASS_LOADER] = weakref.proxy(self._class_loader) + self._root_context[constants.CTX_ENVIRONMENT] = environment + self._root_context[constants.CTX_ATTRIBUTE_STORE] = weakref.proxy( + self._attribute_store) + self._object_store = object_store.ObjectStore(self._root_context) self._locks = {} yaql_functions.register(self._root_context) - self._root_context = yaql.context.Context(self._root_context) @property def object_store(self): @@ -60,234 +61,173 @@ class MuranoDslExecutor(object): def attribute_store(self): return self._attribute_store - def to_yaql_args(self, args): - if not args: - return tuple() - elif isinstance(args, types.TupleType): - return args - elif isinstance(args, types.ListType): - return tuple(args) - elif isinstance(args, types.DictionaryType): - return tuple(args.items()) - else: - raise ValueError() + @property + def class_loader(self): + return self._class_loader - def invoke_method(self, name, this, context, murano_class, *args): - external_call = False - if context is None: - external_call = True - context = self._root_context - method = this.type.find_single_method(name) + def invoke_method(self, method, this, context, args, kwargs, + skip_stub=False): + if isinstance(this, dsl.MuranoObjectInterface): + this = this.object + kwargs = yaql_integration.filter_parameters_dict(kwargs) + if context is None or not skip_stub: + actions_only = context is None and not method.name.startswith('.') + method_context = self._create_method_context( + this, method, context, actions_only, skip_frame=True) + return method.yaql_function_definition( + yaql_integration.ENGINE, method_context, this.real_this)( + *args, **kwargs) - is_special_method = name in ('initialize', 'destroy') - - if external_call and not is_special_method and \ - method.usage != murano_method.MethodUsages.Action: + if (context[constants.CTX_ACTIONS_ONLY] and method.usage != + murano_method.MethodUsages.Action): raise Exception('{0} is not an action'.format(method.name)) - # TODO(slagun): check method accessibility from murano_class - if not external_call and is_special_method: - LOG.warning(_LW('initialize/destroy methods are called ' - 'automatically by engine. This call is no-op ' - 'and will become exception in the future')) - return None - - # restore this from upcast object (no change if there was no upcast) + context = self._create_method_context(this, method, context) this = this.real_this - arguments_scheme = method.arguments_scheme - params = self._evaluate_parameters( - arguments_scheme, context, this, *args) - return self._invoke_method_implementation( - method, this, context, params) - def _invoke_method_implementation(self, method, this, context, params): - result = None - body = method.body - if not body: - return None + if method.arguments_scheme is not None: + args, kwargs = self._canonize_parameters( + method.arguments_scheme, args, kwargs) - murano_class = method.murano_class - current_thread = eventlet.greenthread.getcurrent() - if not hasattr(current_thread, '_muranopl_thread_marker'): - thread_marker = current_thread._muranopl_thread_marker = \ - uuid.uuid4().hex - else: - thread_marker = current_thread._muranopl_thread_marker + with self._acquire_method_lock(method, this): + for i, arg in enumerate(args, 2): + context[str(i)] = arg + for key, value in kwargs.iteritems(): + context[key] = value - method_id = id(body) - this_id = this.object_id - - while True: - event, marker = self._locks.get((method_id, this_id), (None, None)) - if event: - if marker == thread_marker: - return self._invoke_method_implementation_gt( - body, this, params, murano_class, context) - event.wait() - else: - break - - event = eventlet.event.Event() - self._locks[(method_id, this_id)] = (event, thread_marker) - # noinspection PyProtectedMember - method_info = '{0}.{1} ({2})'.format(murano_class.name, method._name, - hash((method_id, this_id))) - # Prepare caller information - caller_ctx = helpers.get_caller_context(context) - if caller_ctx: - caller_info = trace.compose_stack_frame(caller_ctx) - LOG.debug( - '{0}: Begin execution: {1} called from {2}'.format( - thread_marker, method_info, trace.format_frame( - caller_info))) - else: - LOG.debug( - '{0}: Begin execution: {1}'.format( - thread_marker, method_info)) - - try: - gt = eventlet.spawn(self._invoke_method_implementation_gt, body, - this, params, murano_class, context, - thread_marker) - result = gt.wait() - except Exception as e: - LOG.debug( - "{0}: End execution: {1} with exception {2}".format( - thread_marker, method_info, e)) - if method._name != 'destroy': - raise - else: - LOG.debug( - "{0}: End execution: {1}".format(thread_marker, method_info)) - finally: - del self._locks[(method_id, this_id)] - event.send() - - return result - - def _invoke_method_implementation_gt(self, body, this, - params, murano_class, context, - thread_marker=None): - if thread_marker: - current_thread = eventlet.greenthread.getcurrent() - current_thread._muranopl_thread_marker = thread_marker - if callable(body): - if '_context' in inspect.getargspec(body).args: - params['_context'] = self._create_context( - this, murano_class, context, **params) - try: - if inspect.ismethod(body) and not body.__self__: - return body(this, **params) + def call(): + if isinstance(method.body, specs.FunctionDefinition): + native_this = this.cast( + method.murano_class).extension + return method.body( + yaql_integration.ENGINE, context, native_this)( + *args, **kwargs) else: - return body(**params) - except Exception as e: - raise dsl_exception.MuranoPlException.from_python_exception( - e, context), None, sys.exc_info()[2] - elif isinstance(body, expressions.DslExpression): - return self.execute( - body, murano_class, this, context, **params) + return (None if method.body is None + else method.body.execute(context)) - else: - raise ValueError() - - def _evaluate_parameters(self, arguments_scheme, context, this, *args): - arg_names = list(arguments_scheme.keys()) - parameter_values = {} - i = 0 - for arg in args: - value = helpers.evaluate(arg, context) - if isinstance(value, types.TupleType) and len(value) == 2 and \ - isinstance(value[0], types.StringTypes): - name = value[0] - value = value[1] - if name not in arguments_scheme: - raise TypeError() + if (not isinstance(method.body, specs.FunctionDefinition) + or not method.body.meta.get(constants.META_NO_TRACE)): + with self._log_method(context, args, kwargs) as log: + result = call() + log(result) + return result else: - if i >= len(arg_names): - raise TypeError() - name = arg_names[i] - i += 1 + return call() - if callable(value): - value = value() - arg_spec = arguments_scheme[name] - parameter_values[name] = arg_spec.validate( - value, this, None, self._root_context, self._object_store) + @contextlib.contextmanager + def _acquire_method_lock(self, func, this): + method_id = id(func) + this_id = this.object_id + thread_id = helpers.get_current_thread_id() + while True: + event, event_owner = self._locks.get( + (method_id, this_id), (None, None)) + if event: + if event_owner == thread_id: + event = None + break + else: + event.wait() + else: + event = eventlet.event.Event() + self._locks[(method_id, this_id)] = (event, thread_id) + break + try: + yield + finally: + if event is not None: + del self._locks[(method_id, this_id)] + event.send() - for name, arg_spec in arguments_scheme.iteritems(): - if name not in parameter_values: - if not arg_spec.has_default: - raise TypeError() - parameter_context = self._create_context( - this, this.type, context) - parameter_values[name] = arg_spec.validate( - helpers.evaluate(arg_spec.default, parameter_context), - this, None, self._root_context, self._object_store) + @contextlib.contextmanager + def _log_method(self, context, args, kwargs): + method = helpers.get_current_method(context) + param_gen = itertools.chain( + (str(arg) for arg in args), + ('{0} => {1}'.format(name, value) + for name, value in kwargs.iteritems())) + params_str = ', '.join(param_gen) + method_name = '{0}::{1}'.format(method.murano_class.name, method.name) + thread_id = helpers.get_current_thread_id() + caller_str = '' + caller_ctx = helpers.get_caller_context(context) + if caller_ctx is not None: + frame = stack_trace.compose_stack_frame(caller_ctx) + if frame['location']: + caller_str = ' called from ' + stack_trace.format_frame(frame) - return parameter_values + LOG.trace('{0}: Begin execution {1}({2}){3}'.format( + thread_id, method_name, params_str, caller_str)) + try: + def log_result(result): + LOG.trace('{0}: End execution {1} with result {2}'.format( + thread_id, method_name, result)) + yield log_result + except Exception as e: + LOG.trace('{0}: End execution {1} with exception {2}'.format( + thread_id, method_name, e)) + raise - def _create_context(self, this, murano_class, context, **kwargs): + @staticmethod + def _canonize_parameters(arguments_scheme, args, kwargs): + arg_names = arguments_scheme.keys() + parameter_values = yaql_integration.filter_parameters_dict(kwargs) + for i, arg in enumerate(args): + name = arg_names[i] + parameter_values[name] = arg + return tuple(), parameter_values + + def _create_method_context(self, this, method, context=None, + actions_only=False, skip_frame=False): new_context = self._class_loader.create_local_context( - parent_context=self._root_context, - murano_class=murano_class) - new_context.set_data(this) - new_context.set_data(this, 'this') - new_context.set_data(this, '?this') - new_context.set_data(murano_class, '?type') - new_context.set_data(context, '?callerContext') + parent_context=this.context, + murano_class=this.type) + caller = context + while caller is not None and caller[constants.CTX_SKIP_FRAME]: + caller = caller[constants.CTX_CALLER_CONTEXT] + new_context[constants.CTX_CALLER_CONTEXT] = caller + new_context[constants.CTX_CURRENT_METHOD] = method + new_context[constants.CTX_ACTIONS_ONLY] = actions_only + new_context[constants.CTX_SKIP_FRAME] = skip_frame - @yaql.context.EvalArg('obj', arg_type=murano_object.MuranoObject) - @yaql.context.EvalArg('property_name', arg_type=str) - def obj_attribution(obj, property_name): - return obj.get_property(property_name, murano_class) - - @yaql.context.EvalArg('prefix', str) - @yaql.context.EvalArg('name', str) - def validate(prefix, name): - return murano_class.namespace_resolver.resolve_name( - '%s:%s' % (prefix, name)) - - new_context.register_function(obj_attribution, '#operator_.') - new_context.register_function(validate, '#validate') - for key, value in kwargs.iteritems(): - new_context.set_data(value, key) + if context is not None: + new_context[constants.CTX_ALLOW_PROPERTY_WRITES] = context[ + constants.CTX_ALLOW_PROPERTY_WRITES] return new_context - def execute(self, expression, murano_class, this, context, **kwargs): - new_context = self._create_context( - this, murano_class, context, **kwargs) - return expression.execute(new_context, murano_class) - def load(self, data): if not isinstance(data, types.DictionaryType): raise TypeError() - self._attribute_store.load(data.get('Attributes') or []) - result = self._object_store.load(data.get('Objects'), - None, self._root_context) - return result + self._attribute_store.load(data.get(constants.DM_ATTRIBUTES) or []) + result = self._object_store.load(data.get(constants.DM_OBJECTS), None) + if result is None: + return None + return dsl.MuranoObjectInterface( + result, yaql_integration.ENGINE, executor=self) def cleanup(self, data): - objects_copy = data.get('ObjectsCopy') + objects_copy = data.get(constants.DM_OBJECTS_COPY) if not objects_copy: return - gc_object_store = object_store.ObjectStore(self._class_loader) - gc_object_store.load(objects_copy, None, self._root_context) + gc_object_store = object_store.ObjectStore(self._root_context) + gc_object_store.load(objects_copy, None) objects_to_clean = [] for object_id in self._list_potential_object_ids(objects_copy): - if gc_object_store.has(object_id) \ - and not self._object_store.has(object_id): + if (gc_object_store.has(object_id) + and not self._object_store.has(object_id)): obj = gc_object_store.get(object_id) objects_to_clean.append(obj) if objects_to_clean: - backup = self._object_store - try: - self._object_store = gc_object_store - for obj in objects_to_clean: - methods = obj.type.find_all_methods('destroy') - for method in methods: - method.invoke(self, obj, {}) - finally: - self._object_store = backup + for obj in objects_to_clean: + methods = obj.type.find_all_methods('.destroy') + for method in methods: + try: + method.invoke(self, obj, (), {}, None) + except Exception as e: + LOG.warn(_LW( + 'Muted exception during execution of .destroy ' + 'on {1}: {2}').format(obj, e), exc_info=True) def _list_potential_object_ids(self, data): if isinstance(data, types.DictionaryType): @@ -295,9 +235,9 @@ class MuranoDslExecutor(object): for res in self._list_potential_object_ids(val): yield res sys_dict = data.get('?') - if isinstance(sys_dict, types.DictionaryType) \ - and sys_dict.get('id') \ - and sys_dict.get('type'): + if (isinstance(sys_dict, types.DictionaryType) + and sys_dict.get('id') + and sys_dict.get('type')): yield sys_dict['id'] elif isinstance(data, collections.Iterable) and not isinstance( data, types.StringTypes): diff --git a/murano/dsl/expressions.py b/murano/dsl/expressions.py index 85fc738b0..57c4b2570 100644 --- a/murano/dsl/expressions.py +++ b/murano/dsl/expressions.py @@ -14,10 +14,10 @@ import types -import murano.dsl.dsl_exception as dsl_exception -import murano.dsl.helpers as helpers -import murano.dsl.lhs_expression as lhs_expression -import murano.dsl.yaql_expression as yaql_expression +from murano.dsl import dsl_exception +from murano.dsl import helpers +from murano.dsl import lhs_expression +from murano.dsl import yaql_expression _macros = [] @@ -36,7 +36,7 @@ def register_macro(cls): class DslExpression(object): - def execute(self, context, murano_class): + def execute(self, context): pass @@ -64,11 +64,11 @@ class Statement(DslExpression): def expression(self): return self._expression - def execute(self, context, murano_class): + def execute(self, context): try: result = helpers.evaluate(self.expression, context) if self.destination: - self.destination(result, context, murano_class) + self.destination(result, context) return result except dsl_exception.MuranoPlException: raise diff --git a/murano/dsl/helpers.py b/murano/dsl/helpers.py index 4917fef48..4dc2843f2 100644 --- a/murano/dsl/helpers.py +++ b/murano/dsl/helpers.py @@ -1,4 +1,4 @@ -# Copyright (c) 2014 Mirantis, Inc. +# Copyright (c) 2014 #Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -12,96 +12,52 @@ # License for the specific language governing permissions and limitations # under the License. -import collections +import contextlib import re import sys import types import uuid import eventlet.greenpool -import yaql.expressions +import eventlet.greenthread +import yaql.language.exceptions +import yaql.language.expressions +from yaql.language import utils as yaqlutils + from murano.common import utils -import murano.dsl.murano_object -import murano.dsl.yaql_expression as yaql_expression +from murano.dsl import constants +from murano.dsl import dsl_types + +KEYWORD_REGEX = re.compile(r'(?!__)\b[^\W\d]\w*\b') + +_threads_sequencer = 0 -def serialize(value, memo=None): - if memo is None: - memo = set() - if isinstance(value, types.DictionaryType): - result = {} - for d_key, d_value in value.iteritems(): - result[d_key] = serialize(d_value, memo) - return result - elif isinstance(value, murano.dsl.murano_object.MuranoObject): - if value.object_id not in memo: - memo.add(value.object_id) - return serialize(value.to_dictionary(), memo) - else: - return value.object_id - elif isinstance(value, types.ListType): - return [serialize(t, memo) for t in value] +def evaluate(value, context): + if isinstance(value, (dsl_types.YaqlExpression, + yaql.language.expressions.Statement)): + return value(context) + elif isinstance(value, yaqlutils.MappingType): + return yaqlutils.FrozenDict( + (evaluate(d_key, context), + evaluate(d_value, context)) + for d_key, d_value in value.iteritems()) + elif yaqlutils.is_sequence(value): + return tuple(evaluate(t, context) for t in value) + elif isinstance(value, yaqlutils.SetType): + return frozenset(evaluate(t, context) for t in value) + elif yaqlutils.is_iterable(value): + return tuple( + evaluate(t, context) + for t in yaqlutils.limit_iterable( + value, constants.ITERATORS_LIMIT)) + elif isinstance(value, dsl_types.MuranoObjectInterface): + return value.object else: return value -def execute_instruction(instruction, action, context): - old_instruction = context.get_data('$?currentInstruction') - context.set_data(instruction, '?currentInstruction') - result = action() - context.set_data(old_instruction, '?currentInstruction') - return result - - -def evaluate(value, context, max_depth=sys.maxint): - if isinstance(value, yaql.expressions.Expression): - value = yaql_expression.YaqlExpression(value) - - if isinstance(value, yaql_expression.YaqlExpression): - func = lambda: evaluate(value.evaluate(context), context, 1) - if max_depth <= 0: - return func - else: - return execute_instruction(value, func, context) - - elif isinstance(value, types.DictionaryType): - result = {} - for d_key, d_value in value.iteritems(): - result[evaluate(d_key, context, max_depth - 1)] = \ - evaluate(d_value, context, max_depth - 1) - return result - elif isinstance(value, types.ListType): - return [evaluate(t, context, max_depth - 1) for t in value] - elif isinstance(value, types.TupleType): - return tuple(evaluate(list(value), context, max_depth - 1)) - elif callable(value): - return value() - elif isinstance(value, types.StringTypes): - return value - elif isinstance(value, collections.Iterable): - return list(value) - else: - return value - - -def needs_evaluation(value): - if isinstance(value, (yaql_expression.YaqlExpression, - yaql.expressions.Expression)): - return True - elif isinstance(value, types.DictionaryType): - for d_key, d_value in value.iteritems(): - if needs_evaluation(d_value) or needs_evaluation(d_key): - return True - elif isinstance(value, types.StringTypes): - return False - elif isinstance(value, collections.Iterable): - for t in value: - if needs_evaluation(t): - return True - return False - - def merge_lists(list1, list2): result = [] for item in list1 + list2: @@ -144,16 +100,17 @@ def generate_id(): return uuid.uuid4().hex -def parallel_select(collection, func): +def parallel_select(collection, func, limit=1000): # workaround for eventlet issue 232 # https://github.com/eventlet/eventlet/issues/232 def wrapper(element): try: - return func(element), False, None + with contextual(get_context()): + return func(element), False, None except Exception as e: return e, True, sys.exc_info()[2] - gpool = eventlet.greenpool.GreenPool() + gpool = eventlet.greenpool.GreenPool(limit) result = list(gpool.imap(wrapper, collection)) try: exception = next(t for t in result if t[1]) @@ -163,54 +120,101 @@ def parallel_select(collection, func): raise exception[0], None, exception[2] -def to_python_codestyle(name): - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) - return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() - - def enum(**enums): return type('Enum', (), enums) -def get_executor(context): - return context.get_data('$?executor') +def get_context(): + current_thread = eventlet.greenthread.getcurrent() + return getattr(current_thread, '__murano_context', None) -def get_class_loader(context): - return context.get_data('$?classLoader') +def get_executor(context=None): + context = context or get_context() + return context[constants.CTX_EXECUTOR] -def get_type(context): - return context.get_data('$?type') +def get_class_loader(context=None): + context = context or get_context() + return context[constants.CTX_CLASS_LOADER] -def get_environment(context): - return context.get_data('$?environment') +def get_type(context=None): + context = context or get_context() + return context[constants.CTX_TYPE] -def get_object_store(context): - return context.get_data('$?objectStore') +def get_environment(context=None): + context = context or get_context() + return context[constants.CTX_ENVIRONMENT] -def get_this(context): - return context.get_data('$?this') +def get_object_store(context=None): + context = context or get_context() + return context[constants.CTX_OBJECT_STORE] -def get_caller_context(context): - return context.get_data('$?callerContext') +def get_this(context=None): + context = context or get_context() + return context[constants.CTX_THIS] -def get_attribute_store(context): - return context.get_data('$?attributeStore') +def get_caller_context(context=None): + context = context or get_context() + return context[constants.CTX_CALLER_CONTEXT] -def get_current_instruction(context): - return context.get_data('$?currentInstruction') +def get_attribute_store(context=None): + context = context or get_context() + return context[constants.CTX_ATTRIBUTE_STORE] -def get_current_method(context): - return context.get_data('$?currentMethod') +def get_current_instruction(context=None): + context = context or get_context() + return context[constants.CTX_CURRENT_INSTRUCTION] -def get_current_exception(context): - return context.get_data('$?currentException') +def get_current_method(context=None): + context = context or get_context() + return context[constants.CTX_CURRENT_METHOD] + + +def get_current_exception(context=None): + context = context or get_context() + return context[constants.CTX_CURRENT_EXCEPTION] + + +def are_property_modifications_allowed(context=None): + context = context or get_context() + return context[constants.CTX_ALLOW_PROPERTY_WRITES] or False + + +def is_keyword(text): + return KEYWORD_REGEX.match(text) is not None + + +def get_current_thread_id(): + global _threads_sequencer + + current_thread = eventlet.greenthread.getcurrent() + thread_id = getattr(current_thread, '__thread_id', None) + if thread_id is None: + thread_id = 'T' + str(_threads_sequencer) + _threads_sequencer += 1 + setattr(current_thread, '__thread_id', thread_id) + return thread_id + + +@contextlib.contextmanager +def contextual(ctx): + current_thread = eventlet.greenthread.getcurrent() + current_context = getattr(current_thread, '__murano_context', None) + if ctx: + setattr(current_thread, '__murano_context', ctx) + try: + yield + finally: + if current_context: + setattr(current_thread, '__murano_context', current_context) + elif hasattr(current_thread, '__murano_context'): + delattr(current_thread, '__murano_context') diff --git a/murano/dsl/lhs_expression.py b/murano/dsl/lhs_expression.py index c624abda1..626f4c1d8 100644 --- a/murano/dsl/lhs_expression.py +++ b/murano/dsl/lhs_expression.py @@ -12,15 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. +import itertools import types -import yaql -import yaql.context -import yaql.expressions +from yaql.language import expressions +from yaql.language import specs +from yaql.language import utils +from yaql.language import yaqltypes -import murano.dsl.murano_object as murano_object -import murano.dsl.type_scheme as type_scheme -import murano.dsl.yaql_expression as yaql_expression +from murano.dsl import dsl_types +from murano.dsl import exceptions +from murano.dsl import yaql_expression +from murano.dsl import yaql_integration class LhsExpression(object): @@ -37,108 +40,95 @@ class LhsExpression(object): def __init__(self, expression): if isinstance(expression, (yaql_expression.YaqlExpression, - yaql.expressions.Expression)): + expressions.Statement)): self._expression = expression else: - self._expression = yaql.parse(str(expression)) - self._current_obj = None - self._current_obj_name = None - - def _create_context(self, root_context, murano_class): - def _evaluate(thing): - if isinstance(thing, yaql.expressions.Expression.Callable): - thing.yaql_context = root_context - thing = thing() - return thing - - def _get_value(src, key): - key = _evaluate(key) - if isinstance(src, types.DictionaryType): - return src.get(key) - elif isinstance(src, types.ListType) and isinstance( - key, types.IntType): - return src[key] - elif isinstance(src, murano_object.MuranoObject) and isinstance( - key, types.StringTypes): - self._current_obj = src - self._current_obj_name = key - return src.get_property(key, murano_class) - else: - raise TypeError() - - def _set_value(src, key, value): - key = _evaluate(key) - if isinstance(src, types.DictionaryType): - old_value = src.get(key, type_scheme.NoValue) - src[key] = value - if self._current_obj is not None: - try: - p_value = self._current_obj.get_property( - self._current_obj_name, murano_class) - self._current_obj.set_property( - self._current_obj_name, p_value, murano_class) - except Exception as e: - if old_value is not type_scheme.NoValue: - src[key] = old_value - else: - src.pop(key, None) - raise e - elif isinstance(src, types.ListType) and isinstance( - key, types.IntType): - old_value = src[key] - src[key] = value - if self._current_obj is not None: - try: - p_value = self._current_obj.get_property( - self._current_obj_name, murano_class) - self._current_obj.set_property( - self._current_obj_name, p_value, murano_class) - except Exception as e: - src[key] = old_value - raise e - - elif isinstance(src, murano_object.MuranoObject) and isinstance( - key, types.StringTypes): - src.set_property(key, value, murano_class) - else: - raise TypeError() + self._expression = yaql_integration.parse(str(expression)) + def _create_context(self, root_context): + @specs.parameter('path', yaqltypes.Lambda(with_context=True)) def get_context_data(path): - path = path() + path = path(root_context) def set_data(value): if not path or path == '$' or path == '$this': raise ValueError() - root_context.set_data(value, path) + root_context[path] = value return LhsExpression.Property( - lambda: root_context.get_data(path), set_data) + lambda: root_context[path], set_data) + + @specs.parameter('this', LhsExpression.Property) + @specs.parameter('key', yaqltypes.Keyword()) + def attribution(this, key): + def setter(src_property, value): + src = src_property.get() + if isinstance(src, utils.MappingType): + src_property.set( + utils.FrozenDict( + itertools.chain( + src.iteritems(), + ((key, value),)))) + elif isinstance(src, dsl_types.MuranoObject): + src.set_property(key, value, root_context) + else: + raise ValueError( + 'attribution may only be applied to ' + 'objects and dictionaries') + + def getter(src): + if isinstance(src, utils.MappingType): + return src.get(key, {}) + elif isinstance(src, dsl_types.MuranoObject): + self._current_obj = src + self._current_obj_name = key + try: + return src.get_property(key, root_context) + except exceptions.UninitializedPropertyAccessError: + return {} + + else: + raise ValueError( + 'attribution may only be applied to ' + 'objects and dictionaries') - @yaql.context.EvalArg('this', arg_type=LhsExpression.Property) - def attribution(this, arg_name): - arg_name = arg_name() return LhsExpression.Property( - lambda: _get_value(this.get(), arg_name), - lambda value: _set_value(this.get(), arg_name, value)) + lambda: getter(this.get()), + lambda value: setter(this, value)) - @yaql.context.EvalArg("this", LhsExpression.Property) + @specs.parameter('this', LhsExpression.Property) + @specs.parameter('index', yaqltypes.Lambda(with_context=True)) def indexation(this, index): - index = _evaluate(index) + index = index(root_context) - return LhsExpression.Property( - lambda: _get_value(this.get(), index), - lambda value: _set_value(this.get(), index, value)) + def getter(src): + if utils.is_sequence(src): + return src[index] + else: + raise ValueError('indexation may only be applied to lists') - context = yaql.context.Context() + def setter(src_property, value): + src = src_property.get() + if utils.is_sequence(src): + src_property.set(src[:index] + (value,) + src[index + 1:]) + + if isinstance(index, types.IntType): + return LhsExpression.Property( + lambda: getter(this.get()), + lambda value: setter(this, value)) + else: + return attribution(this, index) + + context = yaql_integration.create_empty_context() context.register_function(get_context_data, '#get_context_data') context.register_function(attribution, '#operator_.') - context.register_function(indexation, "where") + context.register_function(indexation, '#indexer') return context - def __call__(self, value, context, murano_class): - new_context = self._create_context(context, murano_class) - new_context.set_data(context.get_data('$')) + def __call__(self, value, context): + new_context = self._create_context(context) + new_context[''] = context['$'] self._current_obj = None self._current_obj_name = None - property = self._expression.evaluate(context=new_context) + property = self._expression(context=new_context) property.set(value) diff --git a/murano/dsl/macros.py b/murano/dsl/macros.py index 44b2738b7..232547c1a 100644 --- a/murano/dsl/macros.py +++ b/murano/dsl/macros.py @@ -14,14 +14,12 @@ import types -import eventlet.greenpool as greenpool -import yaql.context - -import murano.dsl.dsl_exception as dsl_exception -import murano.dsl.exceptions as exceptions -import murano.dsl.expressions as expressions -import murano.dsl.helpers as helpers -import murano.dsl.yaql_expression as yaql_expression +from murano.dsl import constants +from murano.dsl import dsl_exception +from murano.dsl import exceptions +from murano.dsl import expressions +from murano.dsl import helpers +from murano.dsl import yaql_expression class CodeBlock(expressions.DslExpression): @@ -30,23 +28,20 @@ class CodeBlock(expressions.DslExpression): body = [body] self.code_block = map(expressions.parse_expression, body) - def execute(self, context, murano_class): + def execute(self, context): for expr in self.code_block: - def action(): - try: - expr.execute(context, murano_class) - except (dsl_exception.MuranoPlException, - exceptions.InternalFlowException): - raise - except Exception as ex: - raise dsl_exception.MuranoPlException.\ - from_python_exception(ex, context) - if hasattr(expr, 'virtual_instruction'): instruction = expr.virtual_instruction - helpers.execute_instruction(instruction, action, context) - else: - action() + context[constants.CTX_CURRENT_INSTRUCTION] = instruction + + try: + expr.execute(context) + except (dsl_exception.MuranoPlException, + exceptions.InternalFlowException): + raise + except Exception as ex: + raise dsl_exception.MuranoPlException.from_python_exception( + ex, context) class MethodBlock(CodeBlock): @@ -54,11 +49,10 @@ class MethodBlock(CodeBlock): super(MethodBlock, self).__init__(body) self._name = name - def execute(self, context, murano_class): - new_context = yaql.context.Context(context) - new_context.set_data(self._name, '?currentMethod') + def execute(self, context): + new_context = context.create_child_context() try: - super(MethodBlock, self).execute(new_context, murano_class) + super(MethodBlock, self).execute(new_context) except exceptions.ReturnException as e: return e.value except exceptions.BreakException: @@ -75,7 +69,7 @@ class ReturnMacro(expressions.DslExpression): def __init__(self, Return): self._value = Return - def execute(self, context, murano_class): + def execute(self, context): raise exceptions.ReturnException( helpers.evaluate(self._value, context)) @@ -85,7 +79,7 @@ class BreakMacro(expressions.DslExpression): if Break: raise exceptions.DslSyntaxError('Break cannot have value') - def execute(self, context, murano_class): + def execute(self, context): raise exceptions.BreakException() @@ -94,7 +88,7 @@ class ContinueMacro(expressions.DslExpression): if Continue: raise exceptions.DslSyntaxError('Continue cannot have value') - def execute(self, context, murano_class): + def execute(self, context): raise exceptions.ContinueException() @@ -106,14 +100,12 @@ class ParallelMacro(CodeBlock): else: self._limit = len(self.code_block) - def execute(self, context, murano_class): + def execute(self, context): if not self.code_block: return limit = helpers.evaluate(self._limit, context) - gpool = greenpool.GreenPool(helpers.evaluate(limit, context)) - for expr in self.code_block: - gpool.spawn_n(expr.execute, context, murano_class) - gpool.waitall() + helpers.parallel_select( + self.code_block, lambda expr: expr.execute(context), limit) class IfMacro(expressions.DslExpression): @@ -125,15 +117,15 @@ class IfMacro(expressions.DslExpression): self._code2 = None if Else is None else CodeBlock(Else) self._condition = If - def execute(self, context, murano_class): - res = self._condition.evaluate(context) + def execute(self, context): + res = self._condition(context) if not isinstance(res, types.BooleanType): raise exceptions.DslInvalidOperationError( 'Condition must be evaluated to boolean type') if res: - self._code1.execute(context, murano_class) + self._code1.execute(context) elif self._code2 is not None: - self._code2.execute(context, murano_class) + self._code2.execute(context) class WhileDoMacro(expressions.DslExpression): @@ -143,15 +135,15 @@ class WhileDoMacro(expressions.DslExpression): self._code = CodeBlock(Do) self._condition = While - def execute(self, context, murano_class): + def execute(self, context): while True: - res = self._condition.evaluate(context) + res = self._condition(context) if not isinstance(res, types.BooleanType): raise exceptions.DslSyntaxError( 'Condition must be of expression type') try: if res: - self._code.execute(context, murano_class) + self._code.execute(context) else: break except exceptions.BreakException: @@ -169,12 +161,12 @@ class ForMacro(expressions.DslExpression): self._var = For self._collection = In - def execute(self, context, murano_class): + def execute(self, context): collection = helpers.evaluate(self._collection, context) for t in collection: - context.set_data(t, self._var) + context[self._var] = t try: - self._code.execute(context, murano_class) + self._code.execute(context) except exceptions.BreakException: break except exceptions.ContinueException: @@ -189,11 +181,11 @@ class RepeatMacro(expressions.DslExpression): self._count = Repeat self._code = CodeBlock(Do) - def execute(self, context, murano_class): + def execute(self, context): count = helpers.evaluate(self._count, context) for t in range(0, count): try: - self._code.execute(context, murano_class) + self._code.execute(context) except exceptions.BreakException: break except exceptions.ContinueException: @@ -209,14 +201,14 @@ class MatchMacro(expressions.DslExpression): self._value = Value self._default = None if Default is None else CodeBlock(Default) - def execute(self, context, murano_class): + def execute(self, context): match_value = helpers.evaluate(self._value, context) for key, value in self._switch.iteritems(): if key == match_value: - CodeBlock(value).execute(context, murano_class) + CodeBlock(value).execute(context) return if self._default is not None: - self._default.execute(context, murano_class) + self._default.execute(context) class SwitchMacro(expressions.DslExpression): @@ -233,7 +225,7 @@ class SwitchMacro(expressions.DslExpression): 'boolean or expression') self._default = None if Default is None else CodeBlock(Default) - def execute(self, context, murano_class): + def execute(self, context): matched = False for key, value in self._switch.iteritems(): res = helpers.evaluate(key, context) @@ -242,18 +234,18 @@ class SwitchMacro(expressions.DslExpression): 'Switch case must be evaluated to boolean type') if res: matched = True - CodeBlock(value).execute(context, murano_class) + CodeBlock(value).execute(context) if self._default is not None and not matched: - self._default.execute(context, murano_class) + self._default.execute(context) class DoMacro(expressions.DslExpression): def __init__(self, Do): self._code = CodeBlock(Do) - def execute(self, context, murano_class): - self._code.execute(context, murano_class) + def execute(self, context): + self._code.execute(context) def register(): diff --git a/murano/dsl/murano_class.py b/murano/dsl/murano_class.py index a020dabef..ff686ce94 100644 --- a/murano/dsl/murano_class.py +++ b/murano/dsl/murano_class.py @@ -13,23 +13,22 @@ # under the License. import collections -import inspect -import murano.dsl.exceptions as exceptions -import murano.dsl.helpers as helpers -import murano.dsl.murano_method as murano_method -import murano.dsl.murano_object as murano_object -import murano.dsl.typespec as typespec +from murano.dsl import dsl +from murano.dsl import dsl_types +from murano.dsl import exceptions +from murano.dsl import murano_method +from murano.dsl import murano_object +from murano.dsl import typespec +from murano.dsl import yaql_integration -def classname(name): - def wrapper(cls): - cls._murano_class_name = name - return cls - return wrapper +class GeneratedNativeTypeMetaClass(type): + def __str__(cls): + return cls.__name__ -class MuranoClass(object): +class MuranoClass(dsl_types.MuranoClass): def __init__(self, class_loader, namespace_resolver, name, package, parents=None): self._package = package @@ -44,12 +43,7 @@ class MuranoClass(object): else: self._parents = parents or [ class_loader.get_class('io.murano.Object')] - - class_name = 'mc' + helpers.generate_id() - parents_class = [p.object_class for p in self._parents] - bases = tuple(parents_class) or (murano_object.MuranoObject,) - - self.object_class = type(class_name, bases, {}) + self._unique_methods = None @property def name(self): @@ -71,18 +65,35 @@ class MuranoClass(object): def methods(self): return self._methods + def extend_with_class(self, cls): + ctor = yaql_integration.get_class_factory_definition(cls) + self.add_method('__init__', ctor) + + @property + def unique_methods(self): + if self._unique_methods is None: + self._unique_methods = list(self._iterate_unique_methods()) + return self._unique_methods + def get_method(self, name): return self._methods.get(name) def add_method(self, name, payload): method = murano_method.MuranoMethod(self, name, payload) self._methods[name] = method + self._unique_methods = None return method @property def properties(self): return self._properties.keys() + def register_methods(self, context): + for method in self.unique_methods: + context.register_function( + method.yaql_function_definition, + name=method.yaql_function_definition.name) + def add_property(self, name, property_typespec): if not isinstance(property_typespec, typespec.PropertySpec): raise TypeError('property_typespec') @@ -150,6 +161,16 @@ class MuranoClass(object): queue.extend(c.parents) return result + def _iterate_unique_methods(self): + names = set() + queue = collections.deque([self]) + while queue: + c = queue.popleft() + names.update(c.methods.keys()) + queue.extend(c.parents) + for name in names: + yield self.find_single_method(name) + def find_property(self, name): result = [] types = collections.deque([self]) @@ -160,14 +181,13 @@ class MuranoClass(object): types.extend(mc.parents) return result - def invoke(self, name, executor, this, parameters): - if not self.is_compatible(this): - raise Exception("'this' must be of compatible type") - args = executor.to_yaql_args(parameters) - return executor.invoke_method(name, this.cast(self), None, self, *args) + def invoke(self, name, executor, this, args, kwargs, context=None): + method = self.find_single_method(name) + return method.invoke(executor, this, args, kwargs, context) def is_compatible(self, obj): - if isinstance(obj, murano_object.MuranoObject): + if isinstance(obj, (murano_object.MuranoObject, + dsl.MuranoObjectInterface)): return self.is_compatible(obj.type) if obj is self: return True @@ -176,19 +196,20 @@ class MuranoClass(object): return True return False - def new(self, owner, object_store, context, parameters=None, - object_id=None, **kwargs): + def new(self, owner, object_store, context=None, **kwargs): + if context is None: + context = object_store.context + obj = murano_object.MuranoObject( + self, owner, object_store.context, **kwargs) - obj = self.object_class(self, owner, object_store, context, - object_id=object_id, **kwargs) - if parameters is not None: - argspec = inspect.getargspec(obj.initialize).args - if '_context' in argspec: - parameters['_context'] = context - if '_owner' in argspec: - parameters['_owner'] = owner - obj.initialize(**parameters) - return obj + def initializer(**params): + init_context = context.create_child_context() + init_context['?allowPropertyWrites'] = True + obj.initialize(init_context, object_store, params) + return obj + + initializer.object = obj + return initializer def __str__(self): return 'MuranoClass({0})'.format(self.name) diff --git a/murano/dsl/murano_method.py b/murano/dsl/murano_method.py index 49be8ee64..d96ce484c 100644 --- a/murano/dsl/murano_method.py +++ b/murano/dsl/murano_method.py @@ -13,13 +13,16 @@ # under the License. import collections -import inspect import types -import murano.dsl.macros as macros -import murano.dsl.typespec as typespec -import murano.dsl.virtual_exceptions as virtual_exceptions -import murano.dsl.yaql_expression as yaql_expression +from yaql.language import specs + +from murano.dsl import dsl +from murano.dsl import dsl_types +from murano.dsl import macros +from murano.dsl import typespec +from murano.dsl import virtual_exceptions +from murano.dsl import yaql_integration macros.register() @@ -32,26 +35,27 @@ class MethodUsages(object): All = set([Action, Runtime]) -def methodusage(usage): - def wrapper(method): - method._murano_method_usage = usage - return method - return wrapper - - -class MuranoMethod(object): +class MuranoMethod(dsl_types.MuranoMethod): def __init__(self, murano_class, name, payload): self._name = name self._murano_class = murano_class if callable(payload): - self._body = payload - self._arguments_scheme = self._generate_arguments_scheme(payload) - self._usage = getattr(payload, '_murano_method_usage', - MethodUsages.Runtime) + if isinstance(payload, specs.FunctionDefinition): + self._body = payload + else: + self._body = yaql_integration.get_function_definition(payload) + self._arguments_scheme = None + self._usage = (self._body.meta.get('usage') or + self._body.meta.get('Usage') or + MethodUsages.Runtime) + if (self._body.name.startswith('#') + or self._body.name.startswith('*')): + raise ValueError( + 'Import of special yaql functions is forbidden') else: payload = payload or {} - self._body = self._prepare_body(payload.get('Body') or [], name) + self._body = macros.MethodBlock(payload.get('Body') or [], name) self._usage = payload.get('Usage') or MethodUsages.Runtime arguments_scheme = payload.get('Arguments') or [] if isinstance(arguments_scheme, types.DictionaryType): @@ -59,12 +63,14 @@ class MuranoMethod(object): arguments_scheme.iteritems()] self._arguments_scheme = collections.OrderedDict() for record in arguments_scheme: - if not isinstance(record, types.DictionaryType) \ - or len(record) > 1: + if (not isinstance(record, types.DictionaryType) + or len(record) > 1): raise ValueError() name = record.keys()[0] self._arguments_scheme[name] = typespec.ArgumentSpec( - record[name], murano_class) + record[name]) + self._yaql_function_definition = \ + yaql_integration.build_wrapper_function_definition(self) @property def name(self): @@ -78,36 +84,32 @@ class MuranoMethod(object): def arguments_scheme(self): return self._arguments_scheme + @property + def yaql_function_definition(self): + return self._yaql_function_definition + @property def usage(self): return self._usage + @usage.setter + def usage(self, value): + self._usage = value + @property def body(self): return self._body - def _generate_arguments_scheme(self, func): - func_info = inspect.getargspec(func) - data = [(name, {'Contract': yaql_expression.YaqlExpression('$')}) - for name in func_info.args] - if inspect.ismethod(func): - data = data[1:] - defaults = func_info.defaults or tuple() - for i in xrange(len(defaults)): - data[i + len(data) - len(defaults)][1]['Default'] = defaults[i] - result = collections.OrderedDict([ - (name, typespec.ArgumentSpec(declaration, self.murano_class)) - for name, declaration in data]) - if '_context' in result: - del result['_context'] - return result - - def _prepare_body(self, body, name): - return macros.MethodBlock(body, name) - def __repr__(self): return 'MuranoMethod({0}::{1})'.format( self.murano_class.name, self.name) - def invoke(self, executor, this, parameters): - return self.murano_class.invoke(self.name, executor, this, parameters) + def invoke(self, executor, this, args, kwargs, context=None, + skip_stub=False): + if not self.murano_class.is_compatible(this): + raise Exception("'this' must be of compatible type") + if isinstance(this, dsl.MuranoObjectInterface): + this = this.object + return executor.invoke_method( + self, this.cast(self.murano_class), + context, args, kwargs, skip_stub) diff --git a/murano/dsl/murano_object.py b/murano/dsl/murano_object.py index 66ce06b71..512510fab 100644 --- a/murano/dsl/murano_object.py +++ b/murano/dsl/murano_object.py @@ -12,30 +12,31 @@ # License for the specific language governing permissions and limitations # under the License. -import yaml -import yaql.context - -import murano.dsl.exceptions as exceptions -import murano.dsl.helpers -import murano.dsl.type_scheme as type_scheme -import murano.dsl.typespec as typespec +from murano.dsl import constants +from murano.dsl import dsl +from murano.dsl import dsl_types +from murano.dsl import exceptions +from murano.dsl import helpers +from murano.dsl import typespec +from murano.dsl import yaql_integration -class MuranoObject(object): - def __init__(self, murano_class, owner, object_store, context, - object_id=None, known_classes=None, defaults=None, this=None): - +class MuranoObject(dsl_types.MuranoObject): + def __init__(self, murano_class, owner, context, object_id=None, + name=None, known_classes=None, defaults=None, this=None): if known_classes is None: known_classes = {} self.__owner = owner - self.__object_id = object_id or murano.dsl.helpers.generate_id() + self.__object_id = object_id or helpers.generate_id() self.__type = murano_class self.__properties = {} - self.__object_store = object_store self.__parents = {} - self.__context = context self.__defaults = defaults or {} self.__this = this + self.__name = name + self.__extension = None + self.__context = self.__setup_context(context) + object_store = helpers.get_object_store(context) self.__config = object_store.class_loader.get_class_config( murano_class.name) if not isinstance(self.__config, dict): @@ -44,52 +45,118 @@ class MuranoObject(object): for parent_class in murano_class.parents: name = parent_class.name if name not in known_classes: - obj = parent_class.new(owner, object_store, context, - None, object_id=self.__object_id, - known_classes=known_classes, - defaults=defaults, this=self.real_this) + obj = parent_class.new( + owner, object_store, context, + object_id=self.__object_id, + known_classes=known_classes, + defaults=defaults, this=self.real_this).object self.__parents[name] = known_classes[name] = obj else: self.__parents[name] = known_classes[name] + self.__initialized = False - def initialize(self, **kwargs): - used_names = set() + def __setup_context(self, context): + context = context.create_child_context() + context[constants.CTX_THIS] = self.real_this + context[constants.CTX_TYPE] = self.type + context['this'] = self.real_this + context[''] = self.real_this + return context + + @property + def context(self): + return self.__context + + @property + def extension(self): + return self.__extension + + @property + def name(self): + return self.real_this.__name + + @extension.setter + def extension(self, value): + self.__extension = value + + def initialize(self, context, object_store, params): + if self.__initialized: + return for property_name in self.__type.properties: spec = self.__type.get_property(property_name) if spec.usage == typespec.PropertyUsages.Config: if property_name in self.__config: property_value = self.__config[property_name] else: - property_value = type_scheme.NoValue + property_value = dsl.NO_VALUE self.set_property(property_name, property_value) - for i in xrange(2): - for property_name in self.__type.properties: - spec = self.__type.get_property(property_name) + init = self.type.methods.get('.init') + used_names = set() + names = set(self.__type.properties) + if init: + names.update(init.arguments_scheme.iterkeys()) + last_errors = len(names) + init_args = {} + while True: + errors = 0 + for property_name in names: + if init and property_name in init.arguments_scheme: + spec = init.arguments_scheme[property_name] + is_init_arg = True + else: + spec = self.__type.get_property(property_name) + is_init_arg = False + + if property_name in used_names: + continue if spec.usage == typespec.PropertyUsages.Config: + used_names.add(property_name) continue - needs_evaluation = murano.dsl.helpers.needs_evaluation - if i == 0 and needs_evaluation(spec.default) or i == 1\ - and property_name in used_names: - continue - used_names.add(property_name) if spec.usage == typespec.PropertyUsages.Runtime: if not spec.has_default: + used_names.add(property_name) continue - property_value = type_scheme.NoValue + property_value = dsl.NO_VALUE else: - property_value = kwargs.get(property_name, - type_scheme.NoValue) + property_value = params.get(property_name, dsl.NO_VALUE) try: - self.set_property(property_name, property_value) + if is_init_arg: + init_args[property_name] = property_value + else: + self.set_property(property_name, property_value) + used_names.add(property_name) + except exceptions.UninitializedPropertyAccessError: + errors += 1 except exceptions.ContractViolationException: if spec.usage != typespec.PropertyUsages.Runtime: raise + if not errors: + break + if errors >= last_errors: + raise exceptions.CircularExpressionDependenciesError() + last_errors = errors + + executor = helpers.get_executor(context) + if not object_store.initializing and self.__extension is None: + method = self.type.methods.get('__init__') + if method: + filtered_params = yaql_integration.filter_parameters( + method.body, **params) + + self.__extension = method.invoke( + executor, self, filtered_params[0], + filtered_params[1], context) for parent in self.__parents.values(): - parent.initialize(**kwargs) - self.__initialized = True + parent.initialize(context, object_store, params) + + if not object_store.initializing and init: + init_context = context.create_child_context() + init_context[constants.CTX_ARGUMENT_OWNER] = self.real_this + init.invoke(executor, self.real_this, (), init_args, init_context) + self.__initialized = True @property def object_id(self): @@ -107,14 +174,9 @@ class MuranoObject(object): def real_this(self): return self.__this or self - def __getattr__(self, item): - if item.startswith('__'): - raise AttributeError('Access to internal attributes is ' - 'restricted') - return self.get_property(item) - - def get_property(self, name, caller_class=None): + def get_property(self, name, context=None): start_type, derived = self.__type, False + caller_class = None if not context else helpers.get_type(context) if caller_class is not None and caller_class.is_compatible(self): start_type, derived = caller_class, True if name in start_type.properties: @@ -137,8 +199,9 @@ class MuranoObject(object): raise exceptions.UninitializedPropertyAccessError( name, self.__type) - def set_property(self, name, value, caller_class=None): + def set_property(self, name, value, context=None): start_type, derived = self.__type, False + caller_class = None if not context else helpers.get_type(context) if caller_class is not None and caller_class.is_compatible(self): start_type, derived = caller_class, True declared_properties = start_type.find_property(name) @@ -147,28 +210,25 @@ class MuranoObject(object): values_to_assign = [] for mc in declared_properties: spec = mc.get_property(name) - if caller_class is not None: - if spec.usage not in typespec.PropertyUsages.Writable \ - or not derived: - raise exceptions.NoWriteAccessError(name) + if (caller_class is not None and + not helpers.are_property_modifications_allowed(context) + and (spec.usage not in typespec.PropertyUsages.Writable + or not derived)): + raise exceptions.NoWriteAccessError(name) default = self.__config.get(name, spec.default) default = self.__defaults.get(name, default) - child_context = yaql.context.Context( - parent_context=self.__context) - child_context.set_data(self) - default = murano.dsl.helpers.evaluate( - default, child_context, 1) + default = helpers.evaluate(default, context or self.context) obj = self.cast(mc) values_to_assign.append((obj, spec.validate( - value, self, self, - self.__context, self.__object_store, default))) + value, context or self.context, self.real_this, + self.real_this, default=default))) for obj, value in values_to_assign: obj.__properties[name] = value elif derived: - obj = self.cast(caller_class) - obj.__properties[name] = value + obj = self.cast(caller_class) + obj.__properties[name] = value else: raise exceptions.PropertyWriteError(name, start_type) @@ -183,13 +243,18 @@ class MuranoObject(object): raise TypeError('Cannot cast') def __repr__(self): - return yaml.safe_dump(murano.dsl.helpers.serialize(self)) + return '<{0} {1} ({2})>'.format( + self.type.name, self.object_id, id(self)) def to_dictionary(self, include_hidden=False): result = {} for parent in self.__parents.values(): result.update(parent.to_dictionary(include_hidden)) - result.update({'?': {'type': self.type.name, 'id': self.object_id}}) + result.update({'?': { + 'type': self.type.name, + 'id': self.object_id, + 'name': self.name + }}) if include_hidden: result.update(self.__properties) else: @@ -197,6 +262,6 @@ class MuranoObject(object): if property_name in self.__properties: spec = self.type.get_property(property_name) if spec.usage != typespec.PropertyUsages.Runtime: - result[property_name] = \ - self.__properties[property_name] + result[property_name] = self.__properties[ + property_name] return result diff --git a/murano/dsl/murano_package.py b/murano/dsl/murano_package.py index 58461d13d..99d992406 100644 --- a/murano/dsl/murano_package.py +++ b/murano/dsl/murano_package.py @@ -12,8 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. +from murano.dsl import dsl_types -class MuranoPackage(object): + +class MuranoPackage(dsl_types.MuranoPackage): def __init__(self): super(MuranoPackage, self).__init__() self._name = None diff --git a/murano/dsl/object_store.py b/murano/dsl/object_store.py index 70af19799..e0b3b20cb 100644 --- a/murano/dsl/object_store.py +++ b/murano/dsl/object_store.py @@ -12,14 +12,16 @@ # License for the specific language governing permissions and limitations # under the License. -import inspect - -import murano.dsl.helpers as helpers +from murano.dsl import constants +from murano.dsl import dsl_types +from murano.dsl import helpers class ObjectStore(object): - def __init__(self, class_loader, parent_store=None): - self._class_loader = class_loader + def __init__(self, context, parent_store=None): + self._context = context.create_child_context() + self._class_loader = helpers.get_class_loader(context) + self._context[constants.CTX_OBJECT_STORE] = self self._parent_store = parent_store self._store = {} self._designer_attributes_store = {} @@ -33,9 +35,16 @@ class ObjectStore(object): def class_loader(self): return self._class_loader + @property + def context(self): + return self._context + def get(self, object_id): if object_id in self._store: - return self._store[object_id] + result = self._store[object_id] + if not isinstance(result, dsl_types.MuranoObject): + result = None + return result if self._parent_store: return self._parent_store.get(object_id) return None @@ -46,7 +55,7 @@ class ObjectStore(object): def put(self, murano_object): self._store[murano_object.object_id] = murano_object - def load(self, value, owner, context, defaults=None): + def load(self, value, owner, defaults=None): if value is None: return None if '?' not in value or 'type' not in value['?']: @@ -57,39 +66,35 @@ class ObjectStore(object): class_obj = self._class_loader.get_class(obj_type) if not class_obj: raise ValueError() - if object_id in self._store: - obj = self._store[object_id] - else: - obj = class_obj.new(owner, self, context=context, - object_id=object_id, defaults=defaults) - self._store[object_id] = obj - self._designer_attributes_store[object_id] = \ - ObjectStore._get_designer_attributes(system_key) - - argspec = inspect.getargspec(obj.initialize).args - if '_context' in argspec: - value['_context'] = context - if '_parent' in argspec: - value['_owner'] = owner try: if owner is None: self._initializing = True - obj.initialize(**value) + + if object_id in self._store: + factory = self._store[object_id] + if isinstance(factory, dsl_types.MuranoObject): + return factory + else: + factory = class_obj.new( + owner, self, context=self.context, + name=system_key.get('name'), + object_id=object_id, defaults=defaults) + self._store[object_id] = factory + system_value = ObjectStore._get_designer_attributes(system_key) + self._designer_attributes_store[object_id] = system_value + + obj = factory(**value) + if not self._initializing: + self._store[object_id] = obj if owner is None: self._initializing = False - obj.initialize(**value) + self._store[object_id] = factory(**value) finally: if owner is None: self._initializing = False - if not self.initializing: - executor = helpers.get_executor(context) - methods = obj.type.find_all_methods('initialize') - methods.reverse() - for method in methods: - method.invoke(executor, obj, {}) - return obj + return factory.object @staticmethod def _get_designer_attributes(header): diff --git a/murano/dsl/principal_objects/exception.py b/murano/dsl/principal_objects/exception.py index 9cdb0514d..79dcd8c03 100644 --- a/murano/dsl/principal_objects/exception.py +++ b/murano/dsl/principal_objects/exception.py @@ -12,11 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -from murano.dsl import murano_class -from murano.dsl import murano_object +from murano.dsl import dsl -@murano_class.classname('io.murano.Exception') -class DslException(murano_object.MuranoObject): - def toString(self): +@dsl.name('io.murano.Exception') +class DslException(object): + def to_string(self): return self.get_property('nativeException').format() diff --git a/murano/dsl/principal_objects/stack_trace.py b/murano/dsl/principal_objects/stack_trace.py index c18a8c1e7..342cb3f9d 100644 --- a/murano/dsl/principal_objects/stack_trace.py +++ b/murano/dsl/principal_objects/stack_trace.py @@ -15,32 +15,35 @@ import inspect import os.path +from yaql import specs + +from murano.dsl import constants +from murano.dsl import dsl +from murano.dsl import dsl_types from murano.dsl import helpers -from murano.dsl import murano_class -from murano.dsl import murano_object -from murano.dsl import yaql_expression +from murano.dsl import yaql_integration -@murano_class.classname('io.murano.StackTrace') -class StackTrace(murano_object.MuranoObject): - - def initialize(self, _context, includeNativeFrames=True): +@dsl.name('io.murano.StackTrace') +class StackTrace(object): + def __init__(self, this, context, include_native_frames=True): frames = [] - context = _context + caller_context = context while True: - if not context: + if not caller_context: break - frames.append(compose_stack_frame(context)) - context = helpers.get_caller_context(context) - frames.pop() + frame = compose_stack_frame(caller_context) + frames.append(frame) + caller_context = helpers.get_caller_context(caller_context) frames.reverse() + frames.pop() - if includeNativeFrames: + if include_native_frames: native_frames = [] for frame in inspect.trace()[1:]: - location = yaql_expression.YaqlExpressionFilePosition( + location = dsl_types.ExpressionFilePosition( os.path.abspath(frame[1]), frame[2], - -1, -1, -1, -1, -1) + -1, frame[2], -1) method = frame[3] native_frames.append({ 'instruction': frame[4][0].strip(), @@ -50,15 +53,17 @@ class StackTrace(murano_object.MuranoObject): }) frames.extend(native_frames) - self.set_property('frames', frames) + this.data().frames = frames - def toString(self, prefix=''): - return '\n'.join([format_frame(t, prefix)for t in self.get_property( - 'frames')]) + @specs.meta(constants.META_NO_TRACE, True) + @specs.meta('Usage', 'Action') + def to_string(self, this, prefix=''): + return '\n'.join([format_frame(t, prefix)for t in this['frames']]) def compose_stack_frame(context): instruction = helpers.get_current_instruction(context) + method = helpers.get_current_method(context) return { 'instruction': None if instruction is None else str(instruction), @@ -66,8 +71,8 @@ def compose_stack_frame(context): 'location': None if instruction is None else instruction.source_file_position, - 'method': helpers.get_current_method(context), - 'class': helpers.get_type(context) + 'method': None if method is None else method.name, + 'class': None if method is None else method.murano_class.name } @@ -77,7 +82,7 @@ def format_frame(frame, prefix=''): murano_class = frame['class'] location = frame['location'] if murano_class: - method += ' of class ' + murano_class.name + method += ' of class ' + murano_class if location: args = ( @@ -89,8 +94,17 @@ def format_frame(frame, prefix=''): instruction, prefix ) - return '{5}File "{0}", line {1}{2} in method {3}\n' \ - '{5} {4}'.format(*args) + return ('{5}File "{0}", line {1}{2} in method {3}\n' + '{5} {4}').format(*args) else: return '{2}File in method {0}\n{2} {1}'.format( method, instruction, prefix) + + +def create_stack_trace(context, include_native_frames=True): + stacktrace = yaql_integration.call_func( + context, 'new', 'io.murano.StackTrace', + includeNativeFrames=include_native_frames) + executor = helpers.get_executor(context) + return dsl.MuranoObjectInterface( + stacktrace, yaql_integration.ENGINE, executor) diff --git a/murano/dsl/principal_objects/sys_object.py b/murano/dsl/principal_objects/sys_object.py index e79f582df..aacdc8e5d 100644 --- a/murano/dsl/principal_objects/sys_object.py +++ b/murano/dsl/principal_objects/sys_object.py @@ -12,28 +12,24 @@ # License for the specific language governing permissions and limitations # under the License. -import murano.dsl.helpers as helpers -import murano.dsl.murano_class as murano_class +from murano.dsl import dsl +from murano.dsl import helpers -@murano_class.classname('io.murano.Object') +@dsl.name('io.murano.Object') class SysObject(object): - def setAttr(self, _context, name, value, owner=None): + def set_attr(self, this, context, name, value, owner=None): if owner is None: - owner = helpers.get_type(helpers.get_caller_context(_context)) - if not isinstance(owner, murano_class.MuranoClass): - raise TypeError() + owner = helpers.get_type(helpers.get_caller_context(context)) - attribute_store = helpers.get_attribute_store(_context) - attribute_store.set(self, owner, name, value) + attribute_store = helpers.get_attribute_store(context) + attribute_store.set(this.object, owner, name, value) - def getAttr(self, _context, name, default=None, owner=None): + def get_attr(self, this, context, name, default=None, owner=None): if owner is None: - owner = helpers.get_type(helpers.get_caller_context(_context)) - if not isinstance(owner, murano_class.MuranoClass): - raise TypeError() + owner = helpers.get_type(helpers.get_caller_context(context)) - attribute_store = helpers.get_attribute_store(_context) + attribute_store = helpers.get_attribute_store(context) - result = attribute_store.get(self, owner, name) + result = attribute_store.get(this.object, owner, name) return default if result is None else result diff --git a/murano/dsl/serializer.py b/murano/dsl/serializer.py index 2b9cfd0cb..c19521001 100644 --- a/murano/dsl/serializer.py +++ b/murano/dsl/serializer.py @@ -12,12 +12,14 @@ # License for the specific language governing permissions and limitations # under the License. -import collections import types -import murano.dsl.helpers as helpers -import murano.dsl.murano_method as murano_method -import murano.dsl.murano_object as murano_object +from yaql import utils + +from murano.dsl import dsl +from murano.dsl import dsl_types +from murano.dsl import helpers +from murano.dsl import murano_method class ObjRef(object): @@ -25,38 +27,42 @@ class ObjRef(object): self.ref_obj = obj -def serialize_object(obj): - if isinstance(obj, (collections.Sequence, collections.Set)) and not \ - isinstance(obj, types.StringTypes): - return [serialize_object(t) for t in obj] - elif isinstance(obj, collections.Mapping): - result = {} - for key, value in obj.iteritems(): - result[key] = serialize_object(value) - return result - elif isinstance(obj, murano_object.MuranoObject): - return _serialize_object(obj, None)[0] - return obj +def serialize(obj): + return serialize_model(obj, None, True)['Objects'] -def _serialize_object(root_object, designer_attributes=None): +def _serialize_object(root_object, designer_attributes, allow_refs): serialized_objects = set() - tree = _pass1_serialize( - root_object, None, serialized_objects, designer_attributes) - _pass2_serialize(tree, serialized_objects) - return tree, serialized_objects + + obj = root_object + while True: + obj, need_another_pass = _pass12_serialize( + obj, None, serialized_objects, designer_attributes) + if not need_another_pass: + break + tree = [obj] + _pass3_serialize(tree, serialized_objects, allow_refs) + return tree[0], serialized_objects -def serialize_model(root_object, executor): +def serialize_model(root_object, executor, allow_refs=False): + if executor is not None: + designer_attributes = executor.object_store.designer_attributes + else: + designer_attributes = None + if root_object is None: tree = None tree_copy = None attributes = [] else: tree, serialized_objects = _serialize_object( - root_object, executor.object_store.designer_attributes) - tree_copy, _ = _serialize_object(root_object, None) - attributes = executor.attribute_store.serialize(serialized_objects) + root_object, designer_attributes, allow_refs) + tree_copy, _ = _serialize_object(root_object, None, allow_refs) + if executor is not None: + attributes = executor.attribute_store.serialize(serialized_objects) + else: + attributes = [] return { 'Objects': tree, @@ -65,14 +71,6 @@ def serialize_model(root_object, executor): } -def _cmp_objects(obj1, obj2): - if obj1 is None and obj2 is None: - return True - if obj1 is None or obj2 is None: - return False - return obj1.object_id == obj2.object_id - - def _serialize_available_action(obj): def _serialize(obj_type): actions = {} @@ -98,66 +96,91 @@ def _merge_actions(dict1, dict2): return result -def _pass1_serialize(value, parent, serialized_objects, - designer_attributes_getter): +def _pass12_serialize(value, parent, serialized_objects, + designer_attributes_getter): + if isinstance(value, dsl.MuranoObjectInterface): + value = value.object if isinstance(value, (types.StringTypes, types.IntType, types.FloatType, types.BooleanType, types.NoneType)): - return value - elif isinstance(value, murano_object.MuranoObject): - if not _cmp_objects(value.owner, parent) \ - or value.object_id in serialized_objects: - return ObjRef(value) + return value, False + if isinstance(value, dsl_types.MuranoObject): + if value.owner is not parent or value.object_id in serialized_objects: + return ObjRef(value), True + elif isinstance(value, ObjRef): + if (value.ref_obj.object_id not in serialized_objects + and is_nested_in(value.ref_obj.owner, parent)): + value = value.ref_obj else: - result = value.to_dictionary() - if designer_attributes_getter is not None: - result['?'].update(designer_attributes_getter(value.object_id)) - # deserialize and merge list of actions - actions = _serialize_available_action(value) - result['?']['_actions'] = _merge_actions( - result['?'].get('_actions', {}), actions) - serialized_objects.add(value.object_id) - return _pass1_serialize( - result, value, serialized_objects, designer_attributes_getter) - - elif isinstance(value, types.DictionaryType): + return value, False + if isinstance(value, dsl_types.MuranoObject): + result = value.to_dictionary() + if designer_attributes_getter is not None: + result['?'].update(designer_attributes_getter(value.object_id)) + # deserialize and merge list of actions + actions = _serialize_available_action(value) + result['?']['_actions'] = _merge_actions( + result['?'].get('_actions', {}), actions) + serialized_objects.add(value.object_id) + return _pass12_serialize( + result, value, serialized_objects, designer_attributes_getter) + elif isinstance(value, utils.MappingType): result = {} + need_another_pass = False + for d_key, d_value in value.iteritems(): result_key = str(d_key) - result[result_key] = _pass1_serialize( + result_value = _pass12_serialize( d_value, parent, serialized_objects, designer_attributes_getter) - return result - elif isinstance(value, types.ListType): - return [_pass1_serialize(t, parent, serialized_objects, - designer_attributes_getter) for t in value] - elif isinstance(value, types.TupleType): - return _pass1_serialize( - list(value), parent, serialized_objects, - designer_attributes_getter) + result[result_key] = result_value[0] + if result_value[1]: + need_another_pass = True + return result, need_another_pass + elif utils.is_sequence(value) or isinstance(value, utils.SetType): + need_another_pass = False + result = [] + for t in value: + v, nmp = _pass12_serialize( + t, parent, serialized_objects, designer_attributes_getter) + if nmp: + need_another_pass = True + result.append(v) + return result, need_another_pass else: raise ValueError() -def _pass2_serialize(value, serialized_objects): - if isinstance(value, types.DictionaryType): - for d_key, d_value in value.iteritems(): +def _pass3_serialize(value, serialized_objects, allow_refs=False): + if isinstance(value, dict): + for d_key, d_value in value.items(): if isinstance(d_value, ObjRef): - if d_value.ref_obj.object_id in serialized_objects: + if (d_value.ref_obj.object_id in serialized_objects + or allow_refs): value[d_key] = d_value.ref_obj.object_id else: - value[d_key] = None + del value[d_key] else: - _pass2_serialize(d_value, serialized_objects) - elif isinstance(value, types.ListType): + _pass3_serialize(d_value, serialized_objects, allow_refs) + elif isinstance(value, list): index = 0 while index < len(value): item = value[index] if isinstance(item, ObjRef): - if item.ref_obj.object_id in serialized_objects: + if item.ref_obj.object_id in serialized_objects or allow_refs: value[index] = item.ref_obj.object_id else: value.pop(index) index -= 1 else: - _pass2_serialize(item, serialized_objects) + _pass3_serialize(item, serialized_objects, allow_refs) index += 1 + return value + + +def is_nested_in(obj, ancestor): + while True: + if obj is ancestor: + return True + if obj is None: + return False + obj = obj.owner diff --git a/murano/dsl/type_scheme.py b/murano/dsl/type_scheme.py index 70bfe3570..94b624551 100644 --- a/murano/dsl/type_scheme.py +++ b/murano/dsl/type_scheme.py @@ -16,15 +16,15 @@ import sys import types import uuid -import yaql.context +from yaql.language import specs +from yaql.language import utils +from yaql.language import yaqltypes +from murano.dsl import dsl +from murano.dsl import dsl_types from murano.dsl import exceptions -import murano.dsl.helpers -import murano.dsl.murano_object -import murano.dsl.yaql_expression as yaql_expression - - -NoValue = object() +from murano.dsl import helpers +from murano.dsl import yaql_integration class TypeScheme(object): @@ -36,11 +36,11 @@ class TypeScheme(object): self._spec = spec @staticmethod - def prepare_context(root_context, this, owner, object_store, - namespace_resolver, default): - def _int(value): - value = value() - if value is NoValue: + def prepare_context(root_context, this, owner, default): + @specs.parameter('value', nullable=True) + @specs.method + def int_(value): + if value is dsl.NO_VALUE: value = default if value is None: return None @@ -50,9 +50,10 @@ class TypeScheme(object): raise exceptions.ContractViolationException( 'Value {0} violates int() contract'.format(value)) - def _string(value): - value = value() - if value is NoValue: + @specs.parameter('value', nullable=True) + @specs.method + def string(value): + if value is dsl.NO_VALUE: value = default if value is None: return None @@ -62,17 +63,18 @@ class TypeScheme(object): raise exceptions.ContractViolationException( 'Value {0} violates string() contract'.format(value)) - def _bool(value): - value = value() - if value is NoValue: + @specs.parameter('value', nullable=True) + @specs.method + def bool_(value): + if value is dsl.NO_VALUE: value = default if value is None: return None return True if value else False - def _not_null(value): - value = value() - + @specs.parameter('value', nullable=True) + @specs.method + def not_null(value): if isinstance(value, TypeScheme.ObjRef): return value @@ -81,29 +83,34 @@ class TypeScheme(object): 'null value violates notNull() contract') return value - def _error(): + @specs.parameter('value', nullable=True) + @specs.method + def error(value): raise exceptions.ContractViolationException('error() contract') - def _check(value, predicate): - value = value() - if isinstance(value, TypeScheme.ObjRef) or predicate(value): + @specs.parameter('value', nullable=True) + @specs.parameter('predicate', yaqltypes.Lambda(with_context=True)) + @specs.method + def check(value, predicate): + if isinstance(value, TypeScheme.ObjRef) or predicate( + root_context.create_child_context(), value): return value else: raise exceptions.ContractViolationException( "Value {0} doesn't match predicate".format(value)) - @yaql.context.EvalArg('obj', arg_type=( - murano.dsl.murano_object.MuranoObject, - TypeScheme.ObjRef, - types.NoneType - )) - def _owned(obj): + @specs.parameter('obj', TypeScheme.ObjRef, nullable=True) + @specs.name('owned') + @specs.method + def owned_ref(obj): + if obj is None: + return None if isinstance(obj, TypeScheme.ObjRef): return obj - if obj is None: - return None - + @specs.parameter('obj', dsl_types.MuranoObject) + @specs.method + def owned(obj): p = obj.owner while p is not None: if p is this: @@ -114,20 +121,21 @@ class TypeScheme(object): 'Object {0} violates owned() contract'.format( obj.object_id)) - @yaql.context.EvalArg('obj', arg_type=( - murano.dsl.murano_object.MuranoObject, - TypeScheme.ObjRef, - types.NoneType - )) - def _not_owned(obj): + @specs.parameter('obj', TypeScheme.ObjRef, nullable=True) + @specs.name('not_owned') + @specs.method + def not_owned_ref(obj): if isinstance(obj, TypeScheme.ObjRef): return obj if obj is None: return None + @specs.parameter('obj', dsl_types.MuranoObject) + @specs.method + def not_owned(obj): try: - _owned(obj) + owned(obj) except exceptions.ContractViolationException: return obj else: @@ -135,39 +143,34 @@ class TypeScheme(object): 'Object {0} violates notOwned() contract'.format( obj.object_id)) - @yaql.context.EvalArg('name', arg_type=str) - def _class(value, name): - return _class2(value, name, None) - - @yaql.context.EvalArg('name', arg_type=str) - @yaql.context.EvalArg('default_name', arg_type=(str, types.NoneType)) - def _class2(value, name, default_name): - name = namespace_resolver.resolve_name(name) + @specs.parameter('name', dsl.MuranoTypeName( + False, root_context)) + @specs.parameter('default_name', dsl.MuranoTypeName( + True, root_context)) + @specs.parameter('value', nullable=True) + @specs.method + def class_(value, name, default_name=None): + object_store = helpers.get_object_store(root_context) if not default_name: default_name = name - else: - default_name = namespace_resolver.resolve_name(default_name) - value = value() - class_loader = murano.dsl.helpers.get_class_loader(root_context) - murano_class = class_loader.get_class(name) + murano_class = name.murano_class if not murano_class: raise exceptions.NoClassFound( 'Class {0} cannot be found'.format(name)) if value is None: return None - if isinstance(value, murano.dsl.murano_object.MuranoObject): + if isinstance(value, dsl_types.MuranoObject): obj = value - elif isinstance(value, types.DictionaryType): + elif isinstance(value, utils.MappingType): if '?' not in value: new_value = {'?': { 'id': uuid.uuid4().hex, - 'type': default_name + 'type': default_name.murano_class.name }} new_value.update(value) value = new_value - obj = object_store.load(value, owner, root_context, - defaults=default) + obj = object_store.load(value, owner, defaults=default) elif isinstance(value, types.StringTypes): obj = object_store.get(value) if obj is None: @@ -185,30 +188,24 @@ class TypeScheme(object): 'requested type {1}'.format(obj.type.name, name)) return obj - @yaql.context.EvalArg('prefix', str) - @yaql.context.EvalArg('name', str) - def _validate(prefix, name): - return namespace_resolver.resolve_name( - '%s:%s' % (prefix, name)) - - context = yaql.context.Context(parent_context=root_context) - context.register_function(_validate, '#validate') - context.register_function(_int, 'int') - context.register_function(_string, 'string') - context.register_function(_bool, 'bool') - context.register_function(_check, 'check') - context.register_function(_not_null, 'notNull') - context.register_function(_error, 'error') - context.register_function(_class, 'class') - context.register_function(_class2, 'class') - context.register_function(_owned, 'owned') - context.register_function(_not_owned, 'notOwned') + context = yaql_integration.create_context() + context.register_function(int_) + context.register_function(string) + context.register_function(bool_) + context.register_function(check) + context.register_function(not_null) + context.register_function(error) + context.register_function(class_) + context.register_function(owned_ref) + context.register_function(owned) + context.register_function(not_owned_ref) + context.register_function(not_owned) return context def _map_dict(self, data, spec, context): - if data is None or data is NoValue: + if data is None or data is dsl.NO_VALUE: data = {} - if not isinstance(data, types.DictionaryType): + if not isinstance(data, utils.MappingType): raise exceptions.ContractViolationException( 'Supplied is not of a dictionary type') if not spec: @@ -216,7 +213,7 @@ class TypeScheme(object): result = {} yaql_key = None for key, value in spec.iteritems(): - if isinstance(key, yaql_expression.YaqlExpression): + if isinstance(key, dsl_types.YaqlExpression): if yaql_key is not None: raise exceptions.DslContractSyntaxError( 'Dictionary contract ' @@ -231,20 +228,19 @@ class TypeScheme(object): for key, value in data.iteritems(): if key in result: continue - result[self._map(key, yaql_key, context)] = \ - self._map(value, yaql_value, context) + result[self._map(key, yaql_key, context)] = self._map( + value, yaql_value, context) - return result + return utils.FrozenDict(result) def _map_list(self, data, spec, context): - if not isinstance(data, types.ListType): - if data is None or data is NoValue: + if not utils.is_sequence(data): + if data is None or data is dsl.NO_VALUE: data = [] else: data = [data] if len(spec) < 1: return data - result = [] shift = 0 max_length = sys.maxint min_length = 0 @@ -261,11 +257,16 @@ class TypeScheme(object): 'Array length {0} is not within [{1}..{2}] range'.format( len(data), min_length, max_length)) - for index, item in enumerate(data): - spec_item = spec[-1 - shift] \ - if index >= len(spec) - shift else spec[index] - result.append(self._map(item, spec_item, context)) - return result + def map_func(): + for index, item in enumerate(data): + spec_item = ( + spec[-1 - shift] + if index >= len(spec) - shift + else spec[index] + ) + yield self._map(item, spec_item, context) + + return tuple(map_func()) def _map_scalar(self, data, spec): if data != spec: @@ -275,28 +276,24 @@ class TypeScheme(object): return data def _map(self, data, spec, context): - child_context = yaql.context.Context(parent_context=context) - if isinstance(spec, yaql_expression.YaqlExpression): - child_context.set_data(data) - return spec.evaluate(context=child_context) - elif isinstance(spec, types.DictionaryType): + child_context = context.create_child_context() + if isinstance(spec, dsl_types.YaqlExpression): + child_context[''] = data + return spec(context=child_context) + elif isinstance(spec, utils.MappingType): return self._map_dict(data, spec, child_context) - elif isinstance(spec, types.ListType): + elif utils.is_sequence(spec): return self._map_list(data, spec, child_context) - elif isinstance(spec, (types.IntType, - types.StringTypes, - types.NoneType)): + else: return self._map_scalar(data, spec) - def __call__(self, data, context, this, owner, object_store, - namespace_resolver, default): + def __call__(self, data, context, this, owner, default): # TODO(ativelkov, slagun): temporary fix, need a better way of handling # composite defaults # A bug (#1313694) has been filed - if data is NoValue: - data = default + if data is dsl.NO_VALUE: + data = helpers.evaluate(default, context) - context = self.prepare_context( - context, this, owner, object_store, namespace_resolver, default) + context = self.prepare_context(context, this, owner, default) return self._map(data, self._spec, context) diff --git a/murano/dsl/typespec.py b/murano/dsl/typespec.py index 5fcebbe4d..bbef3d7d0 100644 --- a/murano/dsl/typespec.py +++ b/murano/dsl/typespec.py @@ -13,7 +13,7 @@ # under the License. from murano.dsl import exceptions -import murano.dsl.type_scheme as type_scheme +from murano.dsl import type_scheme class PropertyUsages(object): @@ -28,8 +28,7 @@ class PropertyUsages(object): class Spec(object): - def __init__(self, declaration, owner_class): - self._namespace_resolver = owner_class.namespace_resolver + def __init__(self, declaration): self._contract = type_scheme.TypeScheme(declaration['Contract']) self._usage = declaration.get('Usage') or 'In' self._default = declaration.get('Default') @@ -39,12 +38,10 @@ class Spec(object): 'Unknown type {0}. Must be one of ({1})'.format( self._usage, ', '.join(PropertyUsages.All))) - def validate(self, value, this, owner, context, - object_store, default=None): + def validate(self, value, context, this, owner, default=None): if default is None: default = self.default - return self._contract(value, context, this, owner, object_store, - self._namespace_resolver, default) + return self._contract(value, context, this, owner, default) @property def default(self): diff --git a/murano/dsl/virtual_exceptions.py b/murano/dsl/virtual_exceptions.py index e079b9b04..a2323cb40 100644 --- a/murano/dsl/virtual_exceptions.py +++ b/murano/dsl/virtual_exceptions.py @@ -12,11 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -import murano.dsl.dsl_exception as dsl_exception -import murano.dsl.expressions as expressions -import murano.dsl.helpers as helpers -import murano.dsl.macros as macros -import murano.dsl.yaql_functions as yaql_functions +from murano.dsl import constants +from murano.dsl import dsl_exception +from murano.dsl import expressions +from murano.dsl import helpers +from murano.dsl import macros +from murano.dsl.principal_objects import stack_trace +from murano.dsl import yaql_integration class ThrowMacro(expressions.DslExpression): @@ -36,9 +38,8 @@ class ThrowMacro(expressions.DslExpression): for name in names: yield murano_class.namespace_resolver.resolve_name(name) - def execute(self, context, murano_class): - stacktrace = yaql_functions.new('io.murano.StackTrace', context, - includeNativeFrames=False) + def execute(self, context): + stacktrace = stack_trace.create_stack_trace(context, False) cause = None if self._cause: cause = helpers.evaluate(self._cause, context).get_property( @@ -68,10 +69,12 @@ class CatchBlock(expressions.DslExpression): for name in names: yield murano_class.namespace_resolver.resolve_name(name) - def execute(self, context, murano_class): + def execute(self, context): exception = helpers.get_current_exception(context) - names = None if self._with is None else \ - list(self._resolve_names(self._with, context)) + names = ( + None if self._with is None + else list(self._resolve_names(self._with, context)) + ) for name in exception.names: if self._with is None or name in names: @@ -79,13 +82,13 @@ class CatchBlock(expressions.DslExpression): if self._as: wrapped = self._wrap_internal_exception( exception, context, name) - context.set_data(wrapped, self._as) - self._code_block.execute(context, murano_class) + context[self._as] = wrapped + self._code_block.execute(context) return True return False def _wrap_internal_exception(self, exception, context, name): - obj = yaql_functions.new('io.murano.Exception', context) + obj = yaql_integration.call_func(context, 'new', 'io.murano.Exception') obj.set_property('name', name) obj.set_property('message', exception.message) obj.set_property('stackTrace', exception.stacktrace) @@ -102,43 +105,45 @@ class TryBlockMacro(expressions.DslExpression): if not isinstance(Catch, list): Catch = [Catch] self._catch_block = [CatchBlock(**c) for c in Catch] - self._finally_block = None if Finally is None \ - else macros.CodeBlock(Finally) - self._else_block = None if Else is None \ - else macros.CodeBlock(Else) + self._finally_block = ( + None if Finally is None + else macros.CodeBlock(Finally)) + self._else_block = ( + None if Else is None + else macros.CodeBlock(Else)) - def execute(self, context, murano_class): + def execute(self, context): try: - self._try_block.execute(context, murano_class) + self._try_block.execute(context) except dsl_exception.MuranoPlException as e: caught = False if self._catch_block: try: - context.set_data(e, '?currentException') + context[constants.CTX_CURRENT_EXCEPTION] = e for cb in self._catch_block: - if cb.execute(context, murano_class): + if cb.execute(context): caught = True break if not caught: raise finally: - context.set_data(None, '?currentException') + context[constants.CTX_CURRENT_EXCEPTION] = None else: raise else: if self._else_block: - self._else_block.execute(context, murano_class) + self._else_block.execute(context) finally: if self._finally_block: - self._finally_block.execute(context, murano_class) + self._finally_block.execute(context) class RethrowMacro(expressions.DslExpression): def __init__(self, Rethrow): pass - def execute(self, context, murano_class): - exception = context.get_data('$?currentException') + def execute(self, context): + exception = context['$?currentException'] if not exception: raise TypeError('Rethrow must be inside Catch') raise exception diff --git a/murano/dsl/yaql_expression.py b/murano/dsl/yaql_expression.py index fa5e180ea..03f5007a8 100644 --- a/murano/dsl/yaql_expression.py +++ b/murano/dsl/yaql_expression.py @@ -16,23 +16,26 @@ import re import types from oslo_utils import encodeutils -import yaql -import yaql.exceptions -import yaql.expressions +from yaql.language import exceptions as yaql_exceptions +from yaql.language import expressions + +from murano.dsl import constants +from murano.dsl import dsl_types +from murano.dsl import yaql_integration -class YaqlExpression(object): +class YaqlExpression(dsl_types.YaqlExpression): def __init__(self, expression): if isinstance(expression, types.StringTypes): self._expression = encodeutils.safe_encode(expression) - self._parsed_expression = yaql.parse(self._expression) + self._parsed_expression = yaql_integration.parse(self._expression) self._file_position = None elif isinstance(expression, YaqlExpression): self._expression = expression._expression self._parsed_expression = expression._parsed_expression self._file_position = expression._file_position - elif isinstance(expression, yaql.expressions.Expression): - self._expression = str(expression) + elif isinstance(expression, expressions.Statement): + self._expression = unicode(expression) self._parsed_expression = expression self._file_position = None else: @@ -63,52 +66,12 @@ class YaqlExpression(object): if re.match('^[\s\w\d.:]*$', expr): return False try: - yaql.parse(expr) + yaql_integration.parse(expr) return True - except yaql.exceptions.YaqlGrammarException: - return False - except yaql.exceptions.YaqlLexicalException: + except yaql_exceptions.YaqlParsingException: return False - def evaluate(self, context=None): + def __call__(self, context): + if context: + context[constants.CTX_CURRENT_INSTRUCTION] = self return self._parsed_expression.evaluate(context=context) - - -class YaqlExpressionFilePosition(object): - def __init__(self, file_path, start_line, start_column, start_index, - end_line, end_column, length): - self._file_path = file_path - self._start_line = start_line - self._start_column = start_column - self._start_index = start_index - self._end_line = end_line - self._end_column = end_column - self._length = length - - @property - def file_path(self): - return self._file_path - - @property - def start_line(self): - return self._start_line - - @property - def start_column(self): - return self._start_column - - @property - def start_index(self): - return self._start_index - - @property - def end_line(self): - return self._end_line - - @property - def end_column(self): - return self._end_column - - @property - def length(self): - return self._length diff --git a/murano/dsl/yaql_functions.py b/murano/dsl/yaql_functions.py index 29971952f..28db06362 100644 --- a/murano/dsl/yaql_functions.py +++ b/murano/dsl/yaql_functions.py @@ -13,144 +13,169 @@ # under the License. import itertools -import types import eventlet -import yaql.context -import yaql.exceptions +from yaql.language import specs +from yaql.language import utils +from yaql.language import yaqltypes -import murano.dsl.exceptions as exceptions -import murano.dsl.helpers as helpers -import murano.dsl.murano_object as murano_object +from murano.dsl import dsl +from murano.dsl import dsl_types +from murano.dsl import helpers +from murano.dsl import yaql_integration -def _resolve(name, obj): - @yaql.context.EvalArg('this', murano_object.MuranoObject) - @yaql.context.ContextAware() - def invoke(context, this, *args): - try: - executor = helpers.get_executor(context) - murano_class = helpers.get_type(context) - return executor.invoke_method(name, this, context, - murano_class, *args) - except exceptions.NoMethodFound: - raise yaql.exceptions.YaqlExecutionException() - except exceptions.AmbiguousMethodName: - raise yaql.exceptions.YaqlExecutionException() - - if not isinstance(obj, murano_object.MuranoObject): - return None - - return invoke - - -@yaql.context.EvalArg('value', murano_object.MuranoObject) -def _id(value): +@specs.parameter('value', dsl_types.MuranoObject) +@specs.extension_method +def id_(value): return value.object_id -@yaql.context.EvalArg('value', murano_object.MuranoObject) -@yaql.context.EvalArg('type', str) -@yaql.context.ContextAware() -def _cast(context, value, type): - if '.' not in type: - murano_class = helpers.get_type(context) - type = murano_class.namespace_resolver.resolve_name(type) - class_loader = helpers.get_class_loader(context) - return value.cast(class_loader.get_class(type)) +@specs.parameter('value', dsl_types.MuranoObject) +@specs.parameter('type', dsl.MuranoTypeName()) +@specs.extension_method +def cast(value, type): + return value.cast(type.murano_class) -@yaql.context.EvalArg('name', str) -@yaql.context.ContextAware() -def _new(context, name, *args): - murano_class = helpers.get_type(context) - name = murano_class.namespace_resolver.resolve_name(name) - parameters = {} - arg_values = [t() for t in args] - if len(arg_values) == 1 and isinstance( - arg_values[0], types.DictionaryType): - parameters = arg_values[0] - elif len(arg_values) > 0: - for p in arg_values: - if not isinstance(p, types.TupleType) or \ - not isinstance(p[0], types.StringType): - raise SyntaxError() - parameters[p[0]] = p[1] - - object_store = helpers.get_object_store(context) - class_loader = helpers.get_class_loader(context) - new_context = yaql.context.Context(parent_context=context) +@specs.parameter('__type_name', dsl.MuranoTypeName()) +@specs.parameter('__extra', utils.MappingType) +@specs.parameter('__owner', dsl_types.MuranoObject) +@specs.parameter('__object_name', yaqltypes.String(True)) +def new(__context, __type_name, __owner=None, __object_name=None, __extra=None, + **parameters): + object_store = helpers.get_object_store(__context) + new_context = __context.create_child_context() for key, value in parameters.iteritems(): - new_context.set_data(value, key) - return class_loader.get_class(name).new( - None, object_store, new_context, parameters=parameters) + if helpers.is_keyword(key): + new_context[key] = value + return __type_name.murano_class.new( + __owner, object_store, new_context, name=__object_name)(**parameters) -def new(name, context, **kwargs): - return _new(context, name, lambda: kwargs) +@specs.parameter('type_name', dsl.MuranoTypeName()) +@specs.parameter('parameters', utils.MappingType) +@specs.parameter('extra', utils.MappingType) +@specs.parameter('owner', dsl_types.MuranoObject) +@specs.parameter('object_name', yaqltypes.String(True)) +@specs.name('new') +def new_from_dict(type_name, context, parameters, + owner=None, object_name=None, extra=None): + return new(context, type_name, owner, object_name, extra, + **yaql_integration.filter_parameters_dict(parameters)) -@yaql.context.EvalArg('value', murano_object.MuranoObject) -@yaql.context.ContextAware() -def _super(context, value): +@specs.parameter('value', dsl_types.MuranoObject) +@specs.parameter('func', yaqltypes.Lambda()) +@specs.extension_method +def super_(context, value, func=None): cast_type = helpers.get_type(context) - return [value.cast(type) for type in cast_type.parents] + if func is None: + return [value.cast(type) for type in cast_type.parents] + return itertools.imap(func, super_(context, value)) -@yaql.context.EvalArg('value', murano_object.MuranoObject) -@yaql.context.ContextAware() -def _super2(context, value, func): - return itertools.imap(func, _super(context, value)) +@specs.parameter('value', dsl_types.MuranoObject) +@specs.parameter('func', yaqltypes.Lambda()) +@specs.extension_method +def psuper(context, value, func=None): + if func is None: + return super_(context, value) + return helpers.parallel_select(super_(context, value), func) -@yaql.context.EvalArg('value', murano_object.MuranoObject) -@yaql.context.ContextAware() -def _psuper2(context, value, func): - helpers.parallel_select(_super(context, value), func) - - -@yaql.context.EvalArg('value', object) -def _require(value): +@specs.extension_method +def require(value): if value is None: - raise ValueError() + raise ValueError('Required value is missing') return value -@yaql.context.EvalArg('obj', murano_object.MuranoObject) -@yaql.context.EvalArg('class_name', str) -@yaql.context.ContextAware() -def _get_container(context, obj, class_name): - namespace_resolver = helpers.get_type(context).namespace_resolver - class_loader = helpers.get_class_loader(context) - class_name = namespace_resolver.resolve_name(class_name) - murano_class = class_loader.get_class(class_name) +@specs.parameter('obj', dsl_types.MuranoObject) +@specs.parameter('murano_class_ref', dsl.MuranoTypeName()) +@specs.extension_method +def find(obj, murano_class_ref): p = obj.owner while p is not None: - if murano_class.is_compatible(p): + if murano_class_ref.murano_class.is_compatible(p): return p p = p.owner return None -@yaql.context.EvalArg('seconds', (int, float)) -def _sleep(seconds): +@specs.parameter('seconds', yaqltypes.Number()) +def sleep_(seconds): eventlet.sleep(seconds) -@yaql.context.EvalArg('value', murano_object.MuranoObject) -def _type(value): - return value.type.name +@specs.parameter('object_', dsl_types.MuranoObject) +@specs.extension_method +def type_(object_): + return object_.type.name + + +@specs.parameter('object_', dsl_types.MuranoObject) +def name(object_): + return object_.name + + +@specs.parameter('obj', dsl_types.MuranoObject) +@specs.parameter('property_name', yaqltypes.Keyword()) +@specs.name('#operator_.') +def obj_attribution(context, obj, property_name): + return obj.get_property(property_name, context) + + +@specs.parameter('sender', dsl_types.MuranoObject) +@specs.parameter('expr', yaqltypes.Lambda(method=True)) +@specs.inject('operator', yaqltypes.Super(with_context=True)) +@specs.name('#operator_.') +def op_dot(context, sender, expr, operator): + ctx2 = context.create_child_context() + sender.type.register_methods(ctx2) + return operator(ctx2, sender, expr) + + +@specs.parameter('prefix', yaqltypes.Keyword()) +@specs.parameter('name', yaqltypes.Keyword()) +@specs.name('#operator_:') +def ns_resolve(context, prefix, name): + murano_type = helpers.get_type(context) + class_loader = helpers.get_class_loader(context) + return dsl_types.MuranoClassReference( + class_loader.get_class( + murano_type.namespace_resolver.resolve_name( + prefix + ':' + name))) + + +@specs.parameter('obj1', dsl_types.MuranoObject, nullable=True) +@specs.parameter('obj2', dsl_types.MuranoObject, nullable=True) +@specs.name('*equal') +def equal(obj1, obj2): + return obj1 is obj2 + + +@specs.parameter('obj1', dsl_types.MuranoObject, nullable=True) +@specs.parameter('obj2', dsl_types.MuranoObject, nullable=True) +@specs.name('*not_equal') +def not_equal(obj1, obj2): + return obj1 is not obj2 def register(context): - context.register_function(_resolve, '#resolve') - context.register_function(_cast, 'cast') - context.register_function(_new, 'new') - context.register_function(_id, 'id') - context.register_function(_super2, 'super') - context.register_function(_psuper2, 'psuper') - context.register_function(_super, 'super') - context.register_function(_require, 'require') - context.register_function(_get_container, 'find') - context.register_function(_sleep, 'sleep') - context.register_function(_type, 'type') + context.register_function(cast) + context.register_function(new) + context.register_function(new_from_dict) + context.register_function(id_) + context.register_function(super_) + context.register_function(psuper) + context.register_function(require) + context.register_function(find) + context.register_function(sleep_) + context.register_function(type_) + context.register_function(name) + context.register_function(obj_attribution) + context.register_function(op_dot) + context.register_function(ns_resolve) + context.register_function(equal) + context.register_function(not_equal) diff --git a/murano/dsl/yaql_integration.py b/murano/dsl/yaql_integration.py new file mode 100644 index 000000000..e5d0ab2d4 --- /dev/null +++ b/murano/dsl/yaql_integration.py @@ -0,0 +1,245 @@ +# Copyright (c) 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import inspect + +from yaql.language import contexts +from yaql.language import conventions +from yaql.language import factory +from yaql.language import specs +from yaql.language import yaqltypes +from yaql import legacy + +from murano.dsl import constants +from murano.dsl import dsl +from murano.dsl import dsl_types +from murano.dsl import helpers + + +LEGACY_ENGINE_OPTIONS = { + 'yaql.limitIterators': constants.ITERATORS_LIMIT, + 'yaql.memoryQuota': constants.EXPRESSION_MEMORY_QUOTA, + 'yaql.iterableDicts': True +} + + +def _create_engine(): + engine_factory = factory.YaqlFactory() + engine_factory.insert_operator( + '.', True, ':', factory.OperatorType.BINARY_LEFT_ASSOCIATIVE, True) + return engine_factory.create(options=LEGACY_ENGINE_OPTIONS) + + +@specs.name('#finalize') +def _finalize(obj, context): + return helpers.evaluate(obj, context) + + +ENGINE = _create_engine() +CONVENTION = conventions.CamelCaseConvention() +ROOT_CONTEXT = legacy.create_context( + convention=CONVENTION, finalizer=_finalize) + + +class ContractedValue(yaqltypes.GenericType): + def __init__(self, value_spec): + self._value_spec = value_spec + self._last_result = False + + super(ContractedValue, self).__init__( + True, None, + lambda value, sender, context, *args, **kwargs: + self._value_spec.validate( + value, sender.context, helpers.get_this(context), + context[constants.CTX_ARGUMENT_OWNER])) + + def convert(self, value, *args, **kwargs): + if value is None: + return self.converter(value, *args, **kwargs) + return super(ContractedValue, self).convert(value, *args, **kwargs) + + +def create_empty_context(): + context = contexts.Context() + context.register_function(_finalize) + return context + + +def create_context(): + return ROOT_CONTEXT.create_child_context() + + +def parse(expression): + return ENGINE(expression) + + +def call_func(__context, __name, *args, **kwargs): + return __context(__name, ENGINE)(*args, **kwargs) + + +def _infer_parameter_type(name, class_name): + if name == 'context': + return yaqltypes.Context() + if name == 'this': + return dsl.ThisParameterType() + if name == 'interfaces': + return dsl.InterfacesParameterType() + if name == 'yaql_engine': + return yaqltypes.Engine() + + if name.startswith('__'): + return _infer_parameter_type(name[2:], class_name) + if class_name and name.startswith('_{0}__'.format(class_name)): + return _infer_parameter_type(name[3 + len(class_name):], class_name) + + +def get_function_definition(func): + body = func + param_type_func = lambda name: _infer_parameter_type(name, None) + is_method = False + if inspect.ismethod(func): + is_method = True + body = func.im_func + param_type_func = lambda name: _infer_parameter_type( + name, func.im_class.__name__) + fd = specs.get_function_definition( + body, convention=CONVENTION, + parameter_type_func=param_type_func) + if is_method: + fd.is_method = True + fd.is_function = False + fd.set_parameter( + 0, + yaqltypes.PythonType(func.im_class), + overwrite=True) + name = getattr(func, '__murano_name', None) + if name: + fd.name = name + fd.insert_parameter(specs.ParameterDefinition( + '?1', yaqltypes.Context(), 0)) + + def payload(__context, *args, **kwargs): + with helpers.contextual(__context): + return body(*args, **kwargs) + + fd.payload = payload + return fd + + +def build_wrapper_function_definition(murano_method): + if isinstance(murano_method.body, specs.FunctionDefinition): + return _build_native_wrapper_function_definition(murano_method) + else: + return _build_mpl_wrapper_function_definition(murano_method) + + +def _build_native_wrapper_function_definition(murano_method): + def payload(__context, __sender, *args, **kwargs): + executor = helpers.get_executor(__context) + args = tuple(to_mutable(arg) for arg in args) + kwargs = to_mutable(kwargs) + return murano_method.invoke( + executor, __sender, args[1:], kwargs, __context, True) + + fd = murano_method.body.strip_hidden_parameters() + fd.payload = payload + for v in fd.parameters.itervalues(): + if v.position == 0: + v.value_type = yaqltypes.PythonType(dsl_types.MuranoObject, False) + break + fd.insert_parameter(specs.ParameterDefinition( + '?1', yaqltypes.Context(), 0)) + fd.insert_parameter(specs.ParameterDefinition( + '?2', yaqltypes.Sender(), 1)) + return fd + + +def _build_mpl_wrapper_function_definition(murano_method): + def payload(__context, __sender, *args, **kwargs): + executor = helpers.get_executor(__context) + return murano_method.invoke( + executor, __sender.object, args, kwargs, __context, True) + + fd = specs.FunctionDefinition( + murano_method.name, payload, is_function=False, is_method=True) + + for i, (name, arg_spec) in enumerate( + murano_method.arguments_scheme.iteritems(), 2): + p = specs.ParameterDefinition( + name, ContractedValue(arg_spec), + position=i, default=dsl.NO_VALUE) + fd.parameters[name] = p + + fd.set_parameter(specs.ParameterDefinition( + '__context', yaqltypes.Context(), 0)) + + fd.set_parameter(specs.ParameterDefinition( + '__sender', dsl.MuranoObjectType(murano_method.murano_class), 1)) + + return fd + + +def get_class_factory_definition(cls): + def payload(__context, __sender, *args, **kwargs): + assert __sender is None + args = tuple(to_mutable(arg) for arg in args) + kwargs = to_mutable(kwargs) + with helpers.contextual(__context): + return cls(*args, **kwargs) + + if hasattr(cls.__init__, 'im_func'): + fd = specs.get_function_definition( + cls.__init__.im_func, + parameter_type_func=lambda name: _infer_parameter_type( + name, cls.__init__.im_class.__name__)) + else: + fd = specs.get_function_definition(lambda self: None) + fd.meta[constants.META_NO_TRACE] = True + fd.insert_parameter(specs.ParameterDefinition( + '?1', yaqltypes.Context(), position=0)) + fd.is_method = True + fd.is_function = False + fd.name = '__init__' + fd.payload = payload + return fd + + +def filter_parameters(__fd, *args, **kwargs): + if '*' not in __fd.parameters: + position_args = 0 + for p in __fd.parameters.itervalues(): + if p.position is not None: + position_args += 1 + args = args[:position_args] + kwargs = kwargs.copy() + for name in kwargs.keys(): + if not helpers.is_keyword(name): + del kwargs[name] + if '**' not in __fd.parameters: + for name in kwargs.keys(): + if name not in __fd.parameters: + del kwargs[name] + return args, kwargs + + +def filter_parameters_dict(parameters): + parameters = parameters.copy() + for name in parameters.keys(): + if not helpers.is_keyword(name): + del parameters[name] + return parameters + + +def to_mutable(obj): + return dsl.to_mutable(obj, ENGINE) diff --git a/murano/engine/client_manager.py b/murano/engine/client_manager.py index 1d3bc3bec..fbd19a678 100644 --- a/murano/engine/client_manager.py +++ b/murano/engine/client_manager.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import weakref + from eventlet import semaphore import heatclient.client as hclient import keystoneclient @@ -21,8 +23,6 @@ import neutronclient.v2_0.client as nclient from oslo_config import cfg from murano.common import auth_utils -from murano.dsl import helpers -from murano.engine import environment try: @@ -39,23 +39,19 @@ CONF = cfg.CONF class ClientManager(object): - def __init__(self): + def __init__(self, environment): self._trusts_keystone_client = None self._token_keystone_client = None self._cache = {} self._semaphore = semaphore.BoundedSemaphore() + self._environment = weakref.proxy(environment) - def _get_environment(self, context): - if isinstance(context, environment.Environment): - return context - return helpers.get_environment(context) - - def get_client(self, context, name, use_trusts, client_factory): + def get_client(self, name, use_trusts, client_factory): if not CONF.engine.use_trusts: use_trusts = False keystone_client = None if name == 'keystone' else \ - self.get_keystone_client(context, use_trusts) + self.get_keystone_client(use_trusts) self._semaphore.acquire() try: @@ -68,25 +64,24 @@ class ClientManager(object): if not client: token = fresh_token if not use_trusts: - env = self._get_environment(context) - token = env.token + token = self._environment.token client = client_factory(keystone_client, token) self._cache[(name, use_trusts)] = (client, token) return client finally: self._semaphore.release() - def get_keystone_client(self, context, use_trusts=True): + def get_keystone_client(self, use_trusts=True): if not CONF.engine.use_trusts: use_trusts = False - env = self._get_environment(context) factory = lambda _1, _2: \ - auth_utils.get_client_for_trusts(env.trust_id) \ - if use_trusts else auth_utils.get_client(env.token, env.tenant_id) + auth_utils.get_client_for_trusts(self._environment.trust_id) \ + if use_trusts else auth_utils.get_client( + self._environment.token, self._environment.tenant_id) - return self.get_client(context, 'keystone', use_trusts, factory) + return self.get_client('keystone', use_trusts, factory) - def get_congress_client(self, context, use_trusts=True): + def get_congress_client(self, use_trusts=True): """Client for congress services :return: initialized congress client @@ -106,9 +101,9 @@ class ClientManager(object): return congress_client.Client(session=session, service_type='policy') - return self.get_client(context, 'congress', use_trusts, factory) + return self.get_client('congress', use_trusts, factory) - def get_heat_client(self, context, use_trusts=True): + def get_heat_client(self, use_trusts=True): if not CONF.engine.use_trusts: use_trusts = False @@ -134,9 +129,9 @@ class ClientManager(object): }) return hclient.Client('1', heat_url, **kwargs) - return self.get_client(context, 'heat', use_trusts, factory) + return self.get_client('heat', use_trusts, factory) - def get_neutron_client(self, context, use_trusts=True): + def get_neutron_client(self, use_trusts=True): if not CONF.engine.use_trusts: use_trusts = False @@ -153,9 +148,9 @@ class ClientManager(object): ca_cert=neutron_settings.ca_cert or None, insecure=neutron_settings.insecure) - return self.get_client(context, 'neutron', use_trusts, factory) + return self.get_client('neutron', use_trusts, factory) - def get_murano_client(self, context, use_trusts=True): + def get_murano_client(self, use_trusts=True): if not CONF.engine.use_trusts: use_trusts = False @@ -176,9 +171,9 @@ class ClientManager(object): auth_url=keystone_client.auth_url, token=auth_token) - return self.get_client(context, 'murano', use_trusts, factory) + return self.get_client('murano', use_trusts, factory) - def get_mistral_client(self, context, use_trusts=True): + def get_mistral_client(self, use_trusts=True): if not mistralclient: raise mistral_import_error @@ -202,4 +197,4 @@ class ClientManager(object): auth_token=auth_token, user_id=keystone_client.user_id) - return self.get_client(context, 'mistral', use_trusts, factory) + return self.get_client('mistral', use_trusts, factory) diff --git a/murano/engine/environment.py b/murano/engine/environment.py index 467595809..66f251171 100644 --- a/murano/engine/environment.py +++ b/murano/engine/environment.py @@ -16,7 +16,7 @@ from oslo_log import log as logging from murano.common.i18n import _LE - +from murano.engine import client_manager LOG = logging.getLogger(__name__) @@ -27,7 +27,7 @@ class Environment(object): self.tenant_id = None self.trust_id = None self.system_attributes = {} - self.clients = None + self.clients = client_manager.ClientManager(self) self._set_up_list = [] self._tear_down_list = [] diff --git a/murano/engine/package_loader.py b/murano/engine/package_loader.py index c7a67dd84..1af76b6dd 100644 --- a/murano/engine/package_loader.py +++ b/murano/engine/package_loader.py @@ -92,7 +92,7 @@ class ApiPackageLoader(PackageLoader): 'more then 1 package found for query "{0}", ' 'will resolve based on the ownership'. format(filter_opts)) - return self._get_best_package_match(packages, self.tenant_id) + return self._get_best_package_match(packages) elif len(packages) == 1: return packages[0] else: diff --git a/murano/engine/system/agent.py b/murano/engine/system/agent.py index 01934a297..706468cb0 100644 --- a/murano/engine/system/agent.py +++ b/murano/engine/system/agent.py @@ -23,12 +23,11 @@ import uuid import eventlet.event from oslo_config import cfg from oslo_log import log as logging +from yaql import specs import murano.common.exceptions as exceptions import murano.common.messaging as messaging -import murano.dsl.murano_class as murano_class -import murano.dsl.murano_object as murano_object -import murano.dsl.yaql_expression as yaql_expression +from murano.dsl import dsl import murano.engine.system.common as common LOG = logging.getLogger(__name__) @@ -39,24 +38,23 @@ class AgentException(Exception): pass -@murano_class.classname('io.murano.system.Agent') -class Agent(murano_object.MuranoObject): - def initialize(self, _context, host): +@dsl.name('io.murano.system.Agent') +class Agent(object): + def __init__(self, interfaces, host): self._enabled = False if CONF.engine.disable_murano_agent: LOG.debug('Use of murano-agent is disallowed ' 'by the server configuration') return - self._environment = self._get_environment(_context) + self._environment = self._get_environment(interfaces, host) self._enabled = True self._queue = str('e%s-h%s' % ( - self._environment.object_id, host.object_id)).lower() + self._environment.id, host.id)).lower() - def _get_environment(self, _context): - return yaql_expression.YaqlExpression( - "$host.find('io.murano.Environment').require()" - ).evaluate(_context) + def _get_environment(self, interfaces, host): + return interfaces.yaql()( + "$.find('io.murano.Environment').require()", host) @property def enabled(self): @@ -72,7 +70,7 @@ class Agent(murano_object.MuranoObject): with common.create_rmq_client() as client: client.declare(self._queue, enable_ha=True, ttl=86400000) - def queueName(self): + def queue_name(self): return self._queue def _check_enabled(self): @@ -87,13 +85,13 @@ class Agent(murano_object.MuranoObject): msg.id = msg_id return msg - def _send(self, template, wait_results, timeout, _context): + def _send(self, template, wait_results, timeout): """Send a message over the MQ interface.""" msg_id = template.get('ID', uuid.uuid4().hex) if wait_results: event = eventlet.event.Event() - listener = self._environment.agentListener - listener.subscribe(msg_id, event, _context) + listener = self._environment['agentListener'] + listener().subscribe(msg_id, event) msg = self._prepare_message(template, msg_id) with common.create_rmq_client() as client: @@ -105,7 +103,7 @@ class Agent(murano_object.MuranoObject): result = event.wait() except eventlet.Timeout: - listener.unsubscribe(msg_id) + listener().unsubscribe(msg_id) raise exceptions.TimeoutException( 'The Agent does not respond' 'within {0} seconds'.format(timeout)) @@ -122,40 +120,44 @@ class Agent(murano_object.MuranoObject): else: return None - def call(self, template, resources, _context, timeout=None): + @specs.parameter( + 'resources', dsl.MuranoObjectType('io.murano.system.Resources')) + def call(self, template, resources, timeout=None): if timeout is None: timeout = CONF.engine.agent_timeout self._check_enabled() - plan = self.buildExecutionPlan(template, resources) - return self._send(plan, True, timeout, _context) + plan = self.build_execution_plan(template, resources()) + return self._send(plan, True, timeout) - def send(self, template, resources, _context): + @specs.parameter( + 'resources', dsl.MuranoObjectType('io.murano.system.Resources')) + def send(self, template, resources): self._check_enabled() - plan = self.buildExecutionPlan(template, resources) - return self._send(plan, False, 0, _context) + plan = self.build_execution_plan(template, resources()) + return self._send(plan, False, 0) - def callRaw(self, plan, _context, timeout=None): + def call_raw(self, plan, timeout=None): if timeout is None: timeout = CONF.engine.agent_timeout self._check_enabled() - return self._send(plan, True, timeout, _context) + return self._send(plan, True, timeout) - def sendRaw(self, plan, _context): + def send_raw(self, plan): self._check_enabled() - return self._send(plan, False, 0, _context) + return self._send(plan, False, 0) - def isReady(self, _context, timeout=100): + def is_ready(self, timeout=100): try: - self.waitReady(_context, timeout) + self.wait_ready(timeout) except exceptions.TimeoutException: return False else: return True - def waitReady(self, _context, timeout=100): + def wait_ready(self, timeout=100): self._check_enabled() template = {'Body': 'return', 'FormatVersion': '2.0.0', 'Scripts': {}} - self.call(template, False, _context, timeout) + self.call(template, False, timeout) def _process_v1_result(self, result): if result['IsException']: @@ -203,7 +205,7 @@ class Agent(murano_object.MuranoObject): 'timestamp': datetime.datetime.now().isoformat() } - def buildExecutionPlan(self, template, resources): + def build_execution_plan(self, template, resources): template = copy.deepcopy(template) if not isinstance(template, types.DictionaryType): raise ValueError('Incorrect execution plan ') @@ -310,9 +312,8 @@ class Agent(murano_object.MuranoObject): files[name] = file_id else: - template['Files'][file_id] = self._get_file_description(file, - resources, - folder) + template['Files'][file_id] = self._get_file_description( + file, resources, folder) files[name] = file_id return file_id diff --git a/murano/engine/system/agent_listener.py b/murano/engine/system/agent_listener.py index a9c36eb42..aac679e53 100644 --- a/murano/engine/system/agent_listener.py +++ b/murano/engine/system/agent_listener.py @@ -19,11 +19,10 @@ import greenlet from oslo_config import cfg from oslo_log import log as logging -import murano.common.exceptions as exceptions +from murano.common import exceptions +from murano.dsl import dsl from murano.dsl import helpers -import murano.dsl.murano_class as murano_class -import murano.dsl.murano_object as murano_object -import murano.engine.system.common as common +from murano.engine.system import common LOG = logging.getLogger(__name__) CONF = cfg.CONF @@ -33,9 +32,9 @@ class AgentListenerException(Exception): pass -@murano_class.classname('io.murano.system.AgentListener') -class AgentListener(murano_object.MuranoObject): - def initialize(self, _context, name): +@dsl.name('io.murano.system.AgentListener') +class AgentListener(object): + def __init__(self, name): self._enabled = False if CONF.engine.disable_murano_agent: return @@ -58,17 +57,17 @@ class AgentListener(murano_object.MuranoObject): def enabled(self): return self._enabled - def queueName(self): + def queue_name(self): return self._results_queue - def start(self, _context): + def start(self): if CONF.engine.disable_murano_agent: # Noop LOG.debug("murano-agent is disabled by the server") return if self._receive_thread is None: - helpers.get_environment(_context).on_session_finish( + helpers.get_environment().on_session_finish( lambda: self.stop()) self._receive_thread = eventlet.spawn(self._receive) @@ -87,10 +86,10 @@ class AgentListener(murano_object.MuranoObject): finally: self._receive_thread = None - def subscribe(self, message_id, event, _context): + def subscribe(self, message_id, event): self._check_enabled() self._subscriptions[message_id] = event - self.start(_context) + self.start() def unsubscribe(self, message_id): self._check_enabled() diff --git a/murano/engine/system/heat_stack.py b/murano/engine/system/heat_stack.py index 4112145cd..0fe856cb5 100644 --- a/murano/engine/system/heat_stack.py +++ b/murano/engine/system/heat_stack.py @@ -19,10 +19,9 @@ import eventlet import heatclient.exc as heat_exc from oslo_log import log as logging -import murano.common.utils as utils -import murano.dsl.helpers as helpers -import murano.dsl.murano_class as murano_class -import murano.dsl.murano_object as murano_object +from murano.common import utils +from murano.dsl import dsl +from murano.dsl import helpers from murano.common.i18n import _LI, _LW LOG = logging.getLogger(__name__) @@ -34,20 +33,20 @@ class HeatStackError(Exception): pass -@murano_class.classname('io.murano.system.HeatStack') -class HeatStack(murano_object.MuranoObject): - def initialize(self, _context, name, description=None): +@dsl.name('io.murano.system.HeatStack') +class HeatStack(object): + def __init__(self, name, description=None): self._name = name self._template = None self._parameters = {} self._files = {} self._applied = True self._description = description - self._clients = helpers.get_environment(_context).clients + self._clients = helpers.get_environment().clients self._last_stack_timestamps = (None, None) - def current(self, _context): - client = self._clients.get_heat_client(_context) + def current(self): + client = self._clients.get_heat_client() if self._template is not None: return self._template try: @@ -56,7 +55,6 @@ class HeatStack(murano_object.MuranoObject): stack_id='{0}/{1}'.format( stack_info.stack_name, stack_info.id)) - # template = {} self._template = template self._parameters.update( HeatStack._remove_system_params(stack_info.parameters)) @@ -68,36 +66,36 @@ class HeatStack(murano_object.MuranoObject): self._parameters.clear() return {} - def parameters(self, _context): - self.current(_context) + def parameters(self): + self.current() return self._parameters.copy() - def reload(self, _context): + def reload(self): self._template = None self._parameters.clear() - return self.current(_context) + return self.current() - def setTemplate(self, template): + def set_template(self, template): self._template = template self._parameters.clear() self._applied = False - def setParameters(self, parameters): + def set_parameters(self, parameters): self._parameters = parameters self._applied = False - def setFiles(self, files): + def set_files(self, files): self._files = files self._applied = False - def updateTemplate(self, _context, template): + def update_template(self, template): template_version = template.get('heat_template_version', HEAT_TEMPLATE_VERSION) if template_version != HEAT_TEMPLATE_VERSION: err_msg = ("Currently only heat_template_version %s " "is supported." % HEAT_TEMPLATE_VERSION) raise HeatStackError(err_msg) - self.current(_context) + self.current() self._template = helpers.merge_dicts(self._template, template) self._applied = False @@ -106,22 +104,22 @@ class HeatStack(murano_object.MuranoObject): return dict((k, v) for k, v in parameters.iteritems() if not k.startswith('OS::')) - def _get_status(self, context): + def _get_status(self): status = [None] def status_func(state_value): status[0] = state_value return True - self._wait_state(context, status_func) + self._wait_state(status_func) return status[0] - def _wait_state(self, context, status_func, wait_progress=False): + def _wait_state(self, status_func, wait_progress=False): tries = 4 delay = 1 while tries > 0: while True: - client = self._clients.get_heat_client(context) + client = self._clients.get_heat_client() try: stack_info = client.stacks.get( stack_id=self._name) @@ -162,10 +160,10 @@ class HeatStack(murano_object.MuranoObject): return {} return {} - def output(self, _context): - return self._wait_state(_context, lambda status: True) + def output(self): + return self._wait_state(lambda status: True) - def push(self, _context): + def push(self): if self._applied or self._template is None: return @@ -178,11 +176,11 @@ class HeatStack(murano_object.MuranoObject): template = copy.deepcopy(self._template) LOG.info(_LI('Pushing: {0}').format(template)) - current_status = self._get_status(_context) + current_status = self._get_status() resources = template.get('Resources') or template.get('resources') if current_status == 'NOT_FOUND': if resources is not None: - token_client = self._clients.get_heat_client(_context, False) + token_client = self._clients.get_heat_client(use_trusts=False) token_client.stacks.create( stack_name=self._name, parameters=self._parameters, @@ -190,12 +188,10 @@ class HeatStack(murano_object.MuranoObject): files=self._files, disable_rollback=True) - self._wait_state( - _context, - lambda status: status == 'CREATE_COMPLETE') + self._wait_state(lambda status: status == 'CREATE_COMPLETE') else: if resources is not None: - trust_client = self._clients.get_heat_client(_context) + trust_client = self._clients.get_heat_client() trust_client.stacks.update( stack_id=self._name, @@ -204,21 +200,19 @@ class HeatStack(murano_object.MuranoObject): template=template, disable_rollback=True) self._wait_state( - _context, lambda status: status == 'UPDATE_COMPLETE', True) else: - self.delete(_context) + self.delete() self._applied = not utils.is_different(self._template, template) - def delete(self, _context): - client = self._clients.get_heat_client(_context) + def delete(self): + client = self._clients.get_heat_client() try: - if not self.current(_context): + if not self.current(): return client.stacks.delete(stack_id=self._name) self._wait_state( - _context, lambda status: status in ('DELETE_COMPLETE', 'NOT_FOUND')) except heat_exc.NotFound: LOG.warn(_LW('Stack {0} already deleted?').format(self._name)) diff --git a/murano/engine/system/instance_reporter.py b/murano/engine/system/instance_reporter.py index f27c1586f..fb3a1cc52 100644 --- a/murano/engine/system/instance_reporter.py +++ b/murano/engine/system/instance_reporter.py @@ -18,7 +18,7 @@ from oslo_log import log as logging import oslo_messaging as messaging from murano.common import uuidutils -from murano.dsl import murano_class +from murano.dsl import dsl CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -28,23 +28,23 @@ APPLICATION = 100 OS_INSTANCE = 200 -@murano_class.classname('io.murano.system.InstanceNotifier') +@dsl.name('io.murano.system.InstanceNotifier') class InstanceReportNotifier(object): transport = None - def initialize(self, environment): + def __init__(self, environment): if InstanceReportNotifier.transport is None: InstanceReportNotifier.transport = messaging.get_transport(CONF) self._notifier = messaging.Notifier( InstanceReportNotifier.transport, publisher_id=uuidutils.generate_uuid(), topic='murano') - self._environment_id = environment.object_id + self._environment_id = environment.id def _track_instance(self, instance, instance_type, type_title, unit_count): payload = { - 'instance': instance.object_id, + 'instance': instance.id, 'environment': self._environment_id, 'instance_type': instance_type, 'type_name': instance.type.name, @@ -56,21 +56,21 @@ class InstanceReportNotifier(object): def _untrack_instance(self, instance, instance_type): payload = { - 'instance': instance.object_id, + 'instance': instance.id, 'environment': self._environment_id, 'instance_type': instance_type, } self._notifier.info({}, 'murano.untrack_instance', payload) - def trackApplication(self, instance, title=None, unitCount=None): - self._track_instance(instance, APPLICATION, title, unitCount) + def track_application(self, instance, title=None, unit_count=None): + self._track_instance(instance, APPLICATION, title, unit_count) - def untrackApplication(self, instance): + def untrack_application(self, instance): self._untrack_instance(instance, APPLICATION) - def trackCloudInstance(self, instance): + def track_cloud_instance(self, instance): self._track_instance(instance, OS_INSTANCE, None, 1) - def untrackCloudInstance(self, instance): + def untrack_cloud_instance(self, instance): self._untrack_instance(instance, OS_INSTANCE) diff --git a/murano/engine/system/mistralclient.py b/murano/engine/system/mistralclient.py index a8ba36573..858ed32a9 100644 --- a/murano/engine/system/mistralclient.py +++ b/murano/engine/system/mistralclient.py @@ -19,9 +19,8 @@ import json import eventlet from oslo_log import log as logging -import murano.dsl.helpers as helpers -import murano.dsl.murano_class as murano_class -import murano.dsl.murano_object as murano_object +from murano.dsl import dsl +from murano.dsl import helpers LOG = logging.getLogger(__name__) @@ -30,17 +29,17 @@ class MistralError(Exception): pass -@murano_class.classname('io.murano.system.MistralClient') -class MistralClient(murano_object.MuranoObject): - def initialize(self, _context): - self._clients = helpers.get_environment(_context).clients +@dsl.name('io.murano.system.MistralClient') +class MistralClient(object): + def __init__(self, context): + self._clients = helpers.get_environment(context).clients - def upload(self, _context, definition): - mistral_client = self._clients.get_mistral_client(_context) + def upload(self, definition): + mistral_client = self._clients.get_mistral_client() mistral_client.workflows.create(definition) - def run(self, _context, name, timeout=600, inputs=None, params=None): - mistral_client = self._clients.get_mistral_client(_context) + def run(self, name, timeout=600, inputs=None, params=None): + mistral_client = self._clients.get_mistral_client() execution = mistral_client.executions.create(workflow_name=name, workflow_input=inputs, params=params) diff --git a/murano/engine/system/net_explorer.py b/murano/engine/system/net_explorer.py index 6297eb161..e080b38b4 100644 --- a/murano/engine/system/net_explorer.py +++ b/murano/engine/system/net_explorer.py @@ -20,27 +20,24 @@ from oslo_config import cfg from oslo_log import log as logging from oslo_utils import uuidutils -import murano.dsl.helpers as helpers -import murano.dsl.murano_class as murano_class -import murano.dsl.murano_object as murano_object +from murano.dsl import dsl +from murano.dsl import helpers CONF = cfg.CONF LOG = logging.getLogger(__name__) -@murano_class.classname('io.murano.system.NetworkExplorer') -class NetworkExplorer(murano_object.MuranoObject): - # noinspection PyAttributeOutsideInit - def initialize(self, _context): - environment = helpers.get_environment(_context) +@dsl.name('io.murano.system.NetworkExplorer') +class NetworkExplorer(object): + def __init__(self): + environment = helpers.get_environment() self._clients = environment.clients self._tenant_id = environment.tenant_id self._settings = CONF.networking self._available_cidrs = self._generate_possible_cidrs() - # noinspection PyPep8Naming - def getDefaultRouter(self, _context): - client = self._clients.get_neutron_client(_context) + def get_default_router(self): + client = self._clients.get_neutron_client() router_name = self._settings.router_name routers = client.list_routers( @@ -82,15 +79,14 @@ class NetworkExplorer(murano_object.MuranoObject): router_id = routers[0]['id'] return router_id - # noinspection PyPep8Naming - def getAvailableCidr(self, _context, routerId, netId): + def get_available_cidr(self, router_id, net_id): """Uses hash of network IDs to minimize the collisions: different nets will attempt to pick different cidrs out of available range. If the cidr is taken will pick another one """ - taken_cidrs = self._get_cidrs_taken_by_router(_context, routerId) - id_hash = hash(netId) + taken_cidrs = self._get_cidrs_taken_by_router(router_id) + id_hash = hash(net_id) num_fails = 0 while num_fails < len(self._available_cidrs): cidr = self._available_cidrs[ @@ -102,44 +98,40 @@ class NetworkExplorer(murano_object.MuranoObject): return str(cidr) return None - # noinspection PyPep8Naming - def getDefaultDns(self): + def get_default_dns(self): return self._settings.default_dns - # noinspection PyPep8Naming - def getExternalNetworkIdForRouter(self, _context, routerId): - client = self._clients.get_neutron_client(_context) - router = client.show_router(routerId).get('router') + def get_external_network_id_for_router(self, router_id): + client = self._clients.get_neutron_client() + router = client.show_router(router_id).get('router') if not router or 'external_gateway_info' not in router: return None return router['external_gateway_info'].get('network_id') - # noinspection PyPep8Naming - def getExternalNetworkIdForNetwork(self, _context, networkId): - client = self._clients.get_neutron_client(_context) - network = client.show_network(networkId).get('network') + def get_external_network_id_for_network(self, network_id): + client = self._clients.get_neutron_client() + network = client.show_network(network_id).get('network') if network.get('router:external', False): - return networkId + return network_id # Get router interfaces of the network router_ports = client.list_ports( **{'device_owner': 'network:router_interface', - 'network_id': networkId}).get('ports') + 'network_id': network_id}).get('ports') # For each router this network is connected to # check if the router has external_gateway set for router_port in router_ports: ext_net_id = self.getExternalNetworkIdForRouter( - _context, router_port.get('device_id')) if ext_net_id: return ext_net_id return None - def _get_cidrs_taken_by_router(self, _context, router_id): + def _get_cidrs_taken_by_router(self, router_id): if not router_id: return [] - client = self._clients.get_neutron_client(_context) + client = self._clients.get_neutron_client() ports = client.list_ports(device_id=router_id)['ports'] subnet_ids = [] for port in ports: @@ -166,12 +158,10 @@ class NetworkExplorer(murano_object.MuranoObject): '{0}/{1}'.format(self._settings.env_ip_template, mask_width)) return list(net.subnet(width - bits_for_hosts)) - # noinspection PyPep8Naming - def listNetworks(self, _context): - client = self._clients.get_neutron_client(_context) + def list_networks(self): + client = self._clients.get_neutron_client() return client.list_networks()['networks'] - # noinspection PyPep8Naming - def listSubnetworks(self, _context): - client = self._clients.get_neutron_client(_context) + def list_subnetworks(self): + client = self._clients.get_neutron_client() return client.list_subnets()['subnets'] diff --git a/murano/engine/system/resource_manager.py b/murano/engine/system/resource_manager.py index e4b84d60e..22163cad6 100644 --- a/murano/engine/system/resource_manager.py +++ b/murano/engine/system/resource_manager.py @@ -18,7 +18,6 @@ import json as jsonlib import yaml as yamllib import murano.dsl.helpers as helpers -import murano.dsl.murano_object as murano_object if hasattr(yamllib, 'CSafeLoader'): yaml_loader = yamllib.CSafeLoader @@ -40,9 +39,9 @@ yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp', _construct_yaml_str) -class ResourceManager(murano_object.MuranoObject): - def initialize(self, package_loader, _context): - murano_class = helpers.get_type(_context) +class ResourceManager(object): + def __init__(self, package_loader, context): + murano_class = helpers.get_type(helpers.get_caller_context(context)) self._package = package_loader.get_package(murano_class.package.name) def string(self, name): diff --git a/murano/engine/system/status_reporter.py b/murano/engine/system/status_reporter.py index 35b3e209f..d2ee06efe 100644 --- a/murano/engine/system/status_reporter.py +++ b/murano/engine/system/status_reporter.py @@ -13,33 +13,39 @@ # See the License for the specific language governing permissions and # limitations under the License. +import types + from oslo_config import cfg from oslo_log import log as logging import oslo_messaging as messaging from murano.common import uuidutils -from murano.dsl import murano_class +from murano.dsl import dsl CONF = cfg.CONF LOG = logging.getLogger(__name__) -@murano_class.classname('io.murano.system.StatusReporter') +@dsl.name('io.murano.system.StatusReporter') class StatusReporter(object): transport = None - def initialize(self, environment): + def __init__(self, environment): if StatusReporter.transport is None: StatusReporter.transport = messaging.get_transport(CONF) self._notifier = messaging.Notifier( StatusReporter.transport, publisher_id=uuidutils.generate_uuid(), topic='murano') - self._environment_id = environment.object_id + if isinstance(environment, types.StringTypes): + self._environment_id = environment + else: + self._environment_id = environment.id def _report(self, instance, msg, details=None, level='info'): body = { - 'id': instance.object_id, + 'id': (self._environment_id if instance is None + else instance.id), 'text': msg, 'details': details, 'level': level, @@ -50,5 +56,9 @@ class StatusReporter(object): def report(self, instance, msg): self._report(instance, msg) + def report_error_(self, instance, msg): + self._report(instance, msg, None, 'error') + + @dsl.name('report_error') def report_error(self, instance, msg): self._report(instance, msg, None, 'error') diff --git a/murano/engine/system/system_objects.py b/murano/engine/system/system_objects.py index dd0f815df..e349f380e 100644 --- a/murano/engine/system/system_objects.py +++ b/murano/engine/system/system_objects.py @@ -12,10 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - -import inspect - -from murano.dsl import murano_class +from murano.dsl import dsl from murano.engine.system import agent from murano.engine.system import agent_listener from murano.engine.system import heat_stack @@ -26,25 +23,12 @@ from murano.engine.system import resource_manager from murano.engine.system import status_reporter -def _auto_register(class_loader): - globs = globals().copy() - for module_name, value in globs.iteritems(): - if inspect.ismodule(value): - for class_name in dir(value): - class_def = getattr(value, class_name) - if inspect.isclass(class_def) and hasattr( - class_def, '_murano_class_name'): - class_loader.import_class(class_def) - - def register(class_loader, package_loader): - _auto_register(class_loader) - - @murano_class.classname('io.murano.system.Resources') + @dsl.name('io.murano.system.Resources') class ResourceManagerWrapper(resource_manager.ResourceManager): - def initialize(self, _context): - super(ResourceManagerWrapper, self).initialize( - package_loader, _context) + def __init__(self, context): + super(ResourceManagerWrapper, self).__init__( + package_loader, context) class_loader.import_class(agent.Agent) class_loader.import_class(agent_listener.AgentListener) diff --git a/murano/engine/system/yaql_functions.py b/murano/engine/system/yaql_functions.py index ce376006b..9cf8f2cc6 100644 --- a/murano/engine/system/yaql_functions.py +++ b/murano/engine/system/yaql_functions.py @@ -15,7 +15,6 @@ import base64 import collections -import itertools import random import re import string @@ -24,39 +23,56 @@ import types import jsonpatch import jsonpointer -from oslo_config import cfg -import yaql.context +from yaql.language import specs +from yaql.language import utils +from yaql.language import yaqltypes -import murano.dsl.helpers as helpers +from murano.common import config as cfg +from murano.dsl import helpers +from murano.dsl import yaql_integration -CONF = cfg.CONF _random_string_counter = None -def _transform_json(json, mappings): - if isinstance(json, types.ListType): - return [_transform_json(t, mappings) for t in json] +@specs.parameter('value', yaqltypes.String()) +@specs.extension_method +def base64encode(value): + return base64.b64encode(value) - if isinstance(json, types.DictionaryType): - result = {} - for key, value in json.items(): - result[_transform_json(key, mappings)] = \ - _transform_json(value, mappings) - return result - elif isinstance(json, types.ListType): - result = [] - for value in json: - result.append(_transform_json(value, mappings)) - return result +@specs.parameter('value', yaqltypes.String()) +@specs.extension_method +def base64decode(value): + return base64.b64decode(value) - elif isinstance(json, types.StringTypes) and json.startswith('$'): - value = _convert_macro_parameter(json[1:], mappings) + +@specs.parameter('collection', yaqltypes.Iterable()) +@specs.parameter('composer', yaqltypes.Lambda()) +@specs.extension_method +def pselect(collection, composer): + return helpers.parallel_select(collection, composer) + + +@specs.parameter('mappings', collections.Mapping) +@specs.extension_method +def bind(obj, mappings): + if isinstance(obj, types.StringTypes) and obj.startswith('$'): + value = _convert_macro_parameter(obj[1:], mappings) if value is not None: return value - - return json + elif utils.is_sequence(obj): + return [bind(t, mappings) for t in obj] + elif isinstance(obj, collections.Mapping): + result = {} + for key, value in obj.iteritems(): + result[bind(key, mappings)] = bind(value, mappings) + return result + elif isinstance(obj, types.StringTypes) and obj.startswith('$'): + value = _convert_macro_parameter(obj[1:], mappings) + if value is not None: + return value + return obj def _convert_macro_parameter(macro, mappings): @@ -73,155 +89,36 @@ def _convert_macro_parameter(macro, mappings): return mappings[macro] -@yaql.context.EvalArg('format', types.StringTypes) -def _format(format, *args): - return format.format(*[t() for t in args]) +@specs.parameter('group', yaqltypes.String()) +@specs.parameter('setting', yaqltypes.String()) +def config(group, setting): + return cfg.CONF[group][setting] -@yaql.context.EvalArg('src', types.StringTypes) -@yaql.context.EvalArg('substring', types.StringTypes) -@yaql.context.EvalArg('value', types.StringTypes) -def _replace_str(src, substring, value): - return src.replace(substring, value) +@specs.parameter('setting', yaqltypes.String()) +@specs.name('config') +def config_default(setting): + return cfg.CONF[setting] -@yaql.context.EvalArg('src', types.StringTypes) -@yaql.context.EvalArg('replacements', dict) -def _replace_dict(src, replacements): - for key, value in replacements.iteritems(): - if isinstance(src, str): - src = src.replace(key, str(value)) - else: - src = src.replace(key, unicode(value)) - return src +@specs.parameter('string', yaqltypes.String()) +@specs.parameter('start', int) +@specs.parameter('length', int) +@specs.inject('delegate', yaqltypes.Delegate('substring', method=True)) +@specs.extension_method +def substr(delegate, string, start, length=-1): + return delegate(string, start, length) -def _len(value): - return len(value()) - - -def _coalesce(*args): - for t in args: - val = t() - if val: - return val - return None - - -@yaql.context.EvalArg('value', types.StringTypes) -def _base64encode(value): - return base64.b64encode(value) - - -@yaql.context.EvalArg('value', types.StringTypes) -def _base64decode(value): - return base64.b64decode(value) - - -@yaql.context.EvalArg('group', types.StringTypes) -@yaql.context.EvalArg('setting', types.StringTypes) -def _config(group, setting): - return CONF[group][setting] - - -@yaql.context.EvalArg('setting', types.StringTypes) -def _config_default(setting): - return CONF[setting] - - -@yaql.context.EvalArg('value', types.StringTypes) -def _upper(value): - return value.upper() - - -@yaql.context.EvalArg('value', types.StringTypes) -def _lower(value): - return value.lower() - - -@yaql.context.EvalArg('separator', types.StringTypes) -def _join(separator, collection): - return separator.join(str(t) for t in collection()) - - -@yaql.context.EvalArg('value', types.StringTypes) -@yaql.context.EvalArg('separator', types.StringTypes) -def _split(value, separator): - return value.split(separator) - - -@yaql.context.EvalArg('value', types.StringTypes) -@yaql.context.EvalArg('prefix', types.StringTypes) -def _startswith(value, prefix): - return value.startswith(prefix) - - -@yaql.context.EvalArg('value', types.StringTypes) -@yaql.context.EvalArg('suffix', types.StringTypes) -def _endswith(value, suffix): - return value.endswith(suffix) - - -@yaql.context.EvalArg('value', types.StringTypes) -def _trim(value): - return value.strip() - - -@yaql.context.EvalArg('value', types.StringTypes) -@yaql.context.EvalArg('pattern', types.StringTypes) -def _mathces(value, pattern): - return re.match(pattern, value) is not None - - -@yaql.context.EvalArg('value', types.StringTypes) -@yaql.context.EvalArg('index', int) -@yaql.context.EvalArg('length', int) -def _substr3(value, index, length): - if length < 0: - return value[index:] - else: - return value[index:index + length] - - -@yaql.context.EvalArg('value', types.StringTypes) -@yaql.context.EvalArg('index', int) -def _substr2(value, index): - return _substr3(value, index, -1) - - -def _str(value): - value = value() - if value is None: - return '' - elif value is True: - return 'true' - elif value is False: - return 'false' - return unicode(value) - - -def _int(value): - value = value() - if value is None: - return 0 - return int(value) - - -def _pselect(collection, composer): - if isinstance(collection, types.ListType): - return helpers.parallel_select(collection, composer) - else: - return helpers.parallel_select(collection(), composer) - - -def _patch(obj, patch): - obj = obj() - patch = patch() - if not isinstance(patch, types.ListType): - patch = [patch] +@specs.extension_method +def patch_(obj, patch): + if not isinstance(patch, tuple): + patch = (patch,) + patch = yaql_integration.to_mutable(patch) patch = jsonpatch.JsonPatch(patch) try: - return patch.apply(obj) + obj = yaql_integration.to_mutable(obj) + return patch.apply(obj, in_place=True) except jsonpointer.JsonPointerException: return obj @@ -252,7 +149,7 @@ def _int2base(x, base): return ''.join(digits) -def _random_name(): +def random_name(): """Replace '#' char in pattern with supplied number, if no pattern is supplied generate short and unique name for the host. @@ -275,69 +172,10 @@ def _random_name(): return prefix + timestamp + suffix -@yaql.context.EvalArg('self', dict) -def _values(self): - return self.values() - - -@yaql.context.EvalArg('self', dict) -def _keys(self): - return self.keys() - - -@yaql.context.EvalArg('self', collections.Iterable) -def _flatten(self): - for i in self: - if isinstance(i, collections.Iterable): - for ii in i: - yield ii - else: - yield i - - -@yaql.context.EvalArg('self', dict) -@yaql.context.EvalArg('other', dict) -def _merge_with(self, other): - return helpers.merge_dicts(self, other) - - -@yaql.context.EvalArg('collection', collections.Iterable) -@yaql.context.EvalArg('count', int) -def _skip(collection, count): - return itertools.islice(collection, count, None) - - -@yaql.context.EvalArg('collection', collections.Iterable) -@yaql.context.EvalArg('count', int) -def _take(collection, count): - return itertools.islice(collection, count) - - -@yaql.context.EvalArg('collection', collections.Iterable) -def _aggregate(collection, selector): - return reduce(selector, collection) - - -@yaql.context.EvalArg('collection', collections.Iterable) -def _aggregate_with_seed(collection, selector, seed): - return reduce(selector, collection, seed()) - - -@yaql.context.EvalArg('collection', collections.Iterable) -def _first(collection): - return iter(collection).next() - - -@yaql.context.EvalArg('collection', collections.Iterable) -def _first_or_default(collection): - try: - return iter(collection).next() - except StopIteration: - return None - - -@yaql.context.EvalArg('collection', collections.Iterable) -def _first_or_default2(collection, default): +@specs.parameter('collection', yaqltypes.Iterable()) +@specs.parameter('default', nullable=True) +@specs.extension_method +def first_or_default(collection, default=None): try: return iter(collection).next() except StopIteration: @@ -345,42 +183,19 @@ def _first_or_default2(collection, default): def register(context): - context.register_function( - lambda json, mappings: _transform_json(json(), mappings()), 'bind') + context.register_function(base64decode) + context.register_function(base64encode) + context.register_function(pselect) + context.register_function(bind) + context.register_function(random_name) + context.register_function(patch_) + context.register_function(config) + context.register_function(config_default) + context.register_function(substr) + context.register_function(first_or_default) - context.register_function(_format, 'format') - context.register_function(_replace_str, 'replace') - context.register_function(_replace_dict, 'replace') - context.register_function(_len, 'len') - context.register_function(_coalesce, 'coalesce') - context.register_function(_base64decode, 'base64decode') - context.register_function(_base64encode, 'base64encode') - context.register_function(_config, 'config') - context.register_function(_config_default, 'config') - context.register_function(_lower, 'toLower') - context.register_function(_upper, 'toUpper') - context.register_function(_join, 'join') - context.register_function(_split, 'split') - context.register_function(_pselect, 'pselect') - context.register_function(_startswith, 'startsWith') - context.register_function(_endswith, 'endsWith') - context.register_function(_trim, 'trim') - context.register_function(_mathces, 'matches') - context.register_function(_substr2, 'substr') - context.register_function(_substr3, 'substr') - context.register_function(_str, 'str') - context.register_function(_int, 'int') - context.register_function(_patch, 'patch') - context.register_function(_random_name, 'randomName') - # Temporary workaround, these functions should be moved to YAQL - context.register_function(_keys, 'keys') - context.register_function(_values, 'values') - context.register_function(_flatten, 'flatten') - context.register_function(_merge_with, 'mergeWith') - context.register_function(_skip, 'skip') - context.register_function(_take, 'take') - context.register_function(_aggregate, 'aggregate') - context.register_function(_aggregate_with_seed, 'aggregate') - context.register_function(_first, 'first') - context.register_function(_first_or_default, 'firstOrDefault') - context.register_function(_first_or_default2, 'firstOrDefault') + for t in ('to_lower', 'to_upper', 'trim', 'join', 'split', + 'starts_with', 'ends_with', 'matches', 'replace', + 'flatten'): + for spec in utils.to_extension_method(t, context): + context.register_function(spec) diff --git a/murano/engine/yaql_yaml_loader.py b/murano/engine/yaql_yaml_loader.py index bd3a738e4..e1e653ce9 100644 --- a/murano/engine/yaql_yaml_loader.py +++ b/murano/engine/yaql_yaml_loader.py @@ -17,6 +17,7 @@ import yaml import yaml.composer import yaml.constructor +from murano.dsl import dsl_types from murano.dsl import yaql_expression @@ -49,14 +50,12 @@ YaqlYamlLoader.yaml_implicit_resolvers = resolvers def build_position(node): - return yaql_expression.YaqlExpressionFilePosition( + return dsl_types.ExpressionFilePosition( node.start_mark.name, node.start_mark.line + 1, node.start_mark.column + 1, - node.start_mark.index, node.end_mark.line + 1, - node.end_mark.column + 1, - node.end_mark.index - node.start_mark.index) + node.end_mark.column + 1) def yaql_constructor(loader, node): diff --git a/murano/tests/unit/dsl/foundation/runner.py b/murano/tests/unit/dsl/foundation/runner.py index 87d0dec33..5fb18c5f2 100644 --- a/murano/tests/unit/dsl/foundation/runner.py +++ b/murano/tests/unit/dsl/foundation/runner.py @@ -20,6 +20,7 @@ from murano.dsl import dsl_exception from murano.dsl import executor from murano.dsl import murano_object from murano.dsl import serializer +from murano.dsl import yaql_integration from murano.engine import environment from murano.tests.unit.dsl.foundation import object_model @@ -56,12 +57,13 @@ class Runner(object): self.executor = executor.MuranoDslExecutor( class_loader, environment.Environment()) - self._root = self.executor.load(model) + self._root = self.executor.load(model).object def _execute(self, name, object_id, *args, **kwargs): obj = self.executor.object_store.get(object_id) try: final_args = [] + final_kwargs = {} for arg in args: if isinstance(arg, object_model.Object): arg = object_model.build_model(arg) @@ -69,8 +71,9 @@ class Runner(object): for name, arg in kwargs.iteritems(): if isinstance(arg, object_model.Object): arg = object_model.build_model(arg) - final_args.append({name: arg}) - return obj.type.invoke(name, self.executor, obj, tuple(final_args)) + final_kwargs[name] = arg + return yaql_integration.to_mutable(obj.type.invoke( + name, self.executor, obj, tuple(final_args), final_kwargs)) except dsl_exception.MuranoPlException as e: if not self.preserve_exception: original_exception = getattr(e, 'original_exception', None) diff --git a/murano/tests/unit/dsl/foundation/test_case.py b/murano/tests/unit/dsl/foundation/test_case.py index d6eec78cc..ac2e85430 100644 --- a/murano/tests/unit/dsl/foundation/test_case.py +++ b/murano/tests/unit/dsl/foundation/test_case.py @@ -36,7 +36,7 @@ class DslTestCase(base.MuranoTestCase): self._class_loader = test_class_loader.TestClassLoader( directory, 'tests', sys_class_loader) self.register_function( - lambda data: self._traces.append(data()), 'trace') + lambda data: self._traces.append(data), 'trace') self._traces = [] test_class_loader.TestClassLoader.clear_configs() eventlet.debug.hub_exceptions(False) diff --git a/murano/tests/unit/dsl/meta/ContractExamples.yaml b/murano/tests/unit/dsl/meta/ContractExamples.yaml index 038c6780a..ad6af54b3 100644 --- a/murano/tests/unit/dsl/meta/ContractExamples.yaml +++ b/murano/tests/unit/dsl/meta/ContractExamples.yaml @@ -4,6 +4,9 @@ Properties: sampleClass: Contract: $.class(SampleClass1) + ordinaryProperty: + Contract: $.string() + Methods: testStringContract: Arguments: @@ -135,3 +138,11 @@ Methods: Body: Return: $arg + testDefaultExpression: + Arguments: + - arg: + Contract: $.string() + Default: $.ordinaryProperty + Body: + Return: $arg + diff --git a/murano/tests/unit/dsl/meta/CreatedClass1.yaml b/murano/tests/unit/dsl/meta/CreatedClass1.yaml new file mode 100644 index 000000000..2e0dfd226 --- /dev/null +++ b/murano/tests/unit/dsl/meta/CreatedClass1.yaml @@ -0,0 +1,33 @@ +Name: CreatedClass1 + +Properties: + property1: + Contract: $.string() + + property2: + Contract: $.int() + + xxx: + Contract: $ + Usage: Out + + +Methods: + .init: + Arguments: + - property1: + Contract: $.string() + Body: + - trace('CreatedClass1::.init') + - trace($property1) + - $.property1: STRING + - trace($.property1) + - trace($.property2) + + createClass2: + Arguments: + - parent: + Contract: $.class(CreatingClass) + Body: + - $.xxx: new(CreatedClass2, $parent, QQQ, property1 => STR, property2 => 99) + - Return: $ \ No newline at end of file diff --git a/murano/tests/unit/dsl/meta/CreatedClass2.yaml b/murano/tests/unit/dsl/meta/CreatedClass2.yaml new file mode 100644 index 000000000..01178bdb1 --- /dev/null +++ b/murano/tests/unit/dsl/meta/CreatedClass2.yaml @@ -0,0 +1,20 @@ +Name: CreatedClass2 + +Properties: + property1: + Contract: $.string() + + property2: + Contract: $.int() + + + + +Methods: + + .init: + Body: + - $.find(CreatingClass).require() + - trace('CreatedClass2::.init') + + diff --git a/murano/tests/unit/dsl/meta/CreatingClass.yaml b/murano/tests/unit/dsl/meta/CreatingClass.yaml new file mode 100644 index 000000000..db9fe5250 --- /dev/null +++ b/murano/tests/unit/dsl/meta/CreatingClass.yaml @@ -0,0 +1,21 @@ +Name: CreatingClass + +Properties: + yyy: + Contract: $ + Usage: Out + + +Methods: + .init: + Body: + trace('CreatingClass::.init') + + testNew: + Body: + - new(CreatedClass1, property1 => string, property2 => 123) + + testNewWithOwnership: + Body: + - $.yyy: new(CreatedClass1, property1 => string, property2 => 123) + - Return: $.yyy.createClass2($this) diff --git a/murano/tests/unit/dsl/meta/SampleClass1.yaml b/murano/tests/unit/dsl/meta/SampleClass1.yaml index 6836f3da8..f8ac583ba 100644 --- a/murano/tests/unit/dsl/meta/SampleClass1.yaml +++ b/murano/tests/unit/dsl/meta/SampleClass1.yaml @@ -5,6 +5,11 @@ Properties: Contract: $.string().notNull() classProperty: Contract: $.class(SampleClass2).notNull() + assignedProperty: + Contract: $ + Usage: Runtime + arbitraryProperty: + Contract: $ Workflow: testTrace: @@ -54,7 +59,7 @@ Workflow: - $result.Arr[0]: 3 - $result.Arr[$index - 1]: 5 - $result.Arr[$index + 1][1]: 123 - - $result.Dict: {} + #- $result.Dict: {} - $result.Dict.Key1: V1 - $keyName: Key2 - $result.Dict[$keyName]: {} @@ -62,6 +67,22 @@ Workflow: - $result.Dict[$keyName][toUpper($keyName)]: V3 - Return: $result + testAssignmentOnProperty: + Body: + #- $.assignedProperty: {} + - $.assignedProperty.Arr: [1, 2, [10, 11]] + - $index: 1 + - $.assignedProperty.Arr[0]: 3 + - $.assignedProperty.Arr[$index - 1]: 5 + - $.assignedProperty.Arr[$index + 1][1]: 123 + #- $.assignedProperty.Dict: {} + - $.assignedProperty.Dict.Key1: V1 + - $keyName: Key2 + - $.assignedProperty.Dict[$keyName]: {} + - $.assignedProperty.Dict[$keyName]['a_b']: V2 + - $.assignedProperty.Dict[$keyName][toUpper($keyName)]: V3 + - Return: $.assignedProperty + testAssignByCopy: Arguments: diff --git a/murano/tests/unit/dsl/test_agent.py b/murano/tests/unit/dsl/test_agent.py index 833905ae1..e39338a16 100644 --- a/murano/tests/unit/dsl/test_agent.py +++ b/murano/tests/unit/dsl/test_agent.py @@ -14,9 +14,11 @@ # limitations under the License. import mock -import yaql.context from murano.common import exceptions as exc +from murano.dsl import constants +from murano.dsl import helpers +from murano.dsl import yaql_integration from murano.engine import environment from murano.engine.system import agent from murano.engine.system import agent_listener @@ -33,22 +35,26 @@ class TestAgentListener(test_case.DslTestCase): model = om.Object( 'AgentListenerTests') self.runner = self.new_runner(model) - self.context = yaql.context.Context() - self.context.set_data(environment.Environment(), '?environment') + self.context = yaql_integration.create_empty_context() + self.context[constants.CTX_ENVIRONMENT] = environment.Environment() def test_listener_enabled(self): self.override_config('disable_murano_agent', False, 'engine') - al = self.runner.testAgentListener() + al = self.runner.testAgentListener().extension self.assertTrue(al.enabled) - al.subscribe('msgid', 'event', self.context) - self.assertEqual({'msgid': 'event'}, al._subscriptions) + with helpers.contextual(self.context): + try: + al.subscribe('msgid', 'event') + self.assertEqual({'msgid': 'event'}, al._subscriptions) + finally: + al.stop() def test_listener_disabled(self): self.override_config('disable_murano_agent', True, 'engine') - al = self.runner.testAgentListener() + al = self.runner.testAgentListener().extension self.assertFalse(al.enabled) self.assertRaises(exc.PolicyViolationException, - al.subscribe, 'msgid', 'event', None) + al.subscribe, 'msgid', 'event') class TestAgent(test_case.DslTestCase): @@ -68,20 +74,20 @@ class TestAgent(test_case.DslTestCase): agent_cls = 'murano.engine.system.agent.Agent' with mock.patch(agent_cls + '._get_environment') as f: f.return_value = m - a = self.runner.testAgent() + a = self.runner.testAgent().extension self.assertTrue(a.enabled) self.assertEqual(m, a._environment) with mock.patch(agent_cls + '._send') as s: s.return_value = mock.MagicMock() - a.sendRaw({}, None) - s.assert_called_with({}, False, 0, None) + a.send_raw({}) + s.assert_called_with({}, False, 0) def test_agent_disabled(self): self.override_config('disable_murano_agent', True, 'engine') - a = self.runner.testAgent() + a = self.runner.testAgent().extension self.assertFalse(a.enabled) - self.assertRaises(exc.PolicyViolationException, a.call, {}, None, None) - self.assertRaises(exc.PolicyViolationException, a.send, {}, None, None) - self.assertRaises(exc.PolicyViolationException, a.callRaw, {}, None) - self.assertRaises(exc.PolicyViolationException, a.sendRaw, {}, None) + self.assertRaises(exc.PolicyViolationException, a.call, {}, None) + self.assertRaises(exc.PolicyViolationException, a.send, {}, None) + self.assertRaises(exc.PolicyViolationException, a.call_raw, {}) + self.assertRaises(exc.PolicyViolationException, a.send_raw, {}) diff --git a/murano/tests/unit/dsl/test_assignments.py b/murano/tests/unit/dsl/test_assignments.py index 94bfdd013..e463f5fc6 100644 --- a/murano/tests/unit/dsl/test_assignments.py +++ b/murano/tests/unit/dsl/test_assignments.py @@ -37,6 +37,16 @@ class TestAssignments(test_case.DslTestCase): } }, self._runner.testAssignment()) + def test_assignment_on_property(self): + self.assertEqual( + { + 'Arr': [5, 2, [10, 123]], + 'Dict': { + 'Key1': 'V1', + 'Key2': {'KEY2': 'V3', 'a_b': 'V2'} + } + }, self._runner.testAssignmentOnProperty()) + def test_assign_by_copy(self): self.assertEqual( [1, 2, 3], diff --git a/murano/tests/unit/dsl/test_construction.py b/murano/tests/unit/dsl/test_construction.py new file mode 100644 index 000000000..7ad95be16 --- /dev/null +++ b/murano/tests/unit/dsl/test_construction.py @@ -0,0 +1,39 @@ +# Copyright (c) 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from murano.dsl import serializer + +from murano.tests.unit.dsl.foundation import object_model as om +from murano.tests.unit.dsl.foundation import test_case + + +class TestConstruction(test_case.DslTestCase): + def setUp(self): + super(TestConstruction, self).setUp() + self._runner = self.new_runner(om.Object('CreatingClass')) + + def test_new(self): + self._runner.testNew() + self.assertEqual( + ['CreatingClass::.init', 'CreatedClass1::.init', + 'string', 'STRING', 123], + self.traces) + + def test_new_with_ownership(self): + obj = serializer.serialize(self._runner.testNewWithOwnership()) + self.assertEqual('STRING', obj.get('property1')) + self.assertIsNotNone('string', obj.get('xxx')) + self.assertEqual('STR', obj['xxx'].get('property1')) + self.assertEqual('QQQ', obj['xxx']['?'].get('name')) diff --git a/murano/tests/unit/dsl/test_contracts.py b/murano/tests/unit/dsl/test_contracts.py index 239a19f32..ba4b3ed41 100644 --- a/murano/tests/unit/dsl/test_contracts.py +++ b/murano/tests/unit/dsl/test_contracts.py @@ -14,8 +14,8 @@ import types +from murano.dsl import dsl from murano.dsl import exceptions -from murano.dsl import murano_object from murano.tests.unit.dsl.foundation import object_model as om from murano.tests.unit.dsl.foundation import test_case @@ -26,6 +26,7 @@ class TestContracts(test_case.DslTestCase): self._runner = self.new_runner( om.Object( 'ContractExamples', + ordinaryProperty='PROPERTY', sampleClass=om.Object( 'SampleClass1', stringProperty='string1', @@ -96,12 +97,12 @@ class TestContracts(test_case.DslTestCase): def test_class_contract(self): arg = om.Object('SampleClass2', class2Property='qwerty') result = self._runner.testClassContract(arg) - self.assertIsInstance(result, murano_object.MuranoObject) + self.assertIsInstance(result, dsl.MuranoObjectInterface) def test_class_contract_by_ref(self): arg = om.Object('SampleClass2', class2Property='qwerty') result = self._runner.testClassContract(arg) - self.assertEqual(result.object_id, arg.id) + self.assertEqual(result.id, arg.id) def test_class_contract_failure(self): self.assertRaises( @@ -122,8 +123,8 @@ class TestContracts(test_case.DslTestCase): def test_class_from_id_contract(self): object_id = self._runner.root.get_property('sampleClass').object_id result = self._runner.testClassFromIdContract(object_id) - self.assertIsInstance(result, murano_object.MuranoObject) - self.assertEqual(result.object_id, object_id) + self.assertIsInstance(result, dsl.MuranoObjectInterface) + self.assertEqual(result.id, object_id) def test_check_contract(self): arg = om.Object('SampleClass2', class2Property='qwerty') @@ -290,3 +291,7 @@ class TestContracts(test_case.DslTestCase): def test_default(self): self.assertEqual('value', self._runner.testDefault('value')) self.assertEqual('DEFAULT', self._runner.testDefault()) + + def test_default_expression(self): + self.assertEqual('PROPERTY', self._runner.testDefaultExpression()) + self.assertEqual('value', self._runner.testDefaultExpression('value')) diff --git a/murano/tests/unit/dsl/test_engine_yaql_functions.py b/murano/tests/unit/dsl/test_engine_yaql_functions.py index 8e20c00da..4dbe25c7a 100644 --- a/murano/tests/unit/dsl/test_engine_yaql_functions.py +++ b/murano/tests/unit/dsl/test_engine_yaql_functions.py @@ -15,7 +15,7 @@ import types from testtools import matchers -import yaql.exceptions as yaql_exc +from yaql.language import exceptions as yaql_exceptions from murano.tests.unit.dsl.foundation import object_model as om from murano.tests.unit.dsl.foundation import test_case @@ -62,7 +62,7 @@ class TestEngineYaqlFunctions(test_case.DslTestCase): self._runner.testReplaceStr('John Kennedy', 'Kennedy', 'Doe')) self.assertRaises( - yaql_exc.YaqlExecutionException, + yaql_exceptions.NoMatchingMethodException, self._runner.testReplaceStr, None, 'Kennedy', 'Doe') def test_replace_dict(self): @@ -120,7 +120,7 @@ class TestEngineYaqlFunctions(test_case.DslTestCase): 'false', self._runner.testStr(False)) self.assertEqual( - '', + 'null', self._runner.testStr(None)) def test_int(self): diff --git a/murano/tests/unit/dsl/test_results_serializer.py b/murano/tests/unit/dsl/test_results_serializer.py index 83d7fc2fd..c4d6df271 100644 --- a/murano/tests/unit/dsl/test_results_serializer.py +++ b/murano/tests/unit/dsl/test_results_serializer.py @@ -32,6 +32,7 @@ class TestResultsSerializer(test_case.DslTestCase): self._class1 = om.Object( 'SampleClass1', stringProperty='string1', + arbitraryProperty={'a': [1, 2]}, classProperty=self._class2) self._root_class = om.Object('ContractExamples', sampleClass=self._class1) @@ -141,4 +142,4 @@ class TestResultsSerializer(test_case.DslTestCase): 'key5': {'x': 'y'}, 'key6': [{'w': 'q'}] }, - serializer.serialize_object(result)) + serializer.serialize(result)) diff --git a/murano/tests/unit/engine/system/test_agent.py b/murano/tests/unit/engine/system/test_agent.py index fd706bec1..7f38d80e0 100644 --- a/murano/tests/unit/engine/system/test_agent.py +++ b/murano/tests/unit/engine/system/test_agent.py @@ -18,9 +18,10 @@ import mock import yaml as yamllib from murano.dsl import murano_class +from murano.dsl import murano_object from murano.dsl import object_store -import murano.engine.system.agent as agent -import murano.engine.system.resource_manager as resource +from murano.engine.system import agent +from murano.engine.system import resource_manager from murano.tests.unit import base @@ -36,9 +37,14 @@ class TestExecutionPlan(base.MuranoTestCase): self.mock_murano_class.name = 'io.murano.system.Agent' self.mock_murano_class.parents = [] self.mock_object_store = mock.Mock(spec=object_store.ObjectStore) - self.agent = agent.Agent(self.mock_murano_class, None, - self.mock_object_store, None) - self.resources = mock.Mock(spec=resource.ResourceManager) + + object_interface = mock.Mock(spec=murano_object.MuranoObject) + object_interface.id = '1234' + + agent.Agent._get_environment = \ + lambda this, iface, host: object_interface + self.agent = agent.Agent(None, object_interface) + self.resources = mock.Mock(spec=resource_manager.ResourceManager) self.resources.string.return_value = 'text' self.uuids = ['ID1', 'ID2', 'ID3', 'ID4'] self.mock_uuid = self._stub_uuid(self.uuids) @@ -55,34 +61,34 @@ class TestExecutionPlan(base.MuranoTestCase): template = yamllib.load( self._read('application.template'), Loader=self.yaml_loader) - template = self.agent.buildExecutionPlan(template, self.resources) + template = self.agent.build_execution_plan(template, self.resources) self.assertEqual(template, self._get_application()) def test_execution_plan_v2_chef_type(self): template = yamllib.load( self._read('chef.template'), Loader=self.yaml_loader) - template = self.agent.buildExecutionPlan(template, self.resources) + template = self.agent.build_execution_plan(template, self.resources) self.assertEqual(template, self._get_chef()) def test_execution_plan_v2_telnet_application(self): template = yamllib.load( self._read('DeployTelnet.template'), Loader=self.yaml_loader) - template = self.agent.buildExecutionPlan(template, self.resources) + template = self.agent.build_execution_plan(template, self.resources) self.assertEqual(template, self._get_telnet_application()) def test_execution_plan_v2_tomcat_application(self): template = yamllib.load( self._read('DeployTomcat.template'), Loader=self.yaml_loader) - template = self.agent.buildExecutionPlan(template, self.resources) + template = self.agent.build_execution_plan(template, self.resources) def test_execution_plan_v2_app_without_files(self): template = yamllib.load( self._read('application_without_files.template'), Loader=self.yaml_loader) - template = self.agent.buildExecutionPlan(template, self.resources) + template = self.agent.build_execution_plan(template, self.resources) self.assertEqual(template, self._get_app_without_files()) def _get_application(self): diff --git a/murano/tests/unit/test_engine.py b/murano/tests/unit/test_engine.py index 9d38b885b..250781033 100644 --- a/murano/tests/unit/test_engine.py +++ b/murano/tests/unit/test_engine.py @@ -18,6 +18,8 @@ import re import mock import yaql +from yaql.language import exceptions +from yaql.language import utils import murano.dsl.helpers as helpers import murano.dsl.namespace_resolver as ns_resolver @@ -122,37 +124,21 @@ class TestHelperFunctions(base.MuranoTestCase): self.assertTrue(re.match(r'[a-z0-9]{32}', generated_id)) def test_evaluate(self): - yaql_value = mock.Mock(spec=yaql_expression.YaqlExpression, - evaluate=lambda context: 'atom') - complex_value = {yaql_value: ['some', (1, yaql_value), lambda: 'hi!'], + yaql_value = mock.Mock(yaql_expression.YaqlExpression, + return_value='atom') + complex_value = {yaql_value: ['some', (1, yaql_value), 'hi!'], 'sample': [yaql_value, xrange(5)]} - complex_literal = {'atom': ['some', (1, 'atom'), 'hi!'], - 'sample': ['atom', [0, 1, 2, 3, 4]]} - # tuple(evaluate(list)) transformation adds + 1 - complex_literal_depth = 3 + 1 - - context = yaql.create_context(False) - evaluated_value = helpers.evaluate(yaql_value, context, 1) - non_evaluated_value = helpers.evaluate(yaql_value, context, 0) + complex_literal = utils.FrozenDict({ + 'atom': ('some', (1, 'atom'), 'hi!'), + 'sample': ('atom', (0, 1, 2, 3, 4)) + }) + print yaql_value(1) + context = yaql.create_context() + evaluated_value = helpers.evaluate(yaql_value, context) evaluated_complex_value = helpers.evaluate(complex_value, context) - non_evaluated_complex_value = helpers.evaluate( - complex_value, context, complex_literal_depth) self.assertEqual('atom', evaluated_value) - self.assertNotEqual('atom', non_evaluated_value) self.assertEqual(complex_literal, evaluated_complex_value) - self.assertNotEqual(complex_literal, non_evaluated_complex_value) - - def test_needs_evaluation(self): - testee = helpers.needs_evaluation - parsed_expr = yaql.parse("string") - yaql_expr = yaql_expression.YaqlExpression("string") - - self.assertTrue(testee(parsed_expr)) - self.assertTrue(testee(yaql_expr)) - self.assertTrue(testee({yaql_expr: 1})) - self.assertTrue(testee({'label': yaql_expr})) - self.assertTrue(testee([yaql_expr])) class TestYaqlExpression(base.MuranoTestCase): @@ -183,23 +169,23 @@ class TestYaqlExpression(base.MuranoTestCase): expected_calls = [mock.call(string), mock.call().evaluate(context=None)] - with mock.patch('yaql.parse') as mock_parse: + with mock.patch('murano.dsl.yaql_integration.parse') as mock_parse: yaql_expr = yaql_expression.YaqlExpression(string) - yaql_expr.evaluate() + yaql_expr(None) self.assertEqual(expected_calls, mock_parse.mock_calls) def test_match_returns(self): expr = yaql_expression.YaqlExpression('string') - with mock.patch('yaql.parse'): + with mock.patch('murano.dsl.yaql_integration.parse'): self.assertTrue(expr.match('$some')) self.assertTrue(expr.match('$.someMore')) - with mock.patch('yaql.parse') as parse_mock: - parse_mock.side_effect = yaql.exceptions.YaqlGrammarException + with mock.patch('murano.dsl.yaql_integration.parse') as parse_mock: + parse_mock.side_effect = exceptions.YaqlGrammarException self.assertFalse(expr.match('')) - with mock.patch('yaql.parse') as parse_mock: - parse_mock.side_effect = yaql.exceptions.YaqlLexicalException + with mock.patch('murano.dsl.yaql_integration.parse') as parse_mock: + parse_mock.side_effect = exceptions.YaqlLexicalException self.assertFalse(expr.match('')) diff --git a/murano/tests/unit/test_heat_stack.py b/murano/tests/unit/test_heat_stack.py index ca03a9fd8..ce8b6b1ac 100644 --- a/murano/tests/unit/test_heat_stack.py +++ b/murano/tests/unit/test_heat_stack.py @@ -17,9 +17,12 @@ from heatclient.v1 import stacks import mock from murano.dsl import class_loader +from murano.dsl import constants +from murano.dsl import helpers from murano.dsl import murano_class from murano.dsl import object_store from murano.engine import client_manager +from murano.engine import environment from murano.engine.system import heat_stack from murano.tests.unit import base @@ -38,11 +41,12 @@ class TestHeatStack(base.MuranoTestCase): self.mock_object_store = mock.Mock(spec=object_store.ObjectStore) self.mock_object_store.class_loader = mock.Mock( spec=class_loader.MuranoClassLoader) - self.client_manager_mock = mock.Mock( - spec=client_manager.ClientManager) - - self.client_manager_mock.get_heat_client.return_value = \ + self.environment_mock = mock.Mock( + spec=environment.Environment) + client_manager_mock = mock.Mock(spec=client_manager.ClientManager) + client_manager_mock.get_heat_client.return_value = \ self.heat_client_mock + self.environment_mock.clients = client_manager_mock def test_push_adds_version(self): """Assert that if heat_template_version is omitted, it's added.""" @@ -54,16 +58,16 @@ class TestHeatStack(base.MuranoTestCase): status_get.return_value = 'NOT_FOUND' wait_st.return_value = {} - hs = heat_stack.HeatStack(self.mock_murano_class, - None, self.mock_object_store, None) - hs._name = 'test-stack' - hs._description = 'Generated by TestHeatStack' + context = {constants.CTX_ENVIRONMENT: self.environment_mock} + + with helpers.contextual(context): + hs = heat_stack.HeatStack( + 'test-stack', 'Generated by TestHeatStack') hs._template = {'resources': {'test': 1}} hs._files = {} hs._parameters = {} hs._applied = False - hs._clients = self.client_manager_mock - hs.push(None) + hs.push() expected_template = { 'heat_template_version': '2013-05-23', @@ -88,17 +92,15 @@ class TestHeatStack(base.MuranoTestCase): status_get.return_value = 'NOT_FOUND' wait_st.return_value = {} + context = {constants.CTX_ENVIRONMENT: self.environment_mock} - hs = heat_stack.HeatStack(self.mock_murano_class, - None, self.mock_object_store, None) - hs._clients = self.client_manager_mock - hs._name = 'test-stack' - hs._description = None + with helpers.contextual(context): + hs = heat_stack.HeatStack('test-stack', None) hs._template = {'resources': {'test': 1}} hs._files = {} hs._parameters = {} hs._applied = False - hs.push(None) + hs.push() expected_template = { 'heat_template_version': '2013-05-23', @@ -122,17 +124,16 @@ class TestHeatStack(base.MuranoTestCase): status_get.return_value = 'NOT_FOUND' wait_st.return_value = {} + context = {constants.CTX_ENVIRONMENT: self.environment_mock} - hs = heat_stack.HeatStack(self.mock_murano_class, - None, self.mock_object_store, None) - hs._clients = self.client_manager_mock - hs._name = 'test-stack' + with helpers.contextual(context): + hs = heat_stack.HeatStack('test-stack', None) hs._description = None hs._template = {'resources': {'test': 1}} hs._files = {"heatFile": "file"} hs._parameters = {} hs._applied = False - hs.push(None) + hs.push() expected_template = { 'heat_template_version': '2013-05-23', @@ -150,12 +151,11 @@ class TestHeatStack(base.MuranoTestCase): def test_update_wrong_template_version(self): """Template version other than expected should cause error.""" - hs = heat_stack.HeatStack(self.mock_murano_class, - None, self.mock_object_store, None) - hs._name = 'test-stack' - hs._description = 'Generated by TestHeatStack' + context = {constants.CTX_ENVIRONMENT: self.environment_mock} + with helpers.contextual(context): + hs = heat_stack.HeatStack( + 'test-stack', 'Generated by TestHeatStack') hs._template = {'resources': {'test': 1}} - hs.type.properties = {} invalid_template = { 'heat_template_version': 'something else' @@ -165,19 +165,18 @@ class TestHeatStack(base.MuranoTestCase): current.return_value = {} e = self.assertRaises(heat_stack.HeatStackError, - hs.updateTemplate, - None, + hs.update_template, invalid_template) err_msg = "Currently only heat_template_version 2013-05-23 "\ "is supported." self.assertEqual(err_msg, str(e)) # Check it's ok without a version - hs.updateTemplate(None, {}) + hs.update_template({}) expected = {'resources': {'test': 1}} self.assertEqual(expected, hs._template) # .. or with a good version - hs.updateTemplate(None, {'heat_template_version': '2013-05-23'}) + hs.update_template({'heat_template_version': '2013-05-23'}) expected['heat_template_version'] = '2013-05-23' self.assertEqual(expected, hs._template) diff --git a/requirements.txt b/requirements.txt index 54169678c..21f94b9df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ netaddr>=0.7.12 PyYAML>=3.1.0 jsonpatch>=1.1 keystonemiddleware>=2.0.0 -yaql!=0.3.0,>=0.2.7 # Apache 2.0 License +yaql>=1.0.0 # Apache 2.0 License # For paste.util.template used in keystone.common.template Paste