diff --git a/muranodashboard/common/cache.py b/muranodashboard/common/cache.py index af93719e3..a4385978d 100644 --- a/muranodashboard/common/cache.py +++ b/muranodashboard/common/cache.py @@ -12,15 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -try: - import cPickle as pickle -except ImportError: - import pickle import functools import os from oslo_log import log as logging +from muranodashboard.common import utils from muranodashboard.environments import consts @@ -48,11 +45,11 @@ def _get_entry_path(app_id): def _load_from_file(file_name): - if os.path.isfile(file_name): + if os.path.isfile(file_name) and os.path.getsize(file_name) > 0: with open(file_name, 'rb') as f: - return pickle.load(f) - else: - return None + p = utils.CustomUnpickler(f) + return p.load() + return None def _save_to_file(file_name, content): @@ -60,7 +57,8 @@ def _save_to_file(file_name, content): if not os.path.exists(dir_path): os.makedirs(dir_path) with open(file_name, 'wb') as f: - pickle.dump(content, f) + p = utils.CustomPickler(f) + p.dump(content) def with_cache(*dst_parts): diff --git a/muranodashboard/common/utils.py b/muranodashboard/common/utils.py index 00faf01c5..bb0710479 100644 --- a/muranodashboard/common/utils.py +++ b/muranodashboard/common/utils.py @@ -12,9 +12,16 @@ # License for the specific language governing permissions and limitations # under the License. +try: + import cPickle as pickle +except ImportError: + import pickle import bs4 import string +from muranodashboard.dynamic_ui import yaql_expression +import yaql + def parse_api_error(api_error_html): error_html = bs4.BeautifulSoup(api_error_html) @@ -68,3 +75,41 @@ class BlankFormatter(string.Formatter): return kwargs.get(key, self.default) else: return string.Formatter.get_value(self, key, args, kwargs) + + +class CustomPickler(object): + """Custom pickle object to perform correct serializing. + + YAQL Engine is not serializable and it's not necessary to store + it in cache. This class replace YAQL Engine instance to string. + """ + + def __init__(self, file, protocol=0): + pickler = pickle.Pickler(file, protocol) + pickler.persistent_id = self.persistent_id + self.dump = pickler.dump + self.clear_memo = pickler.clear_memo + + def persistent_id(self, obj): + if isinstance(obj, yaql.factory.YaqlEngine): + return "filtered:YaqlEngine" + else: + return None + + +class CustomUnpickler(object): + """Custom pickle object to perform correct deserializing. + + This class replace filtered YAQL Engine to the real instance. + """ + def __init__(self, file): + unpickler = pickle.Unpickler(file) + unpickler.persistent_load = self.persistent_load + self.load = unpickler.load + self.noload = unpickler.noload + + def persistent_load(self, obj_id): + if obj_id == 'filtered:YaqlEngine': + return yaql_expression.YAQL + else: + raise pickle.UnpicklingError('Invalid persistent id') diff --git a/muranodashboard/dynamic_ui/fields.py b/muranodashboard/dynamic_ui/fields.py index e35ca7c46..aec94b561 100644 --- a/muranodashboard/dynamic_ui/fields.py +++ b/muranodashboard/dynamic_ui/fields.py @@ -76,7 +76,7 @@ def make_yaql_validator(validator_property): def validator_func(value): context = yaql.create_context() - context.set_data(value) + context['$'] = value if not expr.evaluate(context=context): raise forms.ValidationError(message) diff --git a/muranodashboard/dynamic_ui/forms.py b/muranodashboard/dynamic_ui/forms.py index cc691ff89..7a3619cbb 100644 --- a/muranodashboard/dynamic_ui/forms.py +++ b/muranodashboard/dynamic_ui/forms.py @@ -19,7 +19,7 @@ import types from django import forms from django.utils.translation import ugettext_lazy as _ from oslo_log import log as logging -import yaql +from yaql import legacy import muranodashboard.dynamic_ui.fields as fields import muranodashboard.dynamic_ui.helpers as helpers @@ -181,7 +181,7 @@ class ServiceConfigurationForm(UpdatableFieldsForm): super(ServiceConfigurationForm, self).__init__(*args, **kwargs) self.auto_id = '{0}_%s'.format(self.initial.get('app_id')) - self.context = yaql.create_context() + self.context = legacy.create_context() yaql_functions.register(self.context) self.finalize_fields() diff --git a/muranodashboard/dynamic_ui/helpers.py b/muranodashboard/dynamic_ui/helpers.py index 4f5aa7688..2e4710754 100644 --- a/muranodashboard/dynamic_ui/helpers.py +++ b/muranodashboard/dynamic_ui/helpers.py @@ -18,7 +18,6 @@ import types import uuid from django.core import validators -from yaql import utils _LOCALIZABLE_KEYS = set(['label', 'help_text', 'error_messages']) @@ -84,7 +83,7 @@ def recursive_apply(predicate, transformer, value, *args): elif isinstance(val, types.TupleType): return tuple([rec(v) for v in val]) elif isinstance(val, types.GeneratorType): - return rec(utils.limit(val)) + return rec(val) else: return val diff --git a/muranodashboard/dynamic_ui/services.py b/muranodashboard/dynamic_ui/services.py index b86e27965..977e63de4 100644 --- a/muranodashboard/dynamic_ui/services.py +++ b/muranodashboard/dynamic_ui/services.py @@ -15,12 +15,12 @@ import os import re import semantic_version -import yaql from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ from oslo_log import log as logging import six +from yaql import legacy from muranodashboard.api import packages as pkg_api from muranodashboard.catalog import forms as catalog_forms @@ -69,7 +69,7 @@ class Service(object): else: self.application = application - self.context = yaql.create_context() + self.context = legacy.create_context() yaql_functions.register(self.context) self.forms = [] @@ -121,13 +121,13 @@ class Service(object): []) def extract_attributes(self): - self.context.set_data(self.cleaned_data) + self.context['$'] = self.cleaned_data for name, template in self.templates.iteritems(): - self.context.set_data(template, name) + self.context[name] = template if semantic_version.Version.coerce(self.spec_version) \ >= semantic_version.Version.coerce('2.2'): management_form = catalog_forms.WorkflowManagementForm.name - name = self.context.get_data()[management_form]['application_name'] + name = self.context['$'][management_form]['application_name'] self.application['?']['name'] = name attributes = helpers.evaluate(self.application, self.context) return attributes diff --git a/muranodashboard/dynamic_ui/yaql_expression.py b/muranodashboard/dynamic_ui/yaql_expression.py index 453af248a..49a5c13bf 100644 --- a/muranodashboard/dynamic_ui/yaql_expression.py +++ b/muranodashboard/dynamic_ui/yaql_expression.py @@ -16,13 +16,23 @@ import re import types import yaql -import yaql.exceptions +from yaql.language import exceptions as yaql_exc + + +def _set_up_yaql(): + legacy_engine_options = { + 'yaql.limitIterators': 100, + 'yaql.memoryQuota': 20000 + } + return yaql.YaqlFactory().create(options=legacy_engine_options) + +YAQL = _set_up_yaql() class YaqlExpression(object): def __init__(self, expression): self._expression = str(expression) - self._parsed_expression = yaql.parse(self._expression) + self._parsed_expression = YAQL(self._expression) def expression(self): return self._expression @@ -40,12 +50,12 @@ class YaqlExpression(object): if re.match('^[\s\w\d.:]*$', expr): return False try: - yaql.parse(expr) + YAQL(expr) return True - except yaql.exceptions.YaqlGrammarException: + except yaql_exc.YaqlGrammarException: return False - except yaql.exceptions.YaqlLexicalException: + except yaql_exc.YaqlLexicalException: return False - def evaluate(self, data=None, context=None): + def evaluate(self, data=yaql.utils.NO_VALUE, context=None): return self._parsed_expression.evaluate(data=data, context=context) diff --git a/muranodashboard/dynamic_ui/yaql_functions.py b/muranodashboard/dynamic_ui/yaql_functions.py index 6e825a8fe..5530dfc13 100644 --- a/muranodashboard/dynamic_ui/yaql_functions.py +++ b/muranodashboard/dynamic_ui/yaql_functions.py @@ -17,25 +17,25 @@ import string import time import types -import yaql.context +from yaql.language import specs +from yaql.language import yaqltypes from muranodashboard.catalog import forms as catalog_forms from muranodashboard.dynamic_ui import helpers -@yaql.context.ContextAware() -@yaql.context.EvalArg('times', types.IntType) +@specs.parameter('times', int) def _repeat(context, template, times): for i in xrange(times): - context.set_data(i + 1, '$index') - yield helpers.evaluate(template(), context) + context['index'] = i + 1 + yield helpers.evaluate(template, context) _random_string_counter = None -@yaql.context.EvalArg('pattern', types.StringTypes) -@yaql.context.EvalArg('number', types.IntType) +@specs.parameter('pattern', yaqltypes.String()) +@specs.parameter('number', int) def _generate_hostname(pattern, number): """Generates hostname based on pattern @@ -66,9 +66,8 @@ def _generate_hostname(pattern, number): return prefix + timestamp + suffix -@yaql.context.ContextAware() def _name(context): - name = context.get_data()[ + name = context.get_data[ catalog_forms.WorkflowManagementForm.name]['application_name'] return name diff --git a/requirements.txt b/requirements.txt index 88b99da5b..2027a2c50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,11 +6,9 @@ pbr<2.0,>=1.4 beautifulsoup4 iso8601>=0.1.9 six>=1.9.0 +python-muranoclient>=0.5.6 PyYAML>=3.1.0 +yaql>=1.0.0 # Apache 2.0 License oslo.log>=1.8.0 # Apache-2.0 semantic-version>=2.3.1 - -# not listed in global requirements -yaql!=0.3.0,>=0.2.7 # Apache 2.0 License -python-muranoclient>=0.5.6