Migrate to new YAQL 1.0.0

All required changes were made: now global object is used,
arguments of registered functions were updated accordingly.

Also custom pickle objects were introduced to perform correct
serializations to keep data between requests.

Partially implements blueprint: migrate-to-yaql-vnext
Change-Id: If1e696dada7e0c2d4d593d5f27ca97f51604adfa

muranodashboard/common/cache.py

@@ -12,15 +12,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
 import functools
 import os
 
 from oslo_log import log as logging
 
+from muranodashboard.common import utils
 from muranodashboard.environments import consts
 
 
@@ -48,11 +45,11 @@ def _get_entry_path(app_id):
 
 
 def _load_from_file(file_name):
-    if os.path.isfile(file_name):
+    if os.path.isfile(file_name) and os.path.getsize(file_name) > 0:
         with open(file_name, 'rb') as f:
-            return pickle.load(f)
-    else:
-        return None
+            p = utils.CustomUnpickler(f)
+            return p.load()
+    return None
 
 
 def _save_to_file(file_name, content):
@@ -60,7 +57,8 @@ def _save_to_file(file_name, content):
     if not os.path.exists(dir_path):
         os.makedirs(dir_path)
     with open(file_name, 'wb') as f:
-        pickle.dump(content, f)
+        p = utils.CustomPickler(f)
+        p.dump(content)
 
 
 def with_cache(*dst_parts):

muranodashboard/common/utils.py

@@ -12,9 +12,16 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
 import bs4
 import string
 
+from muranodashboard.dynamic_ui import yaql_expression
+import yaql
+
 
 def parse_api_error(api_error_html):
     error_html = bs4.BeautifulSoup(api_error_html)
@@ -68,3 +75,41 @@ class BlankFormatter(string.Formatter):
             return kwargs.get(key, self.default)
         else:
             return string.Formatter.get_value(self, key, args, kwargs)
+
+
+class CustomPickler(object):
+    """Custom pickle object to perform correct serializing.
+
+    YAQL Engine is not serializable and it's not necessary to store
+    it in cache. This class replace YAQL Engine instance to string.
+    """
+
+    def __init__(self, file, protocol=0):
+        pickler = pickle.Pickler(file, protocol)
+        pickler.persistent_id = self.persistent_id
+        self.dump = pickler.dump
+        self.clear_memo = pickler.clear_memo
+
+    def persistent_id(self, obj):
+        if isinstance(obj, yaql.factory.YaqlEngine):
+            return "filtered:YaqlEngine"
+        else:
+            return None
+
+
+class CustomUnpickler(object):
+    """Custom pickle object to perform correct deserializing.
+
+    This class replace filtered YAQL Engine to the real instance.
+    """
+    def __init__(self, file):
+        unpickler = pickle.Unpickler(file)
+        unpickler.persistent_load = self.persistent_load
+        self.load = unpickler.load
+        self.noload = unpickler.noload
+
+    def persistent_load(self, obj_id):
+        if obj_id == 'filtered:YaqlEngine':
+            return yaql_expression.YAQL
+        else:
+            raise pickle.UnpicklingError('Invalid persistent id')

muranodashboard/dynamic_ui/fields.py

@@ -76,7 +76,7 @@ def make_yaql_validator(validator_property):
 
     def validator_func(value):
         context = yaql.create_context()
-        context.set_data(value)
+        context['$'] = value
         if not expr.evaluate(context=context):
             raise forms.ValidationError(message)
 

muranodashboard/dynamic_ui/forms.py

@@ -19,7 +19,7 @@ import types
 from django import forms
 from django.utils.translation import ugettext_lazy as _
 from oslo_log import log as logging
-import yaql
+from yaql import legacy
 
 import muranodashboard.dynamic_ui.fields as fields
 import muranodashboard.dynamic_ui.helpers as helpers
@@ -181,7 +181,7 @@ class ServiceConfigurationForm(UpdatableFieldsForm):
         super(ServiceConfigurationForm, self).__init__(*args, **kwargs)
 
         self.auto_id = '{0}_%s'.format(self.initial.get('app_id'))
-        self.context = yaql.create_context()
+        self.context = legacy.create_context()
         yaql_functions.register(self.context)
 
         self.finalize_fields()

muranodashboard/dynamic_ui/helpers.py

@@ -18,7 +18,6 @@ import types
 import uuid
 
 from django.core import validators
-from yaql import utils
 
 _LOCALIZABLE_KEYS = set(['label', 'help_text', 'error_messages'])
 
@@ -84,7 +83,7 @@ def recursive_apply(predicate, transformer, value, *args):
         elif isinstance(val, types.TupleType):
             return tuple([rec(v) for v in val])
         elif isinstance(val, types.GeneratorType):
-            return rec(utils.limit(val))
+            return rec(val)
         else:
             return val
 

muranodashboard/dynamic_ui/services.py

@@ -15,12 +15,12 @@
 import os
 import re
 import semantic_version
-import yaql
 
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 from oslo_log import log as logging
 import six
+from yaql import legacy
 
 from muranodashboard.api import packages as pkg_api
 from muranodashboard.catalog import forms as catalog_forms
@@ -69,7 +69,7 @@ class Service(object):
         else:
             self.application = application
 
-        self.context = yaql.create_context()
+        self.context = legacy.create_context()
         yaql_functions.register(self.context)
 
         self.forms = []
@@ -121,13 +121,13 @@ class Service(object):
                                                                  [])
 
     def extract_attributes(self):
-        self.context.set_data(self.cleaned_data)
+        self.context['$'] = self.cleaned_data
         for name, template in self.templates.iteritems():
-            self.context.set_data(template, name)
+            self.context[name] = template
         if semantic_version.Version.coerce(self.spec_version) \
                 >= semantic_version.Version.coerce('2.2'):
             management_form = catalog_forms.WorkflowManagementForm.name
-            name = self.context.get_data()[management_form]['application_name']
+            name = self.context['$'][management_form]['application_name']
             self.application['?']['name'] = name
         attributes = helpers.evaluate(self.application, self.context)
         return attributes

muranodashboard/dynamic_ui/yaql_expression.py

@@ -16,13 +16,23 @@ import re
 import types
 
 import yaql
-import yaql.exceptions
+from yaql.language import exceptions as yaql_exc
+
+
+def _set_up_yaql():
+    legacy_engine_options = {
+        'yaql.limitIterators': 100,
+        'yaql.memoryQuota': 20000
+    }
+    return yaql.YaqlFactory().create(options=legacy_engine_options)
+
+YAQL = _set_up_yaql()
 
 
 class YaqlExpression(object):
     def __init__(self, expression):
         self._expression = str(expression)
-        self._parsed_expression = yaql.parse(self._expression)
+        self._parsed_expression = YAQL(self._expression)
 
     def expression(self):
         return self._expression
@@ -40,12 +50,12 @@ class YaqlExpression(object):
         if re.match('^[\s\w\d.:]*$', expr):
             return False
         try:
-            yaql.parse(expr)
+            YAQL(expr)
             return True
-        except yaql.exceptions.YaqlGrammarException:
+        except yaql_exc.YaqlGrammarException:
             return False
-        except yaql.exceptions.YaqlLexicalException:
+        except yaql_exc.YaqlLexicalException:
             return False
 
-    def evaluate(self, data=None, context=None):
+    def evaluate(self, data=yaql.utils.NO_VALUE, context=None):
         return self._parsed_expression.evaluate(data=data, context=context)

muranodashboard/dynamic_ui/yaql_functions.py

@@ -17,25 +17,25 @@ import string
 import time
 import types
 
-import yaql.context
+from yaql.language import specs
+from yaql.language import yaqltypes
 
 from muranodashboard.catalog import forms as catalog_forms
 from muranodashboard.dynamic_ui import helpers
 
 
-@yaql.context.ContextAware()
-@yaql.context.EvalArg('times', types.IntType)
+@specs.parameter('times', int)
 def _repeat(context, template, times):
     for i in xrange(times):
-        context.set_data(i + 1, '$index')
-        yield helpers.evaluate(template(), context)
+        context['index'] = i + 1
+        yield helpers.evaluate(template, context)
 
 
 _random_string_counter = None
 
 
-@yaql.context.EvalArg('pattern', types.StringTypes)
-@yaql.context.EvalArg('number', types.IntType)
+@specs.parameter('pattern', yaqltypes.String())
+@specs.parameter('number', int)
 def _generate_hostname(pattern, number):
     """Generates hostname based on pattern
 
@@ -66,9 +66,8 @@ def _generate_hostname(pattern, number):
     return prefix + timestamp + suffix
 
 
-@yaql.context.ContextAware()
 def _name(context):
-    name = context.get_data()[
+    name = context.get_data[
         catalog_forms.WorkflowManagementForm.name]['application_name']
     return name
 

requirements.txt

@@ -6,11 +6,9 @@ pbr<2.0,>=1.4
 beautifulsoup4
 iso8601>=0.1.9
 six>=1.9.0
+python-muranoclient>=0.5.6
 PyYAML>=3.1.0
+yaql>=1.0.0 # Apache 2.0 License
 
 oslo.log>=1.8.0 # Apache-2.0
 semantic-version>=2.3.1
-
-# not listed in global requirements
-yaql!=0.3.0,>=0.2.7 # Apache 2.0 License
-python-muranoclient>=0.5.6