Introduced YAQL helpers

Added syntax sugar to simplify YAQL expressins.

Change-Id: Ifb5ace0302dcf4e041d3962271faec669d494252
Implements: blueprint computable-task-fields-yaql

nailgun/nailgun/expression/expression_parser.py

@@ -77,8 +77,6 @@ def t_error(t):
     t.lexer.skip(1)
 
 
-ply.lex.lex()
-
 expression = None
 
 precedence = (
@@ -143,10 +141,11 @@ def p_error(p):
     raise errors.ParseError("Syntax error at '%s'" % getattr(p, 'value', ''))
 
 
+lexer = ply.lex.lex()
 parser = ply.yacc.yacc(debug=False, write_tables=False)
 
 
 def parse(expr):
     global expression
     expression = expr
-    return parser.parse(expression.expression_text)
+    return parser.parse(expression.expression_text, lexer=lexer)

nailgun/nailgun/test/unit/test_yaql_ext.py

@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_serialization import jsonutils
+import six
+import yaml
+import yaql
+from yaql.language import exceptions
+
+from nailgun.test.base import BaseUnitTest
+from nailgun import yaql_ext
+
+
+class TestYaqlExt(BaseUnitTest):
+    @classmethod
+    def setUpClass(cls):
+        cls.variables = {
+            '$%new': {
+                'nodes': [
+                    {'uid': '1', 'role': 'compute'},
+                    {'uid': '2', 'role': 'controller'}
+                ],
+                'configs': {
+                    'nova': {
+                        'value': 1,
+                        'value2': None,
+                    }
+                },
+                'cluster': {
+                    'status': 'operational'
+                },
+            },
+            '$%old': {
+                'nodes': [
+                    {'uid': '1', 'role': 'controller'},
+                    {'uid': '2', 'role': 'compute'},
+                ],
+                'configs': {
+                    'nova': {
+                        'value': 2,
+                        'value2': None
+                    }
+                },
+                'cluster': {
+                    'status': 'new'
+                },
+            }
+        }
+        cls.variables['$'] = cls.variables['$%new']
+
+    def evaluate(self, expression, variables=None, engine=None):
+        context = yaql_ext.create_context(
+            add_datadiff=True, add_serializers=True
+        )
+        for k, v in six.iteritems(variables or self.variables):
+            context[k] = v
+
+        engine = engine or yaql_ext.create_engine()
+        parsed_exp = engine(expression)
+        return parsed_exp.evaluate(context=context)
+
+    def test_new(self):
+        result = self.evaluate(
+            'new($.nodes.where($.role=compute))'
+        )
+        self.assertEqual([{'uid': '1', 'role': 'compute'}], result)
+
+    def test_old(self):
+        result = self.evaluate(
+            'old($.nodes.where($.role=compute))'
+        )
+        self.assertEqual([{'uid': '2', 'role': 'compute'}], result)
+
+    def test_added(self):
+        self.assertEqual(
+            [{'uid': '1', 'role': 'compute'}],
+            self.evaluate('added($.nodes.where($.role=compute))')
+        )
+
+    def test_deleted(self):
+        self.assertItemsEqual(
+            [{'uid': '2', 'role': 'compute'}],
+            self.evaluate('deleted($.nodes.where($.role=compute))')
+        )
+
+    def test_changed(self):
+        self.assertTrue(self.evaluate('changed($.configs.nova.value)'))
+        self.assertFalse(self.evaluate('changed($.configs.nova.value2)'))
+
+    def test_added_if_no_old(self):
+        variables = self.variables.copy()
+        variables['$%old'] = {}
+        self.assertItemsEqual(
+            [{'uid': '1', 'role': 'compute'}],
+            self.evaluate('added($.nodes.where($.role=compute))', variables)
+        )
+
+    def test_delete_if_no_old(self):
+        variables = self.variables.copy()
+        variables['$%old'] = {}
+        self.assertIsNone(
+            self.evaluate('deleted($.nodes.where($.role=compute))', variables)
+        )
+
+    def test_changed_if_no_old(self):
+        variables = self.variables.copy()
+        variables['$%old'] = {}
+        self.assertTrue(
+            self.evaluate('changed($.configs.nova.value)', variables)
+        )
+        self.assertTrue(
+            self.evaluate('changed($.configs.nova.value2)', variables)
+        )
+
+    def test_undefined(self):
+        variables = self.variables.copy()
+        variables['$%old'] = {}
+        self.assertTrue(
+            self.evaluate('old($.configs.nova.value).isUndef()', variables),
+        )
+
+    def test_to_yaml(self):
+        expected = yaml.safe_dump(self.variables['$%new']['configs'])
+        actual = self.evaluate('$.configs.toYaml()')
+        self.assertEqual(expected, actual)
+
+    def test_to_json(self):
+        expected = jsonutils.dumps(self.variables['$%new']['configs'])
+        actual = self.evaluate('$.configs.toJson()')
+        self.assertEqual(expected, actual)
+
+    def test_limit_iterables(self):
+        engine = yaql.YaqlFactory().create({
+            'yaql.limitIterators': 1,
+            'yaql.convertTuplesToLists': True,
+            'yaql.convertSetsToLists': True
+        })
+        functions = ['added', 'deleted', 'changed']
+
+        expressions = ['$.nodes', '$.configs.nova']
+        for exp in expressions:
+            for func in functions:
+                with self.assertRaises(exceptions.CollectionTooLargeException):
+                    self.evaluate('{0}({1})'.format(func, exp), engine=engine)
+
+        expressions = ['$.configs.nova.value', '$.cluster.status']
+        for exp in expressions:
+            for func in functions:
+                self.evaluate('{0}({1})'.format(func, exp), engine=engine)

nailgun/nailgun/yaql_ext/__init__.py

@@ -0,0 +1,49 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import yaql
+
+from nailgun.yaql_ext import datadiff
+from nailgun.yaql_ext import serializers
+
+
+LIMIT_ITERATORS = 5000
+
+MEMORY_QUOTA = 20000
+
+_global_engine = None
+
+
+def create_context(add_serializers=False, add_datadiff=False, **kwargs):
+    context = yaql.create_context(**kwargs)
+    if add_serializers:
+        serializers.register(context)
+    if add_datadiff:
+        datadiff.register(context)
+    return context
+
+
+def create_engine():
+    global _global_engine
+
+    engine_options = {
+        'yaql.limitIterators': LIMIT_ITERATORS,
+        'yaql.memoryQuota': MEMORY_QUOTA,
+        'yaql.convertTuplesToLists': True,
+        'yaql.convertSetsToLists': True
+    }
+
+    if _global_engine is None:
+        _global_engine = yaql.YaqlFactory().create(engine_options)
+    return _global_engine

nailgun/nailgun/yaql_ext/datadiff.py

@@ -0,0 +1,85 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from yaql.language import specs
+from yaql.language import utils as yaqlutils
+from yaql.language import yaqltypes
+
+from nailgun.logger import logger
+from nailgun.utils import datadiff
+
+
+_UNDEFINED = object()
+
+
+@specs.parameter('expression', yaqltypes.Lambda())
+def get_new(expression, context):
+    return expression(context['$%new'])
+
+
+@specs.parameter('expression', yaqltypes.Lambda())
+def get_old(expression, context):
+    try:
+        return expression(context['$%old'])
+    except Exception as e:
+        # exception in evaluation on old data interprets as data changed
+        logger.debug('Cannot evaluate expression on old data: %s', e)
+        return _UNDEFINED
+
+
+@specs.parameter('expression', yaqltypes.Lambda())
+@specs.inject('finalizer', yaqltypes.Delegate('#finalize'))
+def changed(finalizer, expression, context):
+    new_data = finalizer(get_new(expression, context))
+    old_data = finalizer(get_old(expression, context))
+    return new_data != old_data
+
+
+def get_limited_if_need(data, engine):
+    if (yaqlutils.is_iterable(data) or yaqlutils.is_sequence(data) or
+            isinstance(data, (yaqlutils.MappingType, yaqlutils.SetType))):
+        return yaqlutils.limit_iterable(data, engine)
+    return data
+
+
+@specs.parameter('expression', yaqltypes.Lambda())
+def added(expression, context, engine):
+    new_data = get_limited_if_need(get_new(expression, context), engine)
+    old_data = get_limited_if_need(get_old(expression, context), engine)
+    if old_data is _UNDEFINED:
+        return new_data
+    return datadiff.diff(old_data, new_data).added
+
+
+@specs.parameter('expression', yaqltypes.Lambda())
+def deleted(expression, context, engine):
+    new_data = get_limited_if_need(get_new(expression, context), engine)
+    old_data = get_limited_if_need(get_old(expression, context), engine)
+    if old_data is not _UNDEFINED:
+        return datadiff.diff(old_data, new_data).deleted
+
+
+@specs.method
+@specs.inject('finalizer', yaqltypes.Delegate('#finalize'))
+def is_undef(finalizer, receiver):
+    return finalizer(receiver) is _UNDEFINED
+
+
+def register(context):
+    context.register_function(get_new, name='new')
+    context.register_function(get_old, name='old')
+    context.register_function(changed)
+    context.register_function(added)
+    context.register_function(deleted)
+    context.register_function(is_undef)

nailgun/nailgun/yaql_ext/serializers.py

@@ -0,0 +1,36 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from oslo_serialization import jsonutils
+import yaml
+from yaql.language import specs
+from yaql.language import yaqltypes
+
+
+@specs.method
+@specs.inject('finalizer', yaqltypes.Delegate('#finalize'))
+def to_yaml(finalizer, receiver):
+    return yaml.safe_dump(finalizer(receiver))
+
+
+@specs.method
+@specs.inject('finalizer', yaqltypes.Delegate('#finalize'))
+def to_json(finalizer, receiver):
+    return jsonutils.dumps(finalizer(receiver))
+
+
+def register(context):
+    context.register_function(to_yaml)
+    context.register_function(to_json)

nailgun/requirements.txt

@@ -46,3 +46,4 @@ stevedore>=1.5.0
 # the editable mode is broken
 # See: https://bugs.launchpad.net/fuel/+bug/1519727
 setuptools<=18.5
+yaql>=1.0.0

specs/fuel-nailgun.spec

@@ -55,6 +55,7 @@ Requires:    python-networkx-core >= 1.8.0
 Requires:    python-networkx-core < 1.10.0
 Requires:    python-cinderclient >= 1.0.7
 Requires:    pydot-ng >= 1.0.0
+Requires:    python-yaql >= 1.0.0
 # Workaroud for babel bug
 Requires:    pytz