Moving to YAQL 1.0

Implements blueprint yaql-v1-0

 * Now functions 'env()' and 'execution()' should
 be used instead of '$.__env' and '$.__execution'
 correspondingly.

TODO:
 - Update examples in mistral-extra

Change-Id: Ic96fe10966bf5d27898c08d70914eb294b7efe2f

mistral/expressions.py

@@ -20,14 +20,15 @@ import re
 
 from oslo_log import log as logging
 import six
-import yaql
+from yaql.language import exceptions as yaql_exc
-from yaql import exceptions as yaql_exc
+from yaql.language import factory
 
 from mistral import exceptions as exc
-from mistral import yaql_utils
+from mistral.utils import yaql_utils
 
 
 LOG = logging.getLogger(__name__)
+YAQL_ENGINE = factory.YaqlFactory().create()
 
 
 class Evaluator(object):
@@ -75,7 +76,7 @@ class YAQLEvaluator(Evaluator):
         LOG.debug("Validating YAQL expression [expression='%s']", expression)
 
         try:
-            yaql.parse(expression)
+            YAQL_ENGINE(expression)
         except (yaql_exc.YaqlException, KeyError, ValueError, TypeError) as e:
             raise exc.YaqlEvaluationException(e.message)
 
@@ -85,9 +86,8 @@ class YAQLEvaluator(Evaluator):
                   % (expression, data_context))
 
         try:
-            result = yaql.parse(expression).evaluate(
+            result = YAQL_ENGINE(expression).evaluate(
-                data=data_context,
+                context=yaql_utils.get_yaql_context(data_context)
-                context=yaql_utils.create_yaql_context()
             )
         except (yaql_exc.YaqlException, KeyError, ValueError, TypeError) as e:
             raise exc.YaqlEvaluationException(

mistral/tests/unit/api/v2/test_environment.py

@@ -36,7 +36,7 @@ VARIABLES = {
     'verbose': True,
     '__actions': {
         'std.sql': {
-            'conn': 'mysql://admin:secrete@{$.__env.host}/{$.__env.db}'
+            'conn': 'mysql://admin:secrete@<% env().host %>/<% env().db %>'
         }
     }
 }

mistral/tests/unit/engine/test_dataflow.py

@@ -56,7 +56,7 @@ class DataFlowEngineTest(engine_test_base.EngineTestCase):
 
             task3:
               publish:
-                result: "<% $.hi %>, <% $.to %>! Your <% $.__env.from %>."
+                result: "<% $.hi %>, <% $.to %>! Your <% env().from %>."
         """
 
         wf_service.create_workflows(linear_wf)
@@ -113,7 +113,7 @@ class DataFlowEngineTest(engine_test_base.EngineTestCase):
 
             task3:
               publish:
-                result: "<% $.hi %>, <% $.to %>! Your <% $.__env.from %>."
+                result: "<% $.hi %>, <% $.to %>! Your <% env().from %>."
                 progress: "completed task3"
               on-success:
                 - notify
@@ -338,7 +338,7 @@ class DataFlowEngineTest(engine_test_base.EngineTestCase):
 
             task4:
               publish:
-                result: "<% $.greeting %>, <% $.to %>! <% $.__env.from %>."
+                result: "<% $.greeting %>, <% $.to %>! <% env().from %>."
         """
 
         wf_service.create_workflows(var_overwrite_wf)

mistral/tests/unit/engine/test_default_engine.py

@@ -194,8 +194,8 @@ class DefaultEngineTest(base.DbTestCase):
 
     def test_start_workflow_with_adhoc_env(self):
         wf_input = {
-            'param1': '<% $.__env.key1 %>',
+            'param1': '<% env().key1 %>',
-            'param2': '<% $.__env.key2 %>'
+            'param2': '<% env().key2 %>'
         }
         env = ENVIRONMENT['variables']
 
@@ -215,8 +215,8 @@ class DefaultEngineTest(base.DbTestCase):
     @mock.patch.object(db_api, "get_environment", MOCK_ENVIRONMENT)
     def test_start_workflow_with_saved_env(self):
         wf_input = {
-            'param1': '<% $.__env.key1 %>',
+            'param1': '<% env().key1 %>',
-            'param2': '<% $.__env.key2 %>'
+            'param2': '<% env().key2 %>'
         }
         env = ENVIRONMENT['variables']
 
@@ -238,7 +238,7 @@ class DefaultEngineTest(base.DbTestCase):
         self.assertRaises(exc.NotFoundException,
                           self.engine.start_workflow,
                           'wb.wf',
-                          {'param1': '<% $.__env.key1 %>'},
+                          {'param1': '<% env().key1 %>'},
                           env='foo',
                           task_name='task2')
 
@@ -246,7 +246,7 @@ class DefaultEngineTest(base.DbTestCase):
         self.assertRaises(ValueError,
                           self.engine.start_workflow,
                           'wb.wf',
-                          {'param1': '<% $.__env.key1 %>'},
+                          {'param1': '<% env().key1 %>'},
                           env=True,
                           task_name='task2')
 

mistral/tests/unit/engine/test_direct_workflow_rerun.py

@@ -57,7 +57,7 @@ workflows:
     type: direct
     tasks:
       t1:
-        with-items: i in <% range(0, 3).list() %>
+        with-items: i in <% list(range(0, 3)) %>
         action: std.echo output="Task 1.<% $.i %>"
         publish:
           v1: <% $.t1 %>

mistral/tests/unit/engine/test_environment.py

@@ -51,14 +51,14 @@ workflows:
     tasks:
       task1:
         action: std.echo output=<% $.param1 %>
-        target: <% $.__env.var1 %>
+        target: <% env().var1 %>
         publish:
           result1: <% $.task1 %>
 
       task2:
         requires: [task1]
         action: std.echo output="'<% $.result1 %> & <% $.param2 %>'"
-        target: <% $.__env.var1 %>
+        target: <% env().var1 %>
         publish:
           final_result: <% $.task2 %>
 
@@ -70,11 +70,11 @@ workflows:
       task1:
         workflow: wf1
         input:
-          param1: <% $.__env.var2 %>
+          param1: <% env().var2 %>
-          param2: <% $.__env.var3 %>
+          param2: <% env().var3 %>
           task_name: task2
         publish:
-          slogan: "<% $.task1.final_result %> is a cool <% $.__env.var4 %>!"
+          slogan: "<% $.task1.final_result %> is a cool <% env().var4 %>!"
 """
 
 
@@ -191,7 +191,7 @@ class SubworkflowsTest(base.EngineTestCase):
         env = {
             'var1': TARGET,
             'var2': 'Bonnie',
-            'var3': '<% $.__env.var5 %>',
+            'var3': '<% env().var5 %>',
             'var4': 'movie',
             'var5': 'Clyde'
         }

mistral/tests/unit/engine/test_join.py

@@ -319,8 +319,8 @@ class JoinEngineTest(base.EngineTestCase):
               action: std.echo
               input:
                 output: |
-                  <% result1 in $ %>,<% result2 in $ %>,
+                  <% result1 in $.keys() %>,<% result2 in $.keys() %>,
-                  <% result3 in $ %>,<% result4 in $ %>
+                  <% result3 in $.keys() %>,<% result4 in $.keys() %>
               publish:
                 result5: <% $.task5 %>
         """
@@ -390,8 +390,8 @@ class JoinEngineTest(base.EngineTestCase):
               action: std.echo
               input:
                 output: |
-                  <% result1 in $ %>,<% result2 in $ %>,
+                  <% result1 in $.keys() %>,<% result2 in $.keys() %>,
-                  <% result3 in $ %>
+                  <% result3 in $.keys() %>
               publish:
                 result4: <% $.task4 %>
         """

mistral/tests/unit/test_expressions.py

@@ -59,7 +59,12 @@ class YaqlEvaluatorTest(base.BaseTest):
         res = self._evaluator.evaluate("$.status = 'Invalid value'", DATA)
         self.assertFalse(res)
 
-        self.assertIsNone(self._evaluator.evaluate('$.wrong_key', DATA))
+        self.assertRaises(
+            exc.YaqlEvaluationException,
+            self._evaluator.evaluate,
+            '$.wrong_key',
+            DATA
+        )
 
         expression_str = 'invalid_expression_string'
         res = self._evaluator.evaluate(expression_str, DATA)
@@ -78,12 +83,12 @@ class YaqlEvaluatorTest(base.BaseTest):
         self.assertEqual('3', self._evaluator.evaluate('str($)', 3))
 
     def test_function_len(self):
-        self.assertEqual(3, self._evaluator.evaluate('$.len()', 'hey'))
+        self.assertEqual(3, self._evaluator.evaluate('len($)', 'hey'))
         data = [{'some': 'thing'}]
 
         self.assertEqual(
             1,
-            self._evaluator.evaluate('$[$.some = thing].len()', data)
+            self._evaluator.evaluate('$.where($.some = thing).len()', data)
         )
 
     def test_validate(self):
@@ -289,8 +294,8 @@ class ExpressionsTest(base.BaseTest):
             'verbose': True,
             '__actions': {
                 'std.sql': {
-                    'conn': 'mysql://admin:secrete@<% $.__env.host %>'
+                    'conn': 'mysql://admin:secrete@<% env().host %>'
-                            '/<% $.__env.db %>'
+                            '/<% env().db %>'
                 }
             }
         }

mistral/utils/yaql_utils.py

@@ -0,0 +1,52 @@
+# Copyright 2015 - Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import yaql
+
+
+ROOT_CONTEXT = None
+
+
+def get_yaql_context(data_context):
+    global ROOT_CONTEXT
+
+    if not ROOT_CONTEXT:
+        ROOT_CONTEXT = yaql.create_context()
+        _register_functions(ROOT_CONTEXT)
+
+    new_ctx = ROOT_CONTEXT.create_child_context()
+    new_ctx['$'] = data_context
+
+    if isinstance(data_context, dict):
+        new_ctx['__env'] = data_context.get('__env')
+        new_ctx['__execution'] = data_context.get('__execution')
+
+    return new_ctx
+
+
+def _register_functions(yaql_ctx):
+    yaql_ctx.register_function(env_)
+    yaql_ctx.register_function(execution_)
+
+
+# Additional convenience YAQL functions.
+# If a function name ends with underscore then it doesn't need to pass
+# the name of the function when context registers it.
+
+def env_(context):
+    return context['__env']
+
+
+def execution_(context):
+    return context['__execution']

mistral/yaql_utils.py

@@ -1,49 +0,0 @@
-# Copyright 2015 - Mirantis, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-
-import collections
-
-import yaql
-from yaql import context
-
-
-def create_yaql_context():
-    ctx = yaql.create_context()
-
-    _register_functions(ctx)
-
-    return ctx
-
-
-def _register_functions(yaql_ctx):
-    yaql_ctx.register_function(_sized_length, 'len')
-    yaql_ctx.register_function(_iterable_length, 'len')
-    yaql_ctx.register_function(to_str, 'str')
-
-
-# Additional convenience YAQL functions.
-
-
-@context.EvalArg('a', arg_type=collections.Sized)
-def _sized_length(a):
-    return len(a)
-
-
-@context.EvalArg('a', arg_type=collections.Iterable)
-def _iterable_length(a):
-    return sum(1 for i in a)
-
-
-def to_str(value):
-    return str(value())

requirements.txt

@@ -37,5 +37,5 @@ six>=1.9.0
 SQLAlchemy<1.1.0,>=0.9.7
 stevedore>=1.5.0 # Apache-2.0
 WSME>=0.7
-yaql>=0.2.7,!=0.3.0  # Apache 2.0 License
+yaql>=1.0.0  # Apache 2.0 License
 tooz>=0.16.0 # Apache-2.0