From 6b89f6918ea142c68d4c9eb846b38f91a87a212a Mon Sep 17 00:00:00 2001 From: Nikolay Mahotkin Date: Tue, 28 Jul 2015 15:58:00 +0300 Subject: [PATCH] Moving to YAQL 1.0 Implements blueprint yaql-v1-0 * Now functions 'env()' and 'execution()' should be used instead of '$.__env' and '$.__execution' correspondingly. TODO: - Update examples in mistral-extra Change-Id: Ic96fe10966bf5d27898c08d70914eb294b7efe2f --- mistral/expressions.py | 14 ++--- mistral/tests/unit/api/v2/test_environment.py | 2 +- mistral/tests/unit/engine/test_dataflow.py | 6 +-- .../tests/unit/engine/test_default_engine.py | 12 ++--- .../unit/engine/test_direct_workflow_rerun.py | 2 +- mistral/tests/unit/engine/test_environment.py | 12 ++--- mistral/tests/unit/engine/test_join.py | 8 +-- mistral/tests/unit/test_expressions.py | 15 ++++-- mistral/utils/yaql_utils.py | 52 +++++++++++++++++++ mistral/yaql_utils.py | 49 ----------------- requirements.txt | 2 +- 11 files changed, 91 insertions(+), 83 deletions(-) create mode 100644 mistral/utils/yaql_utils.py delete mode 100644 mistral/yaql_utils.py diff --git a/mistral/expressions.py b/mistral/expressions.py index fd20efa2..c203356b 100644 --- a/mistral/expressions.py +++ b/mistral/expressions.py @@ -20,14 +20,15 @@ import re from oslo_log import log as logging import six -import yaql -from yaql import exceptions as yaql_exc +from yaql.language import exceptions as yaql_exc +from yaql.language import factory from mistral import exceptions as exc -from mistral import yaql_utils +from mistral.utils import yaql_utils LOG = logging.getLogger(__name__) +YAQL_ENGINE = factory.YaqlFactory().create() class Evaluator(object): @@ -75,7 +76,7 @@ class YAQLEvaluator(Evaluator): LOG.debug("Validating YAQL expression [expression='%s']", expression) try: - yaql.parse(expression) + YAQL_ENGINE(expression) except (yaql_exc.YaqlException, KeyError, ValueError, TypeError) as e: raise exc.YaqlEvaluationException(e.message) @@ -85,9 +86,8 @@ class YAQLEvaluator(Evaluator): % (expression, data_context)) try: - result = yaql.parse(expression).evaluate( - data=data_context, - context=yaql_utils.create_yaql_context() + result = YAQL_ENGINE(expression).evaluate( + context=yaql_utils.get_yaql_context(data_context) ) except (yaql_exc.YaqlException, KeyError, ValueError, TypeError) as e: raise exc.YaqlEvaluationException( diff --git a/mistral/tests/unit/api/v2/test_environment.py b/mistral/tests/unit/api/v2/test_environment.py index 770857f2..25c0065b 100644 --- a/mistral/tests/unit/api/v2/test_environment.py +++ b/mistral/tests/unit/api/v2/test_environment.py @@ -36,7 +36,7 @@ VARIABLES = { 'verbose': True, '__actions': { 'std.sql': { - 'conn': 'mysql://admin:secrete@{$.__env.host}/{$.__env.db}' + 'conn': 'mysql://admin:secrete@<% env().host %>/<% env().db %>' } } } diff --git a/mistral/tests/unit/engine/test_dataflow.py b/mistral/tests/unit/engine/test_dataflow.py index 4f01fc13..ebe8dbd0 100644 --- a/mistral/tests/unit/engine/test_dataflow.py +++ b/mistral/tests/unit/engine/test_dataflow.py @@ -56,7 +56,7 @@ class DataFlowEngineTest(engine_test_base.EngineTestCase): task3: publish: - result: "<% $.hi %>, <% $.to %>! Your <% $.__env.from %>." + result: "<% $.hi %>, <% $.to %>! Your <% env().from %>." """ wf_service.create_workflows(linear_wf) @@ -113,7 +113,7 @@ class DataFlowEngineTest(engine_test_base.EngineTestCase): task3: publish: - result: "<% $.hi %>, <% $.to %>! Your <% $.__env.from %>." + result: "<% $.hi %>, <% $.to %>! Your <% env().from %>." progress: "completed task3" on-success: - notify @@ -338,7 +338,7 @@ class DataFlowEngineTest(engine_test_base.EngineTestCase): task4: publish: - result: "<% $.greeting %>, <% $.to %>! <% $.__env.from %>." + result: "<% $.greeting %>, <% $.to %>! <% env().from %>." """ wf_service.create_workflows(var_overwrite_wf) diff --git a/mistral/tests/unit/engine/test_default_engine.py b/mistral/tests/unit/engine/test_default_engine.py index 784d861c..313556ff 100644 --- a/mistral/tests/unit/engine/test_default_engine.py +++ b/mistral/tests/unit/engine/test_default_engine.py @@ -194,8 +194,8 @@ class DefaultEngineTest(base.DbTestCase): def test_start_workflow_with_adhoc_env(self): wf_input = { - 'param1': '<% $.__env.key1 %>', - 'param2': '<% $.__env.key2 %>' + 'param1': '<% env().key1 %>', + 'param2': '<% env().key2 %>' } env = ENVIRONMENT['variables'] @@ -215,8 +215,8 @@ class DefaultEngineTest(base.DbTestCase): @mock.patch.object(db_api, "get_environment", MOCK_ENVIRONMENT) def test_start_workflow_with_saved_env(self): wf_input = { - 'param1': '<% $.__env.key1 %>', - 'param2': '<% $.__env.key2 %>' + 'param1': '<% env().key1 %>', + 'param2': '<% env().key2 %>' } env = ENVIRONMENT['variables'] @@ -238,7 +238,7 @@ class DefaultEngineTest(base.DbTestCase): self.assertRaises(exc.NotFoundException, self.engine.start_workflow, 'wb.wf', - {'param1': '<% $.__env.key1 %>'}, + {'param1': '<% env().key1 %>'}, env='foo', task_name='task2') @@ -246,7 +246,7 @@ class DefaultEngineTest(base.DbTestCase): self.assertRaises(ValueError, self.engine.start_workflow, 'wb.wf', - {'param1': '<% $.__env.key1 %>'}, + {'param1': '<% env().key1 %>'}, env=True, task_name='task2') diff --git a/mistral/tests/unit/engine/test_direct_workflow_rerun.py b/mistral/tests/unit/engine/test_direct_workflow_rerun.py index 62e5d6c4..d9a5cd58 100644 --- a/mistral/tests/unit/engine/test_direct_workflow_rerun.py +++ b/mistral/tests/unit/engine/test_direct_workflow_rerun.py @@ -57,7 +57,7 @@ workflows: type: direct tasks: t1: - with-items: i in <% range(0, 3).list() %> + with-items: i in <% list(range(0, 3)) %> action: std.echo output="Task 1.<% $.i %>" publish: v1: <% $.t1 %> diff --git a/mistral/tests/unit/engine/test_environment.py b/mistral/tests/unit/engine/test_environment.py index 170aa523..9a404211 100644 --- a/mistral/tests/unit/engine/test_environment.py +++ b/mistral/tests/unit/engine/test_environment.py @@ -51,14 +51,14 @@ workflows: tasks: task1: action: std.echo output=<% $.param1 %> - target: <% $.__env.var1 %> + target: <% env().var1 %> publish: result1: <% $.task1 %> task2: requires: [task1] action: std.echo output="'<% $.result1 %> & <% $.param2 %>'" - target: <% $.__env.var1 %> + target: <% env().var1 %> publish: final_result: <% $.task2 %> @@ -70,11 +70,11 @@ workflows: task1: workflow: wf1 input: - param1: <% $.__env.var2 %> - param2: <% $.__env.var3 %> + param1: <% env().var2 %> + param2: <% env().var3 %> task_name: task2 publish: - slogan: "<% $.task1.final_result %> is a cool <% $.__env.var4 %>!" + slogan: "<% $.task1.final_result %> is a cool <% env().var4 %>!" """ @@ -191,7 +191,7 @@ class SubworkflowsTest(base.EngineTestCase): env = { 'var1': TARGET, 'var2': 'Bonnie', - 'var3': '<% $.__env.var5 %>', + 'var3': '<% env().var5 %>', 'var4': 'movie', 'var5': 'Clyde' } diff --git a/mistral/tests/unit/engine/test_join.py b/mistral/tests/unit/engine/test_join.py index ab8a3793..a7130450 100644 --- a/mistral/tests/unit/engine/test_join.py +++ b/mistral/tests/unit/engine/test_join.py @@ -319,8 +319,8 @@ class JoinEngineTest(base.EngineTestCase): action: std.echo input: output: | - <% result1 in $ %>,<% result2 in $ %>, - <% result3 in $ %>,<% result4 in $ %> + <% result1 in $.keys() %>,<% result2 in $.keys() %>, + <% result3 in $.keys() %>,<% result4 in $.keys() %> publish: result5: <% $.task5 %> """ @@ -390,8 +390,8 @@ class JoinEngineTest(base.EngineTestCase): action: std.echo input: output: | - <% result1 in $ %>,<% result2 in $ %>, - <% result3 in $ %> + <% result1 in $.keys() %>,<% result2 in $.keys() %>, + <% result3 in $.keys() %> publish: result4: <% $.task4 %> """ diff --git a/mistral/tests/unit/test_expressions.py b/mistral/tests/unit/test_expressions.py index 2d0a5173..e6b4f853 100644 --- a/mistral/tests/unit/test_expressions.py +++ b/mistral/tests/unit/test_expressions.py @@ -59,7 +59,12 @@ class YaqlEvaluatorTest(base.BaseTest): res = self._evaluator.evaluate("$.status = 'Invalid value'", DATA) self.assertFalse(res) - self.assertIsNone(self._evaluator.evaluate('$.wrong_key', DATA)) + self.assertRaises( + exc.YaqlEvaluationException, + self._evaluator.evaluate, + '$.wrong_key', + DATA + ) expression_str = 'invalid_expression_string' res = self._evaluator.evaluate(expression_str, DATA) @@ -78,12 +83,12 @@ class YaqlEvaluatorTest(base.BaseTest): self.assertEqual('3', self._evaluator.evaluate('str($)', 3)) def test_function_len(self): - self.assertEqual(3, self._evaluator.evaluate('$.len()', 'hey')) + self.assertEqual(3, self._evaluator.evaluate('len($)', 'hey')) data = [{'some': 'thing'}] self.assertEqual( 1, - self._evaluator.evaluate('$[$.some = thing].len()', data) + self._evaluator.evaluate('$.where($.some = thing).len()', data) ) def test_validate(self): @@ -289,8 +294,8 @@ class ExpressionsTest(base.BaseTest): 'verbose': True, '__actions': { 'std.sql': { - 'conn': 'mysql://admin:secrete@<% $.__env.host %>' - '/<% $.__env.db %>' + 'conn': 'mysql://admin:secrete@<% env().host %>' + '/<% env().db %>' } } } diff --git a/mistral/utils/yaql_utils.py b/mistral/utils/yaql_utils.py new file mode 100644 index 00000000..c9ba156a --- /dev/null +++ b/mistral/utils/yaql_utils.py @@ -0,0 +1,52 @@ +# Copyright 2015 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaql + + +ROOT_CONTEXT = None + + +def get_yaql_context(data_context): + global ROOT_CONTEXT + + if not ROOT_CONTEXT: + ROOT_CONTEXT = yaql.create_context() + _register_functions(ROOT_CONTEXT) + + new_ctx = ROOT_CONTEXT.create_child_context() + new_ctx['$'] = data_context + + if isinstance(data_context, dict): + new_ctx['__env'] = data_context.get('__env') + new_ctx['__execution'] = data_context.get('__execution') + + return new_ctx + + +def _register_functions(yaql_ctx): + yaql_ctx.register_function(env_) + yaql_ctx.register_function(execution_) + + +# Additional convenience YAQL functions. +# If a function name ends with underscore then it doesn't need to pass +# the name of the function when context registers it. + +def env_(context): + return context['__env'] + + +def execution_(context): + return context['__execution'] diff --git a/mistral/yaql_utils.py b/mistral/yaql_utils.py deleted file mode 100644 index e89ab3c4..00000000 --- a/mistral/yaql_utils.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2015 - Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import collections - -import yaql -from yaql import context - - -def create_yaql_context(): - ctx = yaql.create_context() - - _register_functions(ctx) - - return ctx - - -def _register_functions(yaql_ctx): - yaql_ctx.register_function(_sized_length, 'len') - yaql_ctx.register_function(_iterable_length, 'len') - yaql_ctx.register_function(to_str, 'str') - - -# Additional convenience YAQL functions. - - -@context.EvalArg('a', arg_type=collections.Sized) -def _sized_length(a): - return len(a) - - -@context.EvalArg('a', arg_type=collections.Iterable) -def _iterable_length(a): - return sum(1 for i in a) - - -def to_str(value): - return str(value()) diff --git a/requirements.txt b/requirements.txt index a58cfa16..0eab900e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,5 +37,5 @@ six>=1.9.0 SQLAlchemy<1.1.0,>=0.9.7 stevedore>=1.5.0 # Apache-2.0 WSME>=0.7 -yaql>=0.2.7,!=0.3.0 # Apache 2.0 License +yaql>=1.0.0 # Apache 2.0 License tooz>=0.16.0 # Apache-2.0