From ebeb68b6dcd6e20a0b4e42ed49e6be05681f66ca Mon Sep 17 00:00:00 2001 From: Renat Akhmerov Date: Mon, 7 Apr 2014 14:14:36 +0700 Subject: [PATCH] BP mistral-actions-design * Renaming task 'input' property to 'parameters' according to BP * Fixing auth token generation related test * Small refactoring in expressions * Formatting changes Partially implements: blueprint mistral-actions-design Change-Id: I8659145d022ac4a2895c9d06f85f0961bf0b4f50 --- mistral/db/sqlalchemy/models.py | 2 +- mistral/engine/data_flow.py | 12 ++- mistral/engine/expressions.py | 18 +++- .../data_flow/task_with_two_dependencies.yaml | 4 +- .../data_flow/three_subsequent_tasks.yaml | 6 +- .../data_flow/two_dependent_tasks.yaml | 4 +- .../data_flow/two_subsequent_tasks.yaml | 4 +- .../tests/unit/db/test_sqlalchemy_db_api.py | 4 +- mistral/tests/unit/engine/test_data_flow.py | 82 +++++++++++-------- .../unit/engine/test_data_flow_module.py | 16 ++-- 10 files changed, 89 insertions(+), 63 deletions(-) diff --git a/mistral/db/sqlalchemy/models.py b/mistral/db/sqlalchemy/models.py index 8d1d4570..631f3517 100644 --- a/mistral/db/sqlalchemy/models.py +++ b/mistral/db/sqlalchemy/models.py @@ -98,7 +98,7 @@ class Task(mb.MistralBase): # Data Flow properties. in_context = sa.Column(st.JsonDictType()) - input = sa.Column(st.JsonDictType()) + parameters = sa.Column(st.JsonDictType()) output = sa.Column(st.JsonDictType()) # Runtime context like iteration_no of a repeater. diff --git a/mistral/engine/data_flow.py b/mistral/engine/data_flow.py index 2f9caa77..23e17c0a 100644 --- a/mistral/engine/data_flow.py +++ b/mistral/engine/data_flow.py @@ -27,10 +27,10 @@ LOG = logging.getLogger(__name__) CONF = cfg.CONF -def evaluate_task_input(task, context): +def evaluate_task_parameters(task, context): res = {} - params = task['task_spec'].get('input', {}) + params = task['task_spec'].get('parameters', {}) if not params: return res @@ -46,13 +46,13 @@ def prepare_tasks(tasks, context): # TODO(rakhmerov): Inbound context should be a merge of outbound # contexts of task dependencies, if any. task['in_context'] = context - task['input'] = evaluate_task_input(task, context) + task['parameters'] = evaluate_task_parameters(task, context) db_api.task_update(task['workbook_name'], task['execution_id'], task['id'], {'in_context': task['in_context'], - 'input': task['input']}) + 'parameters': task['parameters']}) def get_task_output(task, result): @@ -88,10 +88,12 @@ def get_outbound_context(task, output=None): def add_token_to_context(context, db_workbook): if context is None: context = {} + if CONF.pecan.auth_enable: workbook_ctx = trusts.create_context(db_workbook) if workbook_ctx: context.update(workbook_ctx.to_dict()) + return context @@ -110,10 +112,12 @@ def _modify_item(item, context): def apply_context(data, context): if not context: return data + if isinstance(data, dict): for key in data: data[key] = _modify_item(data[key], context) elif isinstance(data, list): for index, item in enumerate(data): data[index] = _modify_item(item, context) + return data diff --git a/mistral/engine/expressions.py b/mistral/engine/expressions.py index b331c515..54a2e245 100644 --- a/mistral/engine/expressions.py +++ b/mistral/engine/expressions.py @@ -17,6 +17,7 @@ import abc import re import yaql +import six from mistral.openstack.common import log as logging @@ -77,17 +78,22 @@ class InlineYAQLEvaluator(YAQLEvaluator): if super(InlineYAQLEvaluator, cls).is_expression(expression): return super(InlineYAQLEvaluator, cls).evaluate(expression, context) + result = expression found_expressions = cls.find_inline_expressions(expression) + if found_expressions: for expr in found_expressions: trim_expr = expr.strip("{}") evaluated = super(InlineYAQLEvaluator, cls).evaluate(trim_expr, context) result = result.replace(expr, evaluated or expr) - return result - else: - return expression + + return result + + @classmethod + def is_expression(cls, s): + return s @classmethod def find_inline_expressions(cls, s): @@ -99,4 +105,10 @@ _EVALUATOR = InlineYAQLEvaluator def evaluate(expression, context): + # Check if the passed value is expression so we don't need to do this + # every time on a caller side. + if not isinstance(expression, six.string_types) or \ + not _EVALUATOR.is_expression(expression): + return expression + return _EVALUATOR.evaluate(expression, context) diff --git a/mistral/tests/resources/data_flow/task_with_two_dependencies.yaml b/mistral/tests/resources/data_flow/task_with_two_dependencies.yaml index 770a0cf2..5925daf7 100644 --- a/mistral/tests/resources/data_flow/task_with_two_dependencies.yaml +++ b/mistral/tests/resources/data_flow/task_with_two_dependencies.yaml @@ -28,7 +28,7 @@ Workflow: tasks: build_full_name: action: MyService:build_full_name - input: + parameters: first_name: $.person.first_name last_name: $.person.last_name publish: @@ -42,6 +42,6 @@ Workflow: send_greeting: requires: [build_full_name, build_greeting] action: MyService:send_greeting - input: + parameters: f_name: $.f_name greet_msg: $.greet_msg diff --git a/mistral/tests/resources/data_flow/three_subsequent_tasks.yaml b/mistral/tests/resources/data_flow/three_subsequent_tasks.yaml index 396dd49a..431edd25 100644 --- a/mistral/tests/resources/data_flow/three_subsequent_tasks.yaml +++ b/mistral/tests/resources/data_flow/three_subsequent_tasks.yaml @@ -29,7 +29,7 @@ Workflow: tasks: build_full_name: action: MyService:build_full_name - input: + parameters: first_name: $.person.first_name last_name: $.person.last_name publish: @@ -38,7 +38,7 @@ Workflow: build_greeting: action: MyService:build_greeting - input: + parameters: full_name: $.f_name publish: greet_msg: greeting @@ -46,7 +46,7 @@ Workflow: send_greeting: action: MyService:send_greeting - input: + parameters: greeting: $.task.build_greeting.greeting publish: sent: greeting_sent diff --git a/mistral/tests/resources/data_flow/two_dependent_tasks.yaml b/mistral/tests/resources/data_flow/two_dependent_tasks.yaml index a2fcb7cc..a44aa8b5 100644 --- a/mistral/tests/resources/data_flow/two_dependent_tasks.yaml +++ b/mistral/tests/resources/data_flow/two_dependent_tasks.yaml @@ -26,7 +26,7 @@ Workflow: tasks: build_full_name: action: MyService:build_full_name - input: + parameters: first_name: $.person.first_name last_name: $.person.last_name publish: @@ -35,5 +35,5 @@ Workflow: build_greeting: requires: [build_full_name] action: MyService:build_greeting - input: + parameters: full_name: $.f_name diff --git a/mistral/tests/resources/data_flow/two_subsequent_tasks.yaml b/mistral/tests/resources/data_flow/two_subsequent_tasks.yaml index 5b7e6163..16534b90 100644 --- a/mistral/tests/resources/data_flow/two_subsequent_tasks.yaml +++ b/mistral/tests/resources/data_flow/two_subsequent_tasks.yaml @@ -26,7 +26,7 @@ Workflow: tasks: build_full_name: action: MyService:build_full_name - input: + parameters: first_name: $.person.first_name last_name: $.person.last_name publish: @@ -35,7 +35,7 @@ Workflow: build_greeting: action: MyService:build_greeting - input: + parameters: full_name: $.f_name publish: greet_msg: greeting diff --git a/mistral/tests/unit/db/test_sqlalchemy_db_api.py b/mistral/tests/unit/db/test_sqlalchemy_db_api.py index 443ac69f..01b995f2 100644 --- a/mistral/tests/unit/db/test_sqlalchemy_db_api.py +++ b/mistral/tests/unit/db/test_sqlalchemy_db_api.py @@ -230,7 +230,7 @@ TASKS = [ 'tags': ['deployment'], 'updated_at': None, 'in_context': None, - 'input': None, + 'parameters': None, 'output': None, 'task_runtime_context': None }, @@ -248,7 +248,7 @@ TASKS = [ 'tags': ['deployment'], 'updated_at': None, 'in_context': {'image_id': '123123'}, - 'input': {'image_id': '123123'}, + 'parameters': {'image_id': '123123'}, 'output': {'vm_id': '343123'}, 'task_runtime_context': None }, diff --git a/mistral/tests/unit/engine/test_data_flow.py b/mistral/tests/unit/engine/test_data_flow.py index fded879f..31927e39 100644 --- a/mistral/tests/unit/engine/test_data_flow.py +++ b/mistral/tests/unit/engine/test_data_flow.py @@ -82,16 +82,16 @@ class DataFlowTest(base.EngineTestCase): self.assertEqual(2, len(tasks)) - build_full_name_task =\ + build_full_name_task = \ self._assert_single_item(tasks, name='build_full_name') - build_greeting_task =\ + build_greeting_task = \ self._assert_single_item(tasks, name='build_greeting') # Check the first task. self.assertEqual(states.SUCCESS, build_full_name_task['state']) self.assertDictEqual(CONTEXT, build_full_name_task['in_context']) self.assertDictEqual({'first_name': 'John', 'last_name': 'Doe'}, - build_full_name_task['input']) + build_full_name_task['parameters']) self.assertDictEqual( { 'f_name': 'John Doe', @@ -111,7 +111,7 @@ class DataFlowTest(base.EngineTestCase): self.assertEqual('John Doe', build_greeting_task['in_context']['f_name']) self.assertDictEqual({'full_name': 'John Doe'}, - build_greeting_task['input']) + build_greeting_task['parameters']) self.assertDictEqual( { 'task': { @@ -148,18 +148,18 @@ class DataFlowTest(base.EngineTestCase): self.assertEqual(3, len(tasks)) - build_full_name_task =\ + build_full_name_task = \ self._assert_single_item(tasks, name='build_full_name') - build_greeting_task =\ + build_greeting_task = \ self._assert_single_item(tasks, name='build_greeting') - send_greeting_task =\ + send_greeting_task = \ self._assert_single_item(tasks, name='send_greeting') # Check the first task. self.assertEqual(states.SUCCESS, build_full_name_task['state']) self.assertDictEqual(CONTEXT, build_full_name_task['in_context']) self.assertDictEqual({'first_name': 'John', 'last_name': 'Doe'}, - build_full_name_task['input']) + build_full_name_task['parameters']) self.assertDictEqual( { 'f_name': 'John Doe', @@ -178,7 +178,7 @@ class DataFlowTest(base.EngineTestCase): self.assertEqual(states.SUCCESS, build_greeting_task['state']) self.assertEqual('John Doe', build_greeting_task['in_context']['f_name']) - self.assertDictEqual({}, build_greeting_task['input']) + self.assertDictEqual({}, build_greeting_task['parameters']) self.assertDictEqual( { 'greet_msg': 'Cheers!', @@ -208,7 +208,7 @@ class DataFlowTest(base.EngineTestCase): self.assertEqual(states.SUCCESS, send_greeting_task['state']) self.assertDictEqual(in_context, send_greeting_task['in_context']) self.assertDictEqual({'f_name': 'John Doe', 'greet_msg': 'Cheers!'}, - send_greeting_task['input']) + send_greeting_task['parameters']) self.assertDictEqual( { 'task': { @@ -240,16 +240,16 @@ class DataFlowTest(base.EngineTestCase): self.assertEqual(2, len(tasks)) - build_full_name_task =\ + build_full_name_task = \ self._assert_single_item(tasks, name='build_full_name') - build_greeting_task =\ + build_greeting_task = \ self._assert_single_item(tasks, name='build_greeting') # Check the first task. self.assertEqual(states.SUCCESS, build_full_name_task['state']) self.assertDictEqual(CONTEXT, build_full_name_task['in_context']) self.assertDictEqual({'first_name': 'John', 'last_name': 'Doe'}, - build_full_name_task['input']) + build_full_name_task['parameters']) self.assertDictEqual( { 'f_name': 'John Doe', @@ -269,7 +269,7 @@ class DataFlowTest(base.EngineTestCase): self.assertEqual('John Doe', build_greeting_task['in_context']['f_name']) self.assertDictEqual({'full_name': 'John Doe'}, - build_greeting_task['input']) + build_greeting_task['parameters']) self.assertDictEqual( { 'greet_msg': 'Hello, John Doe!', @@ -307,18 +307,18 @@ class DataFlowTest(base.EngineTestCase): self.assertEqual(3, len(tasks)) - build_full_name_task =\ + build_full_name_task = \ self._assert_single_item(tasks, name='build_full_name') - build_greeting_task =\ + build_greeting_task = \ self._assert_single_item(tasks, name='build_greeting') - send_greeting_task =\ + send_greeting_task = \ self._assert_single_item(tasks, name='send_greeting') # Check the first task. self.assertEqual(states.SUCCESS, build_full_name_task['state']) self.assertDictEqual(CONTEXT, build_full_name_task['in_context']) self.assertDictEqual({'first_name': 'John', 'last_name': 'Doe'}, - build_full_name_task['input']) + build_full_name_task['parameters']) self.assertDictEqual( { 'f_name': 'John Doe', @@ -338,7 +338,7 @@ class DataFlowTest(base.EngineTestCase): self.assertEqual('John Doe', build_greeting_task['in_context']['f_name']) self.assertDictEqual({'full_name': 'John Doe'}, - build_greeting_task['input']) + build_greeting_task['parameters']) self.assertDictEqual( { 'greet_msg': 'Hello, John Doe!', @@ -366,7 +366,7 @@ class DataFlowTest(base.EngineTestCase): self.assertEqual('Hello, John Doe!', send_greeting_task['in_context']['greet_msg']) self.assertDictEqual({'greeting': 'Hello, John Doe!'}, - send_greeting_task['input']) + send_greeting_task['parameters']) self.assertDictEqual( { 'sent': True, @@ -394,20 +394,30 @@ class DataFlowTest(base.EngineTestCase): mock.MagicMock( side_effect=base.EngineTestCase.mock_run_tasks)) def test_add_token_to_context(self): - cfg.CONF.pecan.auth_enable = True task_name = "create-vms" - workbook = create_workbook("test_rest.yaml") - db_api.workbook_update(workbook['name'], {'trust_id': '123'}) - execution = self.engine.start_workflow_execution(workbook['name'], - task_name, {}) - tasks = db_api.tasks_get(workbook['name'], execution['id']) - task = self._assert_single_item(tasks, name=task_name) - context = task['in_context'] - self.assertIn("auth_token", context) - self.assertEqual(TOKEN, context['auth_token']) - self.assertEqual(USER_ID, context["user_id"]) - self.engine.convey_task_result(workbook['name'], execution['id'], - task['id'], states.SUCCESS, {}) - execution = db_api.execution_get(workbook['name'], execution['id']) - self.assertEqual(states.SUCCESS, execution['state']) - cfg.CONF.pecan.auth_enable = False + + cfg.CONF.pecan.auth_enable = True + try: + workbook = create_workbook("test_rest.yaml") + db_api.workbook_update(workbook['name'], {'trust_id': '123'}) + + execution = self.engine.start_workflow_execution(workbook['name'], + task_name, {}) + tasks = db_api.tasks_get(workbook['name'], execution['id']) + + task = self._assert_single_item(tasks, name=task_name) + + context = task['in_context'] + + self.assertIn("auth_token", context) + self.assertEqual(TOKEN, context['auth_token']) + self.assertEqual(USER_ID, context["user_id"]) + + self.engine.convey_task_result(workbook['name'], execution['id'], + task['id'], states.SUCCESS, {}) + + execution = db_api.execution_get(workbook['name'], execution['id']) + + self.assertEqual(states.SUCCESS, execution['state']) + finally: + cfg.CONF.pecan.auth_enable = False diff --git a/mistral/tests/unit/engine/test_data_flow_module.py b/mistral/tests/unit/engine/test_data_flow_module.py index c58362be..d96bdb56 100644 --- a/mistral/tests/unit/engine/test_data_flow_module.py +++ b/mistral/tests/unit/engine/test_data_flow_module.py @@ -39,7 +39,7 @@ TASK = { 'execution_id': EXEC_ID, 'name': 'my_task', 'task_spec': { - 'input': { + 'parameters': { 'p1': 'My string', 'p2': '$.param3.param32', 'p3': '' @@ -63,12 +63,12 @@ TASK = { class DataFlowTest(base.DbTestCase): - def test_prepare_task_input(self): - input = data_flow.evaluate_task_input(TASK, CONTEXT) + def test_evaluate_task_parameters(self): + parameters = data_flow.evaluate_task_parameters(TASK, CONTEXT) - self.assertEqual(len(input), 3) - self.assertEqual(input['p1'], 'My string') - self.assertEqual(input['p2'], 'val32') + self.assertEqual(len(parameters), 3) + self.assertEqual(parameters['p1'], 'My string') + self.assertEqual(parameters['p2'], 'val32') def test_prepare_tasks(self): task = db_api.task_create(WB_NAME, EXEC_ID, TASK.copy()) @@ -79,7 +79,7 @@ class DataFlowTest(base.DbTestCase): db_task = db_api.task_get(WB_NAME, EXEC_ID, tasks[0]['id']) self.assertDictEqual(db_task['in_context'], CONTEXT) - self.assertDictEqual(db_task['input'], { + self.assertDictEqual(db_task['parameters'], { 'p1': 'My string', 'p2': 'val32', 'p3': '' @@ -104,7 +104,7 @@ class DataFlowTest(base.DbTestCase): modified_task = data_flow.apply_context(task_spec_dict, CONTEXT) self.assertDictEqual( { - 'input': { + 'parameters': { 'p1': 'My string', 'p2': 'val32', 'p3': ''