openstack/deb-mistral
Code Issues Proposed changes

BP mistral-actions-design (switch to new design)

Browse Source 
* Refactored action_factory.py (adhoc actions into a separate method)
* Fixed task db model (service_spec -> action_spec)
* Fixed workbook model (tasks, actions, Service -> Namespace)
* Fixed all tests related to DSL syntax changes

TODO:
* Fix all samples in mistral-extra
* Action plugin architecture

Change-Id: Iac6317dde894f17c81332a8c9a2397b70448f6b2
This commit is contained in:
Renat Akhmerov 2014-04-14 13:52:01 +07:00
parent 4c1e5a2a80
commit b2a072f35c
35 changed files with 383 additions and 498 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
mistral/actions/action_factory.py
View File

@ -92,57 +92,70 @@ def _has_action_context_param(action_cls):
return _ACTION_CTX_PARAM in arg_spec.args
def _create_adhoc_action(db_task):
task_spec = tasks.TaskSpec(db_task['task_spec'])
full_action_name = task_spec.get_full_action_name()
# TODO(rakhmerov): Fix model attributes during refactoring.
# TODO(rakhmerov): ActionSpec should be used instead.
adhoc_action_spec = db_task['action_spec']
if not adhoc_action_spec:
return None
LOG.info('Using ad-hoc action [action=%s, db_task=%s]' %
(full_action_name, db_task))
# Create an ad-hoc action.
base_cls = get_action_class(adhoc_action_spec['class'])
action_context = None
if _has_action_context_param(base_cls):
action_context = _get_action_context(db_task)
if not base_cls:
msg = 'Ad-hoc action base class is not registered ' \
'[workbook_name=%s, action=%s, base_class=%s]' % \
(db_task['workbook_name'], full_action_name, base_cls)
raise exc.ActionException(msg)
action_params = db_task['parameters'] or {}
return std_actions.AdHocAction(action_context,
base_cls,
adhoc_action_spec,
**action_params)
def create_action(db_task):
task_spec = tasks.TaskSpec(db_task['task_spec'])
full_action_name = task_spec.get_full_action_name()
action_cls = get_action_class(full_action_name)
adhoc_action_spec = None
if not action_cls:
# If action is not found in registered actions try to find ad-hoc
# action definition.
# TODO(rakhmerov): Fix model attributes during refactoring.
# TODO(rakhmerov): ActionSpec should be used instead.
adhoc_action_spec = db_task['action_spec']
action = _create_adhoc_action(db_task)
if adhoc_action_spec:
LOG.info('Using ad-hoc action [action=%s, db_task=%s]' %
(full_action_name, db_task))
action_cls = std_actions.AdHocAction
if action:
return action
else:
msg = 'Unknown action [workbook_name=%s, action=%s]' % \
(db_task['workbook_name'], full_action_name)
raise exc.ActionException(msg)
if not action_cls:
msg = 'Unknown action [workbook_name=%s, action=%s]' % \
(db_task['workbook_name'], full_action_name)
raise exc.ActionException(msg)
action_params = db_task['parameters'].copy()
action_params = db_task['parameters'] or {}
if _has_action_context_param(action_cls):
action_params[_ACTION_CTX_PARAM] = _get_action_context(db_task)
try:
if not adhoc_action_spec:
# Create a regular action from a registered class.
action = action_cls(**action_params)
else:
# Create an ad-hoc action.
base_cls = get_action_class(adhoc_action_spec['class'])
if not base_cls:
msg = 'Ad-hoc action base class is not registered ' \
'[workbook_name=%s, action=%s, base_class=%s]' % \
(db_task['workbook_name'], full_action_name, base_cls)
raise exc.ActionException(msg)
action = action_cls(base_cls, adhoc_action_spec, **action_params)
return action_cls(**action_params)
except Exception as e:
raise exc.ActionException('Failed to create action [db_task=%s]: %s' %
(db_task, e))
return action
# Registering standard actions on module load.
_register_standard_action_classes()

15
mistral/actions/std_actions.py
View File

@ -90,7 +90,7 @@ class HTTPAction(base.Action):
class MistralHTTPAction(HTTPAction):
def __init__(self, action_context, url, params, method,
def __init__(self, action_context, url, params={}, method="GET",
headers={}, body={}):
headers.update({
'Mistral-Workbook-Name': action_context['workbook_name'],
@ -101,6 +101,9 @@ class MistralHTTPAction(HTTPAction):
super(MistralHTTPAction, self).__init__(url, params, method,
headers, body)
def is_sync(self):
return False
class SendEmailAction(base.Action):
def __init__(self, params, settings):
@ -156,11 +159,16 @@ class SendEmailAction(base.Action):
class AdHocAction(base.Action):
def __init__(self, base_action_cls, action_spec, **params):
def __init__(self, action_context,
base_action_cls, action_spec, **params):
self.base_action_cls = base_action_cls
self.action_spec = action_spec
base_params = self._convert_params(params)
if action_context:
base_params['action_context'] = action_context
self.base_action = base_action_cls(**base_params)
def _convert_params(self, params):
@ -190,6 +198,9 @@ class AdHocAction(base.Action):
else:
return expr.evaluate(transformer, expr_ctx)
def is_sync(self):
return self.base_action.is_sync()
def run(self):
return self._convert_result(self.base_action.run())

2
mistral/db/sqlalchemy/models.py
View File

@ -92,7 +92,7 @@ class Task(mb.MistralBase):
execution_id = sa.Column(sa.String(36))
description = sa.Column(sa.String())
task_spec = sa.Column(st.JsonDictType())
service_spec = sa.Column(st.JsonDictType())
action_spec = sa.Column(st.JsonDictType())
state = sa.Column(sa.String(20))
tags = sa.Column(st.JsonListType())

19
mistral/engine/abstract_engine.py
View File

@ -42,9 +42,9 @@ class AbstractEngine(object):
def start_workflow_execution(cls, workbook_name, task_name, context):
db_api.start_tx()
workbook = cls._get_workbook(workbook_name)
# Persist execution and tasks in DB.
try:
workbook = cls._get_workbook(workbook_name)
execution = cls._create_execution(workbook_name,
task_name,
context)
@ -81,8 +81,8 @@ class AbstractEngine(object):
task_id, state, result):
db_api.start_tx()
workbook = cls._get_workbook(workbook_name)
try:
workbook = cls._get_workbook(workbook_name)
#TODO(rakhmerov): validate state transition
task = db_api.task_get(workbook_name, execution_id, task_id)
@ -179,20 +179,26 @@ class AbstractEngine(object):
@classmethod
def _create_tasks(cls, task_list, workbook, workbook_name, execution_id):
tasks = []
for task in task_list:
state, task_runtime_context = repeater.get_task_runtime(task)
service_spec = workbook.services.get(task.get_action_service())
action_ns = workbook.namespaces.get(task.get_action_namespace())
action_spec = \
action_ns.actions.get(task.get_action_name())
db_task = db_api.task_create(workbook_name, execution_id, {
"name": task.name,
"requires": task.requires,
"task_spec": task.to_dict(),
"service_spec": {} if not service_spec else
service_spec.to_dict(),
"action_spec": {} if not action_spec
else action_spec.to_dict(),
"state": state,
"tags": task.get_property("tags", None),
"task_runtime_context": task_runtime_context
})
tasks.append(db_task)
return tasks
@classmethod
@ -231,6 +237,7 @@ class AbstractEngine(object):
outbound_context = data_flow.get_outbound_context(task, task_output)
state, exec_flow_context = repeater.get_task_runtime(
task_spec, state, outbound_context, task_runtime_context)
# Update the task.
update_values = {"state": state,
"output": task_output,
@ -257,6 +264,7 @@ class AbstractEngine(object):
workbook_name = task['workbook_name']
execution_id = task['execution_id']
execution = db_api.execution_get(workbook_name, execution_id)
# Change state from DELAYED to IDLE to unblock processing.
db_task = db_api.task_update(workbook_name,
execution_id,
@ -273,6 +281,7 @@ class AbstractEngine(object):
task_spec = workbook.tasks.get(task['name'])
retries, break_on, delay_sec = task_spec.get_repeat_task_parameters()
if delay_sec > 0:
# Run the task after the specified delay.
eventlet.spawn_after(delay_sec, run_delayed_task)

1
mistral/engine/actions/__init__.py
View File

@ -0,0 +1 @@
#TODO(rakhmerov): Remove this package after refactoring.

2
mistral/engine/actions/action_factory.py
View File

@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#TODO(rakhmerov): Remove this module after refactoring.
import copy
from mistral.engine.actions import actions

2
mistral/engine/actions/action_helper.py
View File

@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#TODO(rakhmerov): Remove this module after refactoring.
from mistral.engine.actions import action_types as a_t

1
mistral/engine/actions/action_types.py
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#TODO(rakhmerov): Remove this module after refactoring.
"""Valid action types."""

2
mistral/engine/actions/actions.py
View File

@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#TODO(rakhmerov): Remove this module after refactoring.
#TODO(dzimine):separate actions across different files/modules
import abc

17
mistral/engine/scalable/executor/server.py
View File

@ -19,8 +19,7 @@ from mistral.db import api as db_api
from mistral import exceptions as exc
from mistral.engine import engine
from mistral.engine import states
from mistral.engine.actions import action_factory as a_f
from mistral.engine.actions import action_helper as a_h
from mistral.actions import action_factory as a_f
LOG = logging.getLogger(__name__)
@ -35,13 +34,13 @@ class Executor(object):
:type task: dict
"""
LOG.info("Starting task action [task_id=%s, "
"action='%s', service='%s'" %
"action='%s', action_spec='%s'" %
(task['id'], task['task_spec']['action'],
task['service_spec']))
task['action_spec']))
action = a_f.create_action(task)
if a_h.is_task_synchronous(task):
if action.is_sync():
try:
state, result = states.SUCCESS, action.run()
except exc.ActionException:
@ -101,6 +100,7 @@ class Executor(object):
task = kwargs.get('task', None)
if not task:
raise Exception('No task is provided to the executor.')
LOG.info("Received a task: %s" % task)
db_task = db_api.task_get(task['workbook_name'],
@ -123,7 +123,8 @@ class Executor(object):
task['execution_id'],
task['id'],
{'state': states.RUNNING})
self._do_task_action(db_task)
except Exception as exc:
LOG.exception(exc)
self._handle_task_error(task, exc)
except Exception as e:
LOG.exception(e)
self._handle_task_error(task, e)

1
mistral/engine/workflow.py
View File

@ -91,6 +91,7 @@ def find_tasks_after_completion(task, workbook):
workflow_tasks = []
for t in found_tasks:
workflow_tasks += find_workflow_tasks(workbook, t.name)
LOG.debug("Workflow tasks to schedule: %s" % workflow_tasks)
return workflow_tasks

9
mistral/tests/api/v1/controllers/test_workbook_definition.py
View File

@ -24,17 +24,20 @@ from mistral.db import api as db_api
DEFINITION = "my definition"
NEW_DEFINITION = """
Services:
Namespaces:
Service:
type:
actions:
action:
parameters:
class: std.echo
base-parameters:
output: Haha
Workflow:
tasks:
task1:
parameters:
action: Service:action
triggers:
create-vms:
type: periodic

35
mistral/tests/resources/data_flow/task_with_two_dependencies.yaml
View File

@ -1,16 +1,35 @@
Services:
Namespaces:
MyService:
type: ECHO
# These ad-hoc actions based on std.echo have parameters only for test
# purposes. In practice, it's more convenient just to use std.echo and
# specify parameter 'output'.
actions:
build_full_name:
class: std.echo
base-parameters:
output: '{$.first_name} {$.last_name}'
parameters:
- first_name
- last_name
output:
full_name: John Doe
full_name: $.base_output
build_greeting:
class: std.echo
base-parameters:
output: Cheers!
output:
greeting: Cheers!
greeting: $.base_output
send_greeting:
class: std.echo
base-parameters:
output: True
parameters:
- f_name
- greet_msg
output:
greeting_sent: True
greeting_sent: $.base_output
Workflow:
# context = {
@ -27,7 +46,7 @@ Workflow:
tasks:
build_full_name:
action: MyService:build_full_name
action: MyService.build_full_name
parameters:
first_name: $.person.first_name
last_name: $.person.last_name
@ -35,13 +54,13 @@ Workflow:
f_name: full_name
build_greeting:
action: MyService:build_greeting
action: MyService.build_greeting
publish:
greet_msg: greeting
send_greeting:
requires: [build_full_name, build_greeting]
action: MyService:send_greeting
action: MyService.send_greeting
parameters:
f_name: $.f_name
greet_msg: $.greet_msg

36
mistral/tests/resources/data_flow/three_subsequent_tasks.yaml
View File

@ -1,16 +1,36 @@
Services:
Namespaces:
MyService:
type: ECHO
# These ad-hoc actions based on std.echo have parameters only for test
# purposes. In practice, it's more convenient just to use std.echo and
# specify parameter 'output'.
actions:
build_full_name:
class: std.echo
base-parameters:
output: '{$.first_name} {$.last_name}'
parameters:
- first_name
- last_name
output:
full_name: John Doe
full_name: $.base_output
build_greeting:
class: std.echo
base-parameters:
output: 'Hello, {$.full_name}!'
parameters:
- full_name
output:
greeting: Hello, John Doe!
greeting: $.base_output
send_greeting:
class: std.echo
base-parameters:
output: True
parameters:
- greeting
output:
greeting_sent: True
greeting_sent: $.base_output
Workflow:
@ -28,7 +48,7 @@ Workflow:
tasks:
build_full_name:
action: MyService:build_full_name
action: MyService.build_full_name
parameters:
first_name: $.person.first_name
last_name: $.person.last_name
@ -37,7 +57,7 @@ Workflow:
on-success: build_greeting
build_greeting:
action: MyService:build_greeting
action: MyService.build_greeting
parameters:
full_name: $.f_name
publish:
@ -45,7 +65,7 @@ Workflow:
on-success: send_greeting
send_greeting:
action: MyService:send_greeting
action: MyService.send_greeting
parameters:
greeting: $.task.build_greeting.greeting
publish:

26
mistral/tests/resources/data_flow/two_dependent_tasks.yaml
View File

@ -1,13 +1,27 @@
Services:
Namespaces:
MyService:
type: ECHO
actions:
# These ad-hoc actions based on std.echo have parameters only for test
# purposes. In practice, it's more convenient just to use std.echo and
# specify parameter 'output'.
build_full_name:
class: std.echo
base-parameters:
output: '{$.first_name} {$.last_name}'
parameters:
- first_name
- last_name
output:
full_name: John Doe
full_name: $.base_output
build_greeting:
class: std.echo
base-parameters:
output: 'Hello, {$.full_name}!'
parameters:
- full_name
output:
greeting: Hello, John Doe!
greeting: $.base_output
Workflow:
@ -25,7 +39,7 @@ Workflow:
tasks:
build_full_name:
action: MyService:build_full_name
action: MyService.build_full_name
parameters:
first_name: $.person.first_name
last_name: $.person.last_name
@ -34,6 +48,6 @@ Workflow:
build_greeting:
requires: [build_full_name]
action: MyService:build_greeting
action: MyService.build_greeting
parameters:
full_name: $.f_name

26
mistral/tests/resources/data_flow/two_subsequent_tasks.yaml
View File

@ -1,13 +1,27 @@
Services:
Namespaces:
MyService:
type: ECHO
# These ad-hoc actions based on std.echo have parameters only for test
# purposes. In practice, it's more convenient just to use std.echo and
# specify parameter 'output'.
actions:
build_full_name:
class: std.echo
base-parameters:
output: '{$.first_name} {$.last_name}'
parameters:
- first_name
- last_name
output:
full_name: John Doe
full_name: $.base_output
build_greeting:
class: std.echo
base-parameters:
output: 'Hello, {$.full_name}!'
parameters:
- full_name
output:
greeting: Hello, John Doe!
greeting: $.base_output
Workflow:
@ -25,7 +39,7 @@ Workflow:
tasks:
build_full_name:
action: MyService:build_full_name
action: MyService.build_full_name
parameters:
first_name: $.person.first_name
last_name: $.person.last_name
@ -34,7 +48,7 @@ Workflow:
on-success: build_greeting
build_greeting:
action: MyService:build_greeting
action: MyService.build_greeting
parameters:
full_name: $.f_name
publish:

14
mistral/tests/resources/repeat_task/no_repeat_task.yaml
View File

@ -1,10 +1,10 @@
Services:
MyService:
type: ECHO
Namespaces:
MyService:
actions:
repeatable-action:
output:
greeting: Cheers!
repeatable-action:
class: std.echo
output:
greeting: Cheers!
Workflow:
tasks:
@ -12,6 +12,6 @@ Workflow:
repeat:
iterations: 0
delay: 0
action: MyService:repeatable-action
action: MyService.repeatable-action
publish:
greet_msg: greeting

20
mistral/tests/resources/repeat_task/single_repeat_task.yaml
View File

@ -1,10 +1,12 @@
Services:
MyService:
type: ECHO
Namespaces:
MyService:
actions:
repeatable-action:
output:
greeting: Cheers!
repeatable-action:
class: std.echo
base-parameters:
output: Cheers!
output:
greeting: $.base_output
Workflow:
tasks:
@ -12,7 +14,7 @@ Workflow:
repeat:
iterations: 3
delay: 1
action: MyService:repeatable-action
action: MyService.repeatable-action
publish:
greet_msg: greeting
@ -21,12 +23,12 @@ Workflow:
iterations: 5
delay: 0
break-on: $.greet_msg != null
action: MyService:repeatable-action
action: MyService.repeatable-action
publish:
greet_msg: greeting
not_repeat_task:
action: MyService:repeatable-action
action: MyService.repeatable-action
publish:
greet_msg: greeting
on-success: repeater_task

173
mistral/tests/resources/test_rest.yaml
View File

@ -1,100 +1,99 @@
Services:
MyRest:
type: MISTRAL_REST_API
parameters:
baseUrl: http://some_host
actions:
create-vm:
parameters:
url: /service/action/execute
method: GET
task-parameters:
flavor_id:
image_id:
headers:
X-Auth-Token: $.auth_token
Namespaces:
MyRest:
actions:
create-vm:
class: std.mistral_http
base-parameters:
url: http://some_host/service/action/execute
method: GET
headers:
X-Auth-Token: $.auth_token
parameters:
- image_id
- flavor_id
output:
backup-vm:
parameters:
url: /url_for_backup
method: GET
task-parameters:
server_id:
backup-vm:
class: std.mistral_http
base-parameters:
url: http://some_host/url_for_backup
method: GET
parameters:
- server_id
attach-volume:
parameters:
url: /url_for_attach
method: GET
task-parameters:
size:
mnt_path:
attach-volume:
class: std.mistral_http
base-parameters:
url: /url_for_attach
method: GET
parameters:
- size
- mnt_path
format-volume:
parameters:
url: /url_for_format
method: GET
task-parameters:
volume_id:
server_id:
Nova:
type: REST_API
parameters:
baseUrl: http://path_to_nova
actions:
create-vm:
parameters:
url: /url_for_create
task-parameters:
flavor_id:
image_id:
output:
select: '$.server_id'
store_as: vm_id
format-volume:
class: std.mistral_http
base-parameters:
url: /url_for_format
method: GET
parameters:
- server_id
Nova:
actions:
create-vm:
class: std.http
base-parameters:
url: http://path_to_nova/url_for_create
parameters:
- image_id
- flavor_id
output:
vm_id: $.base_output.server_id
Workflow:
tasks:
create-vms:
action: MyRest:create-vm
parameters:
image_id: 1234
flavor_id: 42
tasks:
create-vms:
action: MyRest.create-vm
parameters:
image_id: 1234
flavor_id: 42
attach-volumes:
requires: [create-vms]
action: MyRest:attach-volume
parameters:
size:
mnt_path:
attach-volumes:
requires: [create-vms]
action: MyRest.attach-volume
parameters:
size:
mnt_path:
format-volumes:
requires: [attach-volumes]
action: MyRest:format-volume
parameters:
server_id:
format-volumes:
requires: [attach-volumes]
action: MyRest.format-volume
parameters:
server_id:
backup-vms:
requires: [create-vms]
action: MyRest:backup-vm
parameters:
server_id:
backup-vms:
requires: [create-vms]
action: MyRest.backup-vm
parameters:
server_id:
create-vm-nova:
action: Nova:create-vm
parameters:
image_id: 1234
flavor_id: 2
create-vm-nova:
action: Nova.create-vm
parameters:
image_id: 1234
flavor_id: 2
test_subsequent:
action: MyRest:backup-vm
parameters:
server_id:
on-success:
attach-volumes
on-error:
- backup-vms: $.status != 'OK'
- attach-volumes
on-finish:
create-vms
test_subsequent:
action: MyRest.backup-vm
parameters:
server_id: 1
on-success:
attach-volumes
on-error:
- backup-vms: $.status != 'OK'
- attach-volumes
on-finish:
create-vms
triggers:
create-vms:

6
mistral/tests/unit/actions/test_std_adhoc_action.py
View File

@ -38,7 +38,7 @@ class AdHocActionTest(base.BaseTest):
action_spec = NS_SPEC['my_namespace']['my_action'].copy()
# With dic-like output formatter.
action = std.AdHocAction(std.EchoAction, action_spec,
action = std.AdHocAction(None, std.EchoAction, action_spec,
first="Tango", second="Cash")
self.assertDictEqual({'res': 'Tango and Cash'}, action.run())
@ -46,7 +46,7 @@ class AdHocActionTest(base.BaseTest):
# With list-like output formatter.
action_spec['output'] = ['$.base_output', '$.base_output']
action = std.AdHocAction(std.EchoAction, action_spec,
action = std.AdHocAction(None, std.EchoAction, action_spec,
first="Tango", second="Cash")
self.assertListEqual(['Tango and Cash', 'Tango and Cash'],
@ -55,7 +55,7 @@ class AdHocActionTest(base.BaseTest):
# With single-object output formatter.
action_spec['output'] = "'{$.base_output}' is a cool movie!"
action = std.AdHocAction(std.EchoAction, action_spec,
action = std.AdHocAction(None, std.EchoAction, action_spec,
first="Tango", second="Cash")
self.assertEqual("'Tango and Cash' is a cool movie!", action.run())

4
mistral/tests/unit/db/test_sqlalchemy_db_api.py
View File

@ -224,7 +224,7 @@ TASKS = [
'description': 'my description',
'requires': {'my_task2': '', 'my_task3': ''},
'task_spec': None,
'service_spec': None,
'action_spec': None,
'action': {'name': 'Nova:create-vm'},
'state': 'IDLE',
'tags': ['deployment'],
@ -242,7 +242,7 @@ TASKS = [
'description': 'my description',
'requires': {'my_task4': '', 'my_task5': ''},
'task_spec': None,
'service_spec': None,
'action_spec': None,
'action': {'name': 'Cinder:create-volume'},
'state': 'IDLE',
'tags': ['deployment'],

0
mistral/tests/unit/engine/actions/__init__.py
View File

27
mistral/tests/unit/engine/actions/test_fake_action.py
View File

@ -1,27 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright 2013 - StackStorm, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from mistral.engine.actions import actions
from mistral.engine.actions import action_types
from mistral.tests import base
class FakeActionTest(base.BaseTest):
def test_fake_action(self):
expected = "my output"
action = actions.EchoAction(action_types.ECHO, "test", output=expected)
self.assertEqual(action.run(), expected)

97
mistral/tests/unit/engine/actions/test_result_helper.py
View File

@ -1,97 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright 2014 - Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import mock
import requests
from mistral.engine.actions import action_types
from mistral.engine.actions import action_factory
from mistral.tests import base
SAMPLE_TASK = {
'task_spec': {
'action': 'MyRest:create-vm',
'parameters': {
'a': 'b'
},
'headers': {
'Cookie': 'abc'
},
'name': 'create-vms'
},
'service_spec': {
'parameters': {
'baseUrl': 'http://some_host'
},
'actions': {
'create-vm': {
'parameters': {
'url': '/task1'
}
}
},
'name': 'MyRest'
},
'workbook_name': 'wb',
'execution_id': '1234',
'id': '123'
}
SAMPLE_RESULT_HELPER = {
'output': {
'vm_id': '$.server.id'
}
}
SAMPLE_RESULT = {
'server': {
'id': 'afd1254-1ad3fb0'
}
}
class FakeResponse(object):
def __init__(self):
self.status_code = 200
self.content = SAMPLE_RESULT
def json(self):
return self.content
class ResultHelperTest(base.BaseTest):
@mock.patch.object(requests, "request",
mock.MagicMock(return_value=FakeResponse()))
def test_action_result_with_results(self):
task = copy.deepcopy(SAMPLE_TASK)
task['service_spec'].update({'type': action_types.REST_API})
create_vm = task['service_spec']['actions']['create-vm']
create_vm.update(SAMPLE_RESULT_HELPER)
action = action_factory.create_action(task)
result = action.run()
self.assertEqual(result, {'vm_id': SAMPLE_RESULT['server']['id']})
@mock.patch.object(requests, "request",
mock.MagicMock(return_value=FakeResponse()))
def test_action_result_without_results(self):
task = copy.deepcopy(SAMPLE_TASK)
task['service_spec'].update({'type': action_types.REST_API})
action = action_factory.create_action(task)
result = action.run()
self.assertEqual(result, SAMPLE_RESULT)

124
mistral/tests/unit/engine/actions/test_send_email_action.py
View File

@ -1,124 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright 2013 - StackStorm, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest2
from mock import patch, call
from email.parser import Parser
from mistral.engine.actions import actions
from mistral.engine.actions import action_types
from mistral import exceptions as exc
ACTION_TYPE = action_types.SEND_EMAIL
ACTION_NAME = "TEMPORARY"
'''
To try against a real SNMP server:
1) set LOCAL_SMPTD = True
run debug snmpd on the local machine:
`sudo python -m smtpd -c DebuggingServer -n localhost:25`
Debugging server doesn't support password.
2) set REMOTE_SMPT = True
use external SNMP (like gmail), change the configuration,
provide actual username and password
self.settings = {
'host': 'smtp.gmail.com:587',
'from': "youraccount@gmail.com",
'password': "secret"
}
'''
LOCAL_SMTPD = False
REMOTE_SMTP = False
class SendEmailActionTest(unittest2.TestCase):
def setUp(self):
self.params = {
'to': ["dz@example.com, deg@example.com", "xyz@example.com"],
'subject': "Multi word subject с русскими буквами",
'body': "short multiline\nbody\nc русскими буквами",
}
self.settings = {
'smtp_server': 'mail.example.com:25',
'from': "bot@example.com",
}
self.to_addrs = ', '.join(self.params['to'])
@unittest2.skipIf(not LOCAL_SMTPD, "Setup local smtpd to run it")
def test_send_email_real(self):
action = actions.SendEmailAction(
ACTION_TYPE, ACTION_NAME, self.params, self.settings)
action.run()
@unittest2.skipIf(not REMOTE_SMTP, "Configure Remote SMTP to run it")
def test_with_password_real(self):
self.params['to'] = ["dz@stackstorm.com"]
self.settings = {
'smtp_server': 'smtp.gmail.com:587',
'from': "username@gmail.com",
}
self.settings['password'] = "secret"
action = actions.SendEmailAction(
ACTION_TYPE, ACTION_NAME, self.params, self.settings)
action.run()
@patch('smtplib.SMTP')
def test_send_email(self, smtp):
action = actions.SendEmailAction(
ACTION_TYPE, ACTION_NAME, self.params, self.settings)
action.run()
smtp.assert_called_once_with(self.settings['smtp_server'])
sendmail = smtp.return_value.sendmail
self.assertTrue(sendmail.called, "should call sendmail")
self.assertEqual(
sendmail.call_args[1]['from_addr'], self.settings['from'])
self.assertEqual(
sendmail.call_args[1]['to_addrs'], self.to_addrs)
message = Parser().parsestr(sendmail.call_args[1]['msg'])
self.assertEqual(message['from'], self.settings['from'])
self.assertEqual(message['to'], self.to_addrs)
self.assertEqual(message['subject'], self.params['subject'])
self.assertEqual(message.get_payload(), self.params['body'])
@patch('smtplib.SMTP')
def test_with_password(self, smtp):
self.settings['password'] = "secret"
action = actions.SendEmailAction(
ACTION_TYPE, ACTION_NAME, self.params, self.settings)
action.run()
smtpmock = smtp.return_value
calls = [call.ehlo(), call.starttls(), call.ehlo(),
call.login(self.settings['from'], self.settings['password'])]
smtpmock.assert_has_calls(calls)
self.assertTrue(smtpmock.sendmail.called, "should call sendmail")
@patch('mistral.engine.actions.actions.LOG')
def test_exception(self, log):
self.params['smtp_server'] = "wrong host"
action = actions.SendEmailAction(
ACTION_TYPE, ACTION_NAME, self.params, self.settings)
try:
action.run()
except exc.ActionException:
pass
else:
self.assertFalse("Must throw exception")

13
mistral/tests/unit/engine/scalable/test_engine.py
View File

@ -20,13 +20,12 @@ from oslo.config import cfg
from mistral.openstack.common import log as logging
from mistral.openstack.common import importutils
from mistral.db import api as db_api
from mistral.engine.actions import actions
from mistral.actions import std_actions
from mistral.engine import expressions
from mistral.engine.scalable import engine
from mistral.engine import states
from mistral.tests import base
# We need to make sure that all configuration properties are registered.
importutils.import_module("mistral.config")
LOG = logging.getLogger(__name__)
@ -48,7 +47,7 @@ class TestScalableEngine(base.EngineTestCase):
mock.MagicMock(return_value={
'definition': base.get_resource("test_rest.yaml")
}))
@mock.patch.object(actions.RestAction, "run",
@mock.patch.object(std_actions.HTTPAction, "run",
mock.MagicMock(return_value="result"))
def test_engine_one_task(self):
execution = self.engine.start_workflow_execution(WB_NAME, "create-vms",
@ -71,7 +70,7 @@ class TestScalableEngine(base.EngineTestCase):
mock.MagicMock(return_value={
'definition': base.get_resource("test_rest.yaml")
}))
@mock.patch.object(actions.RestAction, "run",
@mock.patch.object(std_actions.HTTPAction, "run",
mock.MagicMock(return_value="result"))
def test_engine_multiple_tasks(self):
execution = self.engine.start_workflow_execution(WB_NAME, "backup-vms",
@ -113,7 +112,7 @@ class TestScalableEngine(base.EngineTestCase):
@mock.patch.object(engine.ScalableEngine, '_run_tasks',
mock.MagicMock(
side_effect=base.EngineTestCase.mock_run_tasks))
@mock.patch.object(actions.RestAction, "run",
@mock.patch.object(std_actions.HTTPAction, "run",
mock.MagicMock(return_value={'state': states.SUCCESS}))
@mock.patch.object(db_api, "workbook_get",
mock.MagicMock(return_value={
@ -141,7 +140,7 @@ class TestScalableEngine(base.EngineTestCase):
mock.MagicMock(return_value={
'definition': base.get_resource("test_rest.yaml")
}))
@mock.patch.object(actions.RestAction, "run",
@mock.patch.object(std_actions.HTTPAction, "run",
mock.MagicMock(return_value={'state': states.SUCCESS}))
@mock.patch.object(expressions, "evaluate",
mock.MagicMock(side_effect=lambda x, y: x))
@ -213,7 +212,7 @@ class TestScalableEngine(base.EngineTestCase):
mock.MagicMock(return_value={
'definition': base.get_resource("test_rest.yaml")
}))
@mock.patch.object(actions.RestAction, "run",
@mock.patch.object(std_actions.HTTPAction, "run",
mock.MagicMock(return_value={'state': states.SUCCESS}))
@mock.patch.object(expressions, "evaluate",
mock.MagicMock(side_effect=lambda x, y: x))

34
mistral/tests/unit/engine/scalable/test_executor.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
import eventlet
eventlet.monkey_patch()
import uuid
@ -24,11 +25,9 @@ from mistral.tests import base
from mistral.cmd import launch
from mistral.engine import states
from mistral.db import api as db_api
from mistral.engine.actions import actions
from mistral.engine.actions import action_types
from mistral.actions import std_actions
from mistral.engine.scalable.executor import client
WORKBOOK_NAME = 'my_workbook'
TASK_NAME = 'my_task'
@ -56,25 +55,21 @@ SAMPLE_EXECUTION = {
SAMPLE_TASK = {
'name': TASK_NAME,
'workbook_name': WORKBOOK_NAME,
'service_spec': {
'type': action_types.REST_API,
'parameters': {
'baseUrl': 'http://localhost:8989/v1/'},
'actions': {
'my-action': {
'parameters': {
'url': 'workbooks',
'method': 'GET'}}},
'name': 'MyService'
'action_spec': {
'name': 'my-action',
'class': 'std.http',
'base-parameters': {
'url': 'http://localhost:8989/v1/workbooks',
'method': 'GET'
},
'namespace': 'MyRest'
},
'task_spec': {
'action': 'MyRest:my-action',
'service_name': 'MyRest',
'action': 'MyRest.my-action',
'name': TASK_NAME},
'requires': {},
'state': states.IDLE}
SAMPLE_CONTEXT = {
'user': 'admin',
'tenant': 'mistral'
@ -82,10 +77,9 @@ SAMPLE_CONTEXT = {
class TestExecutor(base.DbTestCase):
def mock_action_run(self):
actions.RestAction.run = mock.MagicMock(return_value={})
return actions.RestAction.run
std_actions.HTTPAction.run = mock.MagicMock(return_value={})
return std_actions.HTTPAction.run
def setUp(self):
super(TestExecutor, self).setUp()
@ -101,7 +95,7 @@ class TestExecutor(base.DbTestCase):
super(TestExecutor, self).tearDown()
def test_handle_task(self):
# Mock the RestAction
# Mock HTTP action.
mock_rest_action = self.mock_action_run()
# Create a new workbook.

1
mistral/tests/unit/engine/test_repeat_task.py
View File

@ -101,6 +101,7 @@ class RepeatTaskTest(base.EngineTestCase):
# Wait until all iterations are finished
eventlet.sleep(delay * iterations + 1)
tasks = db_api.tasks_get(wb['name'], execution['id'])
self._assert_multiple_items(tasks, 2)
self._assert_single_item(tasks, name='repeater_task')
self._assert_single_item(tasks, task_runtime_context=None)

1
mistral/tests/unit/events/test_events.py
View File

@ -25,6 +25,7 @@ from mistral.services import scheduler
class TriggersTest(base.DbTestCase):
def setUp(self):
super(TriggersTest, self).setUp()
self.doc = open(pkg.resource_filename(
version.version_info.package,
"tests/resources/test_rest.yaml")).read()

46
mistral/tests/unit/workbook/test_workbook.py
View File

@ -17,7 +17,6 @@
import pkg_resources as pkg
import unittest2
from mistral.engine.actions import action_types as a_t
from mistral import dsl_parser as parser
from mistral import version
@ -34,23 +33,31 @@ class DSLModelTest(unittest2.TestCase):
self.workbook.tasks.items)
self.assertEqual(self.workbook.tasks.get("create-vms").name,
"create-vms")
self.assertEqual(self.workbook.services.get("MyRest").type,
"MISTRAL_REST_API")
self.assertEqual(4,
len(self.workbook.namespaces.get("MyRest").actions))
def test_tasks(self):
self.workbook = parser.get_workbook(self.doc)
self.assertEqual(len(self.workbook.tasks), 6)
attach_volumes = self.workbook.tasks.get("attach-volumes")
self.assertEqual(attach_volumes.get_action_service(), "MyRest")
self.assertEqual(attach_volumes.get_action_namespace(), "MyRest")
t_parameters = {"image_id": 1234, "flavor_id": 2}
create_vm_nova = self.workbook.tasks.get("create-vm-nova")
self.assertEqual(create_vm_nova.parameters, t_parameters)
attach_requires = {"create-vms": ''}
self.assertEqual(attach_volumes.requires, attach_requires)
subsequent = self.workbook.tasks.get("test_subsequent")
subseq_success = subsequent.get_on_success()
subseq_error = subsequent.get_on_error()
subseq_finish = subsequent.get_on_finish()
self.assertEqual(subseq_success, {"attach-volumes": ''})
self.assertEqual(subseq_error, {"backup-vms": "$.status != 'OK'",
"attach-volumes": ''})
@ -58,20 +65,27 @@ class DSLModelTest(unittest2.TestCase):
def test_actions(self):
self.workbook = parser.get_workbook(self.doc)
actions = self.workbook.services.get("MyRest").actions
self.assertEqual(len(actions), 4)
create_vm = actions.get("create-vm")
self.assertIn('method', create_vm.parameters)
def test_services(self):
actions = self.workbook.namespaces.get("MyRest").actions
self.assertEqual(len(actions), 4)
create_vm = actions.get("create-vm")
self.assertIn('method', create_vm.base_parameters)
def test_namespaces(self):
self.workbook = parser.get_workbook(self.doc)
services = self.workbook.services
self.assertEqual(len(services), 2)
nova_service = services.get("Nova")
self.assertEqual(nova_service.type, a_t.REST_API)
self.assertIn("baseUrl", nova_service.parameters)
namespaces = self.workbook.namespaces
self.assertEqual(len(namespaces), 2)
nova_namespace = namespaces.get("Nova")
self.assertEqual(1, len(nova_namespace.actions))
def test_triggers(self):
self.workbook = parser.get_workbook(self.doc)
triggers = self.workbook.get_triggers()
self.assertEqual(len(triggers), 1)
self.assertEqual(len(self.workbook.get_triggers()), 1)

8
mistral/workbook/actions.py
View File

@ -18,18 +18,18 @@ from mistral.workbook import base
class ActionSpec(base.BaseSpec):
# TODO(rakhmerov): Add parameters required by new design (class etc.)
_required_keys = ['name']
_required_keys = ['name', 'class', 'namespace']
def __init__(self, action):
super(ActionSpec, self).__init__(action)
if self.validate():
self.name = action['name']
self.clazz = action['class']
self.namespace = action['namespace']
self.base_parameters = action.get('base-parameters', {})
self.parameters = action.get('parameters', {})
self.output = action.get('output', {})
# TODO(rakhmerov): Get rid of it once new design is finished.
self.task_parameters = action.get('task-parameters', {})
class ActionSpecList(base.BaseSpecList):

17
mistral/workbook/services.py → mistral/workbook/namespaces.py
View File

@ -18,17 +18,20 @@ from mistral.workbook import actions
from mistral.workbook import base
class ServiceSpec(base.BaseSpec):
_required_keys = ['name', 'type', 'actions']
class NamespaceSpec(base.BaseSpec):
_required_keys = ['name', 'actions']
def __init__(self, service):
super(ServiceSpec, self).__init__(service)
super(NamespaceSpec, self).__init__(service)
if self.validate():
self.type = service['type']
self.name = service['name']
self.parameters = service.get('parameters', {})
for _, action in service['actions'].iteritems():
action['namespace'] = self.name
self.actions = actions.ActionSpecList(service['actions'])
class ServiceSpecList(base.BaseSpecList):
item_class = ServiceSpec
class NamespaceSpecList(base.BaseSpecList):
item_class = NamespaceSpec

11
mistral/workbook/tasks.py
View File

@ -24,7 +24,9 @@ class TaskSpec(base.BaseSpec):
def __init__(self, task):
super(TaskSpec, self).__init__(task)
self._prepare(task)
if self.validate():
self.requires = task['requires']
self.action = task['action']
@ -34,6 +36,7 @@ class TaskSpec(base.BaseSpec):
def _prepare(self, task):
if task:
req = task.get("requires", {})
if req and isinstance(req, list):
task["requires"] = dict(zip(req, ['']*len(req)))
elif isinstance(req, dict):
@ -41,8 +44,10 @@ class TaskSpec(base.BaseSpec):
def _get_on_state(self, key):
tasks = self.get_property(key)
if not tasks:
return None
if isinstance(tasks, dict):
return tasks
elif isinstance(tasks, list):
@ -65,11 +70,11 @@ class TaskSpec(base.BaseSpec):
def get_on_finish(self):
return self._get_on_state("on-finish")
def get_action_service(self):
return self.action.split(':')[0]
def get_action_namespace(self):
return self.action.split('.')[0]
def get_action_name(self):
return self.action.split(':')[1]
return self.action.split('.')[1]
def get_full_action_name(self):
return self.action

15
mistral/workbook/workbook.py
View File

@ -15,18 +15,19 @@
# limitations under the License.
from mistral.workbook import base
from mistral.workbook import services
from mistral.workbook import namespaces
from mistral.workbook import workflow
class WorkbookSpec(base.BaseSpec):
_required_keys = ['Services', 'Workflow']
_required_keys = ['Namespaces', 'Workflow']
def __init__(self, doc):
super(WorkbookSpec, self).__init__(doc)
if self.validate():
self.services = services.ServiceSpecList(self._data['Services'])
self.namespaces =\
namespaces.NamespaceSpecList(self._data['Namespaces'])
self.workflow = workflow.WorkflowSpec(self._data['Workflow'])
self.tasks = self.workflow.tasks
@ -48,14 +49,14 @@ class WorkbookSpec(base.BaseSpec):
if task_action_name.find(":") == -1:
return {}
service_name = task_action_name.split(':')[0]
namespace_name = task_action_name.split(':')[0]
action_name = task_action_name.split(':')[1]
action = self.services.get(service_name).actions.get(action_name)
action = self.namespaces.get(namespace_name).actions.get(action_name)
return action
def get_actions(self, service_name):
return self.services.get(service_name).actions
def get_actions(self, namespace_name):
return self.namespaces.get(namespace_name).actions
def get_trigger_task_name(self, trigger_name):
trigger = self._data["triggers"].get(trigger_name)

1
mistral/workbook/workflow.py
View File

@ -23,5 +23,6 @@ class WorkflowSpec(base.BaseSpec):
def __init__(self, workflow):
super(WorkflowSpec, self).__init__(workflow)
if self.validate():
self.tasks = tasks.TaskSpecList(workflow['tasks'])