257 lines
7.2 KiB
Python
257 lines
7.2 KiB
Python
# Copyright 2013 - Mirantis, Inc.
|
|
# Copyright 2015 - StackStorm, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import cachetools
|
|
import threading
|
|
import yaml
|
|
from yaml import error
|
|
|
|
import six
|
|
|
|
from mistral.db.v2 import api as db_api
|
|
from mistral import exceptions as exc
|
|
from mistral.workbook import base
|
|
from mistral.workbook.v2 import actions as actions_v2
|
|
from mistral.workbook.v2 import tasks as tasks_v2
|
|
from mistral.workbook.v2 import workbook as wb_v2
|
|
from mistral.workbook.v2 import workflows as wf_v2
|
|
|
|
V2_0 = '2.0'
|
|
|
|
ALL_VERSIONS = [V2_0]
|
|
|
|
|
|
_WF_EX_CACHE = cachetools.LRUCache(maxsize=100)
|
|
_WF_EX_CACHE_LOCK = threading.RLock()
|
|
|
|
_WF_DEF_CACHE = cachetools.LRUCache(maxsize=100)
|
|
_WF_DEF_CACHE_LOCK = threading.RLock()
|
|
|
|
|
|
def parse_yaml(text):
|
|
"""Loads a text in YAML format as dictionary object.
|
|
|
|
:param text: YAML text.
|
|
:return: Parsed YAML document as dictionary.
|
|
"""
|
|
|
|
try:
|
|
return yaml.safe_load(text) or {}
|
|
except error.YAMLError as e:
|
|
raise exc.DSLParsingException(
|
|
"Definition could not be parsed: %s\n" % e
|
|
)
|
|
|
|
|
|
def _get_spec_version(spec_dict):
|
|
# If version is not specified it will '2.0' by default.
|
|
ver = V2_0
|
|
|
|
if 'version' in spec_dict:
|
|
ver = spec_dict['version']
|
|
|
|
if not ver or str(float(ver)) not in ALL_VERSIONS:
|
|
raise exc.DSLParsingException('Unsupported DSL version: %s' % ver)
|
|
|
|
return ver
|
|
|
|
|
|
# Factory methods to get specifications either from raw YAML formatted text or
|
|
# from dictionaries parsed from YAML formatted text.
|
|
|
|
def get_workbook_spec(spec_dict):
|
|
if _get_spec_version(spec_dict) == V2_0:
|
|
return base.instantiate_spec(wb_v2.WorkbookSpec, spec_dict)
|
|
|
|
return None
|
|
|
|
|
|
def get_workbook_spec_from_yaml(text):
|
|
return get_workbook_spec(parse_yaml(text))
|
|
|
|
|
|
def get_action_spec(spec_dict):
|
|
if _get_spec_version(spec_dict) == V2_0:
|
|
return base.instantiate_spec(actions_v2.ActionSpec, spec_dict)
|
|
|
|
return None
|
|
|
|
|
|
def get_action_spec_from_yaml(text, action_name):
|
|
spec_dict = parse_yaml(text)
|
|
|
|
spec_dict['name'] = action_name
|
|
|
|
return get_action_spec(spec_dict)
|
|
|
|
|
|
def get_action_list_spec(spec_dict):
|
|
return base.instantiate_spec(actions_v2.ActionListSpec, spec_dict)
|
|
|
|
|
|
def get_action_list_spec_from_yaml(text):
|
|
return get_action_list_spec(parse_yaml(text))
|
|
|
|
|
|
def get_workflow_spec(spec_dict):
|
|
"""Get workflow specification object from dictionary.
|
|
|
|
NOTE: For large workflows this method can work very long (seconds).
|
|
For this reason, method 'get_workflow_spec_by_definition_id' or
|
|
'get_workflow_spec_by_execution_id' should be used whenever possible
|
|
because they cache specification objects.
|
|
|
|
:param spec_dict: Raw specification dictionary.
|
|
"""
|
|
if _get_spec_version(spec_dict) == V2_0:
|
|
return base.instantiate_spec(wf_v2.WorkflowSpec, spec_dict)
|
|
|
|
return None
|
|
|
|
|
|
def get_workflow_list_spec(spec_dict):
|
|
return base.instantiate_spec(wf_v2.WorkflowListSpec, spec_dict)
|
|
|
|
|
|
def get_workflow_spec_from_yaml(text):
|
|
return get_workflow_spec(parse_yaml(text))
|
|
|
|
|
|
def get_workflow_list_spec_from_yaml(text):
|
|
return get_workflow_list_spec(parse_yaml(text))
|
|
|
|
|
|
def get_task_spec(spec_dict):
|
|
if _get_spec_version(spec_dict) == V2_0:
|
|
return base.instantiate_spec(tasks_v2.TaskSpec, spec_dict)
|
|
|
|
return None
|
|
|
|
|
|
def get_workflow_definition(wb_def, wf_name):
|
|
wf_name = wf_name + ":"
|
|
|
|
return _parse_def_from_wb(wb_def, "workflows:", wf_name)
|
|
|
|
|
|
def get_action_definition(wb_def, action_name):
|
|
action_name += ":"
|
|
|
|
return _parse_def_from_wb(wb_def, "actions:", action_name)
|
|
|
|
|
|
def _parse_def_from_wb(wb_def, section_name, item_name):
|
|
io = six.StringIO(wb_def[wb_def.index(section_name):])
|
|
io.readline()
|
|
definition = []
|
|
ident = 0
|
|
# Get the indentation of the action/workflow name tag.
|
|
for line in io:
|
|
if item_name == line.strip():
|
|
ident = line.index(item_name)
|
|
definition.append(line.lstrip())
|
|
break
|
|
|
|
# Add strings to list unless same/less indentation is found.
|
|
for line in io:
|
|
new_line = line.strip()
|
|
|
|
if not new_line:
|
|
definition.append(line)
|
|
elif new_line.startswith("#"):
|
|
new_line = line if ident > line.index("#") else line[ident:]
|
|
definition.append(new_line)
|
|
else:
|
|
temp = line.index(line.lstrip())
|
|
if ident < temp:
|
|
definition.append(line[ident:])
|
|
else:
|
|
break
|
|
|
|
io.close()
|
|
definition = ''.join(definition).rstrip() + '\n'
|
|
|
|
return definition
|
|
|
|
|
|
# Methods for obtaining specifications in a more efficient way using
|
|
# caching techniques.
|
|
|
|
@cachetools.cached(_WF_EX_CACHE, lock=_WF_EX_CACHE_LOCK)
|
|
def get_workflow_spec_by_execution_id(wf_ex_id):
|
|
"""Gets workflow specification by workflow execution id.
|
|
|
|
The idea is that when a workflow execution is running we
|
|
must be getting the same workflow specification even if
|
|
|
|
:param wf_ex_id: Workflow execution id.
|
|
:return: Workflow specification.
|
|
"""
|
|
if not wf_ex_id:
|
|
return None
|
|
|
|
wf_ex = db_api.get_workflow_execution(wf_ex_id)
|
|
|
|
return get_workflow_spec(wf_ex.spec)
|
|
|
|
|
|
@cachetools.cached(_WF_DEF_CACHE, lock=_WF_DEF_CACHE_LOCK)
|
|
def get_workflow_spec_by_definition_id(wf_def_id, wf_def_updated_at):
|
|
"""Gets specification by workflow definition id and its 'updated_at'.
|
|
|
|
The idea of this method is to return a cached specification for the
|
|
given workflow id and workflow definition 'updated_at'. As long as the
|
|
given workflow definition remains the same in DB users of this method
|
|
will be getting a cached value. Once the workflow definition has
|
|
changed clients will be providing a different 'updated_at' value and
|
|
hence this method will be called and spec is updated for this combination
|
|
of parameters. Old cached values will be kicked out by LRU algorithm
|
|
if the cache runs out of space.
|
|
|
|
:param wf_def_id: Workflow definition id.
|
|
:param wf_def_updated_at: Workflow definition 'updated_at' value. It
|
|
serves only as part of cache key and is not explicitly used in the
|
|
method.
|
|
:return: Workflow specification.
|
|
"""
|
|
if not wf_def_id:
|
|
return None
|
|
|
|
wf_def = db_api.get_workflow_definition(wf_def_id)
|
|
|
|
return get_workflow_spec(wf_def.spec)
|
|
|
|
|
|
def cache_workflow_spec_by_execution_id(wf_ex_id, wf_spec):
|
|
with _WF_EX_CACHE_LOCK:
|
|
_WF_EX_CACHE[cachetools.keys.hashkey(wf_ex_id)] = wf_spec
|
|
|
|
|
|
def get_wf_execution_spec_cache_size():
|
|
return len(_WF_EX_CACHE)
|
|
|
|
|
|
def get_wf_definition_spec_cache_size():
|
|
return len(_WF_DEF_CACHE)
|
|
|
|
|
|
def clear_caches():
|
|
"""Clears all specification caches."""
|
|
with _WF_EX_CACHE_LOCK:
|
|
_WF_EX_CACHE.clear()
|
|
|
|
with _WF_DEF_CACHE_LOCK:
|
|
_WF_DEF_CACHE.clear()
|