From 72dcdfbf2a84ba8ee05bf6aa82a6d80b90c28a05 Mon Sep 17 00:00:00 2001 From: Xavier Hardy Date: Tue, 5 Jul 2016 10:09:04 +0200 Subject: [PATCH] Add support for custom YAQL functions Custom YAQL functions must be added in the 'mistral.yaql_functions' namespace in the entry points. The entry point name is used as the function name in YAQL. All of the Mistral YAQL functions have been changed to a custom functions for the sake of testing. DocImpact: The developer section in the documention has been updated. Change-Id: I518c7a4b616089d4b70585ed85fcd81a30c93090 Implements: blueprint mistral-custom-yaql-functions --- doc/source/developer/extending_yaql.rst | 175 ++++++++++++++++++++++++ doc/source/developer/index.rst | 1 + mistral/utils/yaql_utils.py | 22 ++- setup.cfg | 6 + 4 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 doc/source/developer/extending_yaql.rst diff --git a/doc/source/developer/extending_yaql.rst b/doc/source/developer/extending_yaql.rst new file mode 100644 index 00000000..6724ed72 --- /dev/null +++ b/doc/source/developer/extending_yaql.rst @@ -0,0 +1,175 @@ +====================================== +How to extend YAQL with a new function +====================================== + +******** +Tutorial +******** + +1. Create a new Python project, an empty folder, containing a basic ``setup.py`` file. + +.. code-block:: bash + + mkdir my_project + cd my_project + vim setup.py + +.. code-block:: python + + try: + from setuptools import setup, find_packages + except ImportError: + from distutils.core import setup, find_packages + + setup( + name="project_name", + version="0.1.0", + packages=find_packages(), + install_requires=["mistral", "yaql"], + entry_points={ + "mistral.yaql_functions": [ + "random_uuid = my_package.sub_package.yaql:random_uuid_" + ] + } + ) + + +Publish the ``random_uuid_`` function in the ``entry_points`` section, in the +``mistral.yaql_functions`` namespace in ``setup.py``. This function will be +defined later. + +Note that the package name will be used in Pip and must not overlap with +other packages installed. ``project_name`` may be replaced by something else. +The package name (``my_package`` here) may overlap with other +packages, but module paths (``.py`` files) may not. + +For example, it is possible to have a ``mistral`` package (though not +recommended), but there must not be a ``mistral/version.py`` file, which +would overlap with the file existing in the original ``mistral`` package. + +``yaql`` and ``mistral`` are the required packages. ``mistral`` is necessary +in this example only because calls to the Mistral Python DB API are made. + +For each entry point, the syntax is: + +.. code-block:: python + + " = :" + +``stevedore`` will detect all the entry points and make them available to +all Python applications needing them. Using this feature, there is no need +to modify Mistral's core code. + +2. Create a package folder. + +A package folder is directory with a ``__init__.py`` file. Create a file +that will contain the custom YAQL functions. There are no restrictions on +the paths or file names used. + +.. code-block:: bash + + mkdir -p my_package/sub_package + touch my_package/__init__.py + touch my_package/sub_package/__init__.py + +3. Write a function in ``yaql.py``. + +That function might have ``context`` as first argument to have the current +YAQL context available inside the function. + +.. code-block:: bash + + cd my_package/sub_package + vim yaql.py + +.. code-block:: python + + from uuid import uuid5, UUID + from time import time + + + def random_uuid_(context): + """generate a UUID using the execution ID and the clock""" + + # fetch the current workflow execution ID found in the context + execution_id = context['__execution']['id'] + + time_str = str(time()) + execution_uuid = UUID(execution_id) + return uuid5(execution_uuid, time_str) + +This function returns a random UUID using the current workflow execution ID +as a namespace. + +The ``context`` argument will be passed by Mistral YAQL engine to the +function. It is invisble to the user. It contains variables from the current +task execution scope, such as ``__execution`` which is a dictionary with +information about the current workflow execution such as its ``id``. + +Note that errors can be raised and will be displayed in the task execution +state information in case they are raised. Any valid Python primitives may +be returned. + +The ``context`` argument is optional. There can be as many arguments as wanted, +even list arguments such as ``*args`` or dictionary arguments such as +``**kwargs`` can be used as function arguments. + +For more information about YAQL, read the `official YAQL documentation `_. + +4. Install ``pip`` and ``setuptools``. + +.. code-block:: bash + + curl https://bootstrap.pypa.io/get-pip.py | python + pip install --upgrade setuptools + cd - + +5. Install the package (note that there is a dot ``.`` at the end of the line). + +.. code-block:: bash + + pip install . + +6. The YAQL function can be called in Mistral using its name ``random_uuid``. + +The function name in Python ``random_uuid_`` does not matter, only the entry +point name ``random_uuid`` does. + +.. code-block:: yaml + + my_workflow: + tasks: + my_action_task: + action: std.echo + publish: + random_id: <% random_uuid() %> + input: + output: "hello world" + +**************** +Updating changes +**************** + +After any new created functions or any modification in the code, re-run +``pip install .`` and restart Mistral. + +*********** +Development +*********** + +While developing, it is sufficient to add the root source folder (the parent +folder of ``my_package``) to the ``PYTHONPATH`` environment variable and the +line ``random_uuid = my_package.sub_package.yaql:random_uuid_`` in the Mistral +entry points in the ``mistral.yaql_functions`` namespace. If the path to the +parent folder of ``my_package`` is ``/path/to/my_project``. + +.. code-block:: bash + + export PYTHONPATH=$PYTHONPATH:/path/to/my_project + vim $(find / -name "mistral.*egg-info*")/entry_points.txt + +.. code-block:: ini + + [entry_points] + mistral.yaql_functions = + random_uuid = my_package.sub_package.yaql:random_uuid_ diff --git a/doc/source/developer/index.rst b/doc/source/developer/index.rst index 3e3b6de3..956b96be 100644 --- a/doc/source/developer/index.rst +++ b/doc/source/developer/index.rst @@ -6,5 +6,6 @@ Developer's Reference creating_custom_action asynchronous_actions + extending_yaql devstack troubleshooting diff --git a/mistral/utils/yaql_utils.py b/mistral/utils/yaql_utils.py index fc32d8e0..8e42c846 100644 --- a/mistral/utils/yaql_utils.py +++ b/mistral/utils/yaql_utils.py @@ -18,6 +18,7 @@ import yaql from mistral.db.v2 import api as db_api from mistral.workflow import utils as wf_utils from oslo_serialization import jsonutils +from stevedore import extension ROOT_CONTEXT = None @@ -39,11 +40,24 @@ def get_yaql_context(data_context): return new_ctx +def _register_custom_functions(yaql_ctx): + """Register custom YAQL functions + + Custom YAQL functions must be added as entry points in the + 'mistral.yaql_functions' namespace + :param yaql_ctx: YAQL context object + """ + mgr = extension.ExtensionManager( + namespace='mistral.yaql_functions', + invoke_on_load=False + ) + for name in mgr.names(): + yaql_function = mgr[name].plugin + yaql_ctx.register_function(yaql_function, name=name) + + def _register_functions(yaql_ctx): - yaql_ctx.register_function(env_) - yaql_ctx.register_function(execution_) - yaql_ctx.register_function(task_) - yaql_ctx.register_function(json_pp_, name='json_pp') + _register_custom_functions(yaql_ctx) # Additional YAQL functions needed by Mistral. diff --git a/setup.cfg b/setup.cfg index bbd7b409..5ce69d80 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,3 +61,9 @@ mistral.actions = std.email = mistral.actions.std_actions:SendEmailAction std.javascript = mistral.actions.std_actions:JavaScriptAction std.sleep = mistral.actions.std_actions:SleepAction + +mistral.yaql_functions = + json_pp = mistral.utils.yaql_utils:json_pp_ + task = mistral.utils.yaql_utils:task_ + execution = mistral.utils.yaql_utils:execution_ + env = mistral.utils.yaql_utils:env_