Change the mistralclient for Mistral action pack

The purpose of this patch is to make the new Mistral action pack
actions more useful when they are coded in a workflow or workbook.

The new Mistral action pack actions are executed by calling the
appropriate python-mistralclient API methods. The API methods to
create/update/validate the workbook/workflow/action/environment
resources do not read the definitions/environments from files as the
CLI commands do. They require the actual definition/environement be
passed in as an argument. This can lead to problems when trying to
use the Mistral action pack actions from within a workflow or
workbook.

For example, the workflow create/update/validate methods all take a
definition argument. The mistralclient APIs are currently coded to
take the actual definition, and not the definition file name as the
CLI does. This leads to problems when coding the
mistral.workflows_create action in a workflow for example. If the
actual definition is passed into the action in the workflow YAML, any
YAQL expressions in the workflow being created by that action get
evaluated when read, and are not stored in the database as desired.
This essentially makes it useless unless the workflow you are
creating does not contain any YAQL. That is unrealistic.

This patch enhances those methods of the mistralclient so that they
can take either the actual definition, or the definition file name or
file URI. This allows the action to be coded in a workflow in a way
similar to the CLI by allowing the definition file name, or file URI
to be specified instead of the actual definition. The change allows
either the actual definition or the definition file name, or file
URI to be specified when calling the affected mistralclient API
methods. No changes to the API signatures are required. Each method
affected will determine if the value passed in is a definition file
name or file URI, or the actual definition.

New unit test cases were also added for these changes.

Change-Id: I9ca07bedf685839db130693a7aa6e091a9b8c4ad
Implements: blueprint mistral-mistral-actions

mistralclient/api/v2/actions.py

@@ -15,7 +15,7 @@
 import six
 
 from mistralclient.api import base
-
+from mistralclient import utils
 
 urlparse = six.moves.urllib.parse
 
@@ -30,6 +30,10 @@ class ActionManager(base.ResourceManager):
     def create(self, definition, scope='private'):
         self._ensure_not_empty(definition=definition)
 
+        # If the specified definition is actually a file, read in the
+        # definition file
+        definition = utils.get_contents_if_file(definition)
+
         resp = self.client.http_client.post(
             '/actions?scope=%s' % scope,
             definition,
@@ -45,6 +49,10 @@ class ActionManager(base.ResourceManager):
     def update(self, definition, scope='private'):
         self._ensure_not_empty(definition=definition)
 
+        # If the specified definition is actually a file, read in the
+        # definition file
+        definition = utils.get_contents_if_file(definition)
+
         resp = self.client.http_client.put(
             '/actions?scope=%s' % scope,
             definition,

mistralclient/api/v2/environments.py

@@ -17,6 +17,7 @@ import json
 import six
 
 from mistralclient.api import base
+from mistralclient import utils
 
 
 class Environment(base.Resource):
@@ -39,6 +40,12 @@ class EnvironmentManager(base.ResourceManager):
     resource_class = Environment
 
     def create(self, **kwargs):
+        # Check to see if the file name or URI is being passed in. If so,
+        # read it's contents first.
+        if 'file' in kwargs:
+            file = kwargs['file']
+            kwargs = utils.load_content(utils.get_contents_if_file(file))
+
         self._ensure_not_empty(name=kwargs.get('name', None),
                                variables=kwargs.get('variables', None))
 
@@ -49,6 +56,12 @@ class EnvironmentManager(base.ResourceManager):
         return self._create('/environments', kwargs)
 
     def update(self, **kwargs):
+        # Check to see if the file name or URI is being passed in. If so,
+        # read it's contents first.
+        if 'file' in kwargs:
+            file = kwargs['file']
+            kwargs = utils.load_content(utils.get_contents_if_file(file))
+
         name = kwargs.get('name', None)
         self._ensure_not_empty(name=name)
 

mistralclient/api/v2/workbooks.py

@@ -14,6 +14,7 @@
 #    limitations under the License.
 
 from mistralclient.api import base
+from mistralclient import utils
 
 
 class Workbook(base.Resource):
@@ -26,6 +27,10 @@ class WorkbookManager(base.ResourceManager):
     def create(self, definition):
         self._ensure_not_empty(definition=definition)
 
+        # If the specified definition is actually a file, read in the
+        # definition file
+        definition = utils.get_contents_if_file(definition)
+
         resp = self.client.http_client.post(
             '/workbooks',
             definition,
@@ -40,6 +45,10 @@ class WorkbookManager(base.ResourceManager):
     def update(self, definition):
         self._ensure_not_empty(definition=definition)
 
+        # If the specified definition is actually a file, read in the
+        # definition file
+        definition = utils.get_contents_if_file(definition)
+
         resp = self.client.http_client.put(
             '/workbooks',
             definition,
@@ -67,6 +76,10 @@ class WorkbookManager(base.ResourceManager):
     def validate(self, definition):
         self._ensure_not_empty(definition=definition)
 
+        # If the specified definition is actually a file, read in the
+        # definition file
+        definition = utils.get_contents_if_file(definition)
+
         resp = self.client.http_client.post(
             '/workbooks/validate',
             definition,

mistralclient/api/v2/workflows.py

@@ -16,6 +16,7 @@
 import six
 
 from mistralclient.api import base
+from mistralclient import utils
 
 
 urlparse = six.moves.urllib.parse
@@ -31,6 +32,10 @@ class WorkflowManager(base.ResourceManager):
     def create(self, definition, scope='private'):
         self._ensure_not_empty(definition=definition)
 
+        # If the specified definition is actually a file, read in the
+        # definition file
+        definition = utils.get_contents_if_file(definition)
+
         resp = self.client.http_client.post(
             '/workflows?scope=%s' % scope,
             definition,
@@ -48,6 +53,10 @@ class WorkflowManager(base.ResourceManager):
 
         url_pre = ('/workflows/%s' % id) if id else '/workflows'
 
+        # If the specified definition is actually a file, read in the
+        # definition file
+        definition = utils.get_contents_if_file(definition)
+
         resp = self.client.http_client.put(
             '%s?scope=%s' % (url_pre, scope),
             definition,
@@ -99,6 +108,10 @@ class WorkflowManager(base.ResourceManager):
     def validate(self, definition):
         self._ensure_not_empty(definition=definition)
 
+        # If the specified definition is actually a file, read in the
+        # definition file
+        definition = utils.get_contents_if_file(definition)
+
         resp = self.client.http_client.post(
             '/workflows/validate',
             definition,

mistralclient/tests/unit/resources/action_v2.yaml

@@ -0,0 +1,10 @@
+
+---
+version: 2.0
+
+my_action:
+  base: std.echo
+  base-input:
+    output: 'Bye!'
+  output:
+    info: <% $.output %>

mistralclient/tests/unit/resources/env_v2.json

@@ -0,0 +1,8 @@
+{
+    "name": "env1",
+    "description": "Test Environment #1",
+    "scope": "private",
+    "variables": {
+        "server": "localhost"
+    }
+}

mistralclient/tests/unit/resources/env_v2.yaml

@@ -0,0 +1,7 @@
+---
+
+"name": "env1"
+"description": "Test Environment #1"
+"scope": "private"
+"variables":
+    "server": "localhost"

mistralclient/tests/unit/resources/wb_v2.yaml

@@ -0,0 +1,21 @@
+
+---
+version: 2.0
+
+name: wb
+
+workflows:
+  wf1:
+    type: direct
+    input:
+      - param1
+      - param2
+
+    tasks:
+      task1:
+        action: std.http url="localhost:8989"
+        on-success:
+          - test_subsequent
+
+      test_subsequent:
+        action: std.http url="http://some_url" server_id=1

mistralclient/tests/unit/resources/wf_v2.yaml

@@ -0,0 +1,10 @@
+
+---
+version: 2.0
+
+my_wf:
+  type: direct
+
+  tasks:
+    task1:
+      action: std.echo output="hello, world"

mistralclient/tests/unit/v2/test_actions.py

@@ -11,6 +11,11 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
+
+import pkg_resources as pkg
+from six.moves.urllib import parse
+from six.moves.urllib import request
+
 from mistralclient.api.v2 import actions
 from mistralclient.tests.unit.v2 import base
 
@@ -54,6 +59,26 @@ class TestActionsV2(base.BaseClientV2Test):
             headers={'content-type': 'text/plain'}
         )
 
+    def test_create_with_file(self):
+        mock = self.mock_http_post(content={'actions': [ACTION]})
+
+        # The contents of action_v2.yaml must be identical to ACTION_DEF
+        path = pkg.resource_filename(
+            'mistralclient',
+            'tests/unit/resources/action_v2.yaml'
+        )
+
+        actions = self.actions.create(path)
+
+        self.assertIsNotNone(actions)
+        self.assertEqual(ACTION_DEF, actions[0].definition)
+
+        mock.assert_called_once_with(
+            URL_TEMPLATE_SCOPE,
+            ACTION_DEF,
+            headers={'content-type': 'text/plain'}
+        )
+
     def test_update(self):
         mock = self.mock_http_put(content={'actions': [ACTION]})
 
@@ -68,6 +93,29 @@ class TestActionsV2(base.BaseClientV2Test):
             headers={'content-type': 'text/plain'}
         )
 
+    def test_update_with_file_uri(self):
+        mock = self.mock_http_put(content={'actions': [ACTION]})
+
+        # The contents of action_v2.yaml must be identical to ACTION_DEF
+        path = pkg.resource_filename(
+            'mistralclient',
+            'tests/unit/resources/action_v2.yaml'
+        )
+
+        # Convert the file path to file URI
+        uri = parse.urljoin('file:', request.pathname2url(path))
+
+        actions = self.actions.update(uri)
+
+        self.assertIsNotNone(actions)
+        self.assertEqual(ACTION_DEF, actions[0].definition)
+
+        mock.assert_called_once_with(
+            URL_TEMPLATE_SCOPE,
+            ACTION_DEF,
+            headers={'content-type': 'text/plain'}
+        )
+
     def test_list(self):
         mock = self.mock_http_get(content={'actions': [ACTION]})
 

mistralclient/tests/unit/v2/test_environments.py

@@ -11,12 +11,17 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
-
+from collections import OrderedDict
 import copy
 import json
 
+import pkg_resources as pkg
+from six.moves.urllib import parse
+from six.moves.urllib import request
+
 from mistralclient.api.v2 import environments
 from mistralclient.tests.unit.v2 import base
+from mistralclient import utils
 
 
 ENVIRONMENT = {
@@ -33,6 +38,7 @@ URL_TEMPLATE_NAME = '/environments/%s'
 
 
 class TestEnvironmentsV2(base.BaseClientV2Test):
+
     def test_create(self):
         data = copy.deepcopy(ENVIRONMENT)
 
@@ -46,6 +52,32 @@ class TestEnvironmentsV2(base.BaseClientV2Test):
 
         mock.assert_called_once_with(URL_TEMPLATE, json.dumps(expected_data))
 
+    def test_create_with_json_file_uri(self):
+        # The contents of env_v2.json must be equivalent to ENVIRONMENT
+        path = pkg.resource_filename(
+            'mistralclient',
+            'tests/unit/resources/env_v2.json'
+        )
+
+        # Convert the file path to file URI
+        uri = parse.urljoin('file:', request.pathname2url(path))
+        data = OrderedDict(
+            utils.load_content(
+                utils.get_contents_if_file(uri)
+            )
+        )
+
+        mock = self.mock_http_post(content=data)
+        file_input = {'file': uri}
+        env = self.environments.create(**file_input)
+
+        self.assertIsNotNone(env)
+
+        expected_data = copy.deepcopy(data)
+        expected_data['variables'] = json.dumps(expected_data['variables'])
+
+        mock.assert_called_once_with(URL_TEMPLATE, json.dumps(expected_data))
+
     def test_update(self):
         data = copy.deepcopy(ENVIRONMENT)
 
@@ -59,6 +91,29 @@ class TestEnvironmentsV2(base.BaseClientV2Test):
 
         mock.assert_called_once_with(URL_TEMPLATE, json.dumps(expected_data))
 
+    def test_update_with_yaml_file(self):
+        # The contents of env_v2.json must be equivalent to ENVIRONMENT
+        path = pkg.resource_filename(
+            'mistralclient',
+            'tests/unit/resources/env_v2.json'
+        )
+        data = OrderedDict(
+            utils.load_content(
+                utils.get_contents_if_file(path)
+            )
+        )
+
+        mock = self.mock_http_put(content=data)
+        file_input = {'file': path}
+        env = self.environments.update(**file_input)
+
+        self.assertIsNotNone(env)
+
+        expected_data = copy.deepcopy(data)
+        expected_data['variables'] = json.dumps(expected_data['variables'])
+
+        mock.assert_called_once_with(URL_TEMPLATE, json.dumps(expected_data))
+
     def test_list(self):
         mock = self.mock_http_get(content={'environments': [ENVIRONMENT]})
 

mistralclient/tests/unit/v2/test_workbooks.py

@@ -13,6 +13,10 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+import pkg_resources as pkg
+from six.moves.urllib import parse
+from six.moves.urllib import request
+
 from mistralclient.api import base as api_base
 from mistralclient.api.v2 import workbooks
 from mistralclient.tests.unit.v2 import base
@@ -79,6 +83,29 @@ class TestWorkbooksV2(base.BaseClientV2Test):
             headers={'content-type': 'text/plain'}
         )
 
+    def test_create_with_file_uri(self):
+        mock = self.mock_http_post(content=WORKBOOK)
+
+        # The contents of wb_v2.yaml must be identical to WB_DEF
+        path = pkg.resource_filename(
+            'mistralclient',
+            'tests/unit/resources/wb_v2.yaml'
+        )
+
+        # Convert the file path to file URI
+        uri = parse.urljoin('file:', request.pathname2url(path))
+
+        wb = self.workbooks.create(uri)
+
+        self.assertIsNotNone(wb)
+        self.assertEqual(WB_DEF, wb.definition)
+
+        mock.assert_called_once_with(
+            URL_TEMPLATE,
+            WB_DEF,
+            headers={'content-type': 'text/plain'}
+        )
+
     def test_update(self):
         mock = self.mock_http_put(content=WORKBOOK)
 
@@ -93,6 +120,26 @@ class TestWorkbooksV2(base.BaseClientV2Test):
             headers={'content-type': 'text/plain'}
         )
 
+    def test_update_with_file(self):
+        mock = self.mock_http_put(content=WORKBOOK)
+
+        # The contents of wb_v2.yaml must be identical to WB_DEF
+        path = pkg.resource_filename(
+            'mistralclient',
+            'tests/unit/resources/wb_v2.yaml'
+        )
+
+        wb = self.workbooks.update(path)
+
+        self.assertIsNotNone(wb)
+        self.assertEqual(WB_DEF, wb.definition)
+
+        mock.assert_called_once_with(
+            URL_TEMPLATE,
+            WB_DEF,
+            headers={'content-type': 'text/plain'}
+        )
+
     def test_list(self):
         mock = self.mock_http_get(content={'workbooks': [WORKBOOK]})
 
@@ -145,6 +192,28 @@ class TestWorkbooksV2(base.BaseClientV2Test):
             headers={'content-type': 'text/plain'}
         )
 
+    def test_validate_with_file(self):
+        mock = self.mock_http_post(status_code=200,
+                                   content={'valid': True})
+
+        # The contents of wb_v2.yaml must be identical to WB_DEF
+        path = pkg.resource_filename(
+            'mistralclient',
+            'tests/unit/resources/wb_v2.yaml'
+        )
+
+        result = self.workbooks.validate(path)
+
+        self.assertIsNotNone(result)
+        self.assertIn('valid', result)
+        self.assertTrue(result['valid'])
+
+        mock.assert_called_once_with(
+            URL_TEMPLATE_VALIDATE,
+            WB_DEF,
+            headers={'content-type': 'text/plain'}
+        )
+
     def test_validate_failed(self):
         mock_result = {
             "valid": False,

mistralclient/tests/unit/v2/test_workflows.py

@@ -11,6 +11,11 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
+
+import pkg_resources as pkg
+from six.moves.urllib import parse
+from six.moves.urllib import request
+
 from mistralclient.api.v2 import workflows
 from mistralclient.tests.unit.v2 import base
 
@@ -54,6 +59,26 @@ class TestWorkflowsV2(base.BaseClientV2Test):
             headers={'content-type': 'text/plain'}
         )
 
+    def test_create_with_file(self):
+        mock = self.mock_http_post(content={'workflows': [WORKFLOW]})
+
+        # The contents of wf_v2.yaml must be identical to WF_DEF
+        path = pkg.resource_filename(
+            'mistralclient',
+            'tests/unit/resources/wf_v2.yaml'
+        )
+
+        wfs = self.workflows.create(path)
+
+        self.assertIsNotNone(wfs)
+        self.assertEqual(WF_DEF, wfs[0].definition)
+
+        mock.assert_called_once_with(
+            URL_TEMPLATE_SCOPE,
+            WF_DEF,
+            headers={'content-type': 'text/plain'}
+        )
+
     def test_update(self):
         mock = self.mock_http_put(content={'workflows': [WORKFLOW]})
 
@@ -82,6 +107,29 @@ class TestWorkflowsV2(base.BaseClientV2Test):
             headers={'content-type': 'text/plain'}
         )
 
+    def test_update_with_file_uri(self):
+        mock = self.mock_http_put(content={'workflows': [WORKFLOW]})
+
+        # The contents of wf_v2.yaml must be identical to WF_DEF
+        path = pkg.resource_filename(
+            'mistralclient',
+            'tests/unit/resources/wf_v2.yaml'
+        )
+
+        # Convert the file path to file URI
+        uri = parse.urljoin('file:', request.pathname2url(path))
+
+        wfs = self.workflows.update(uri)
+
+        self.assertIsNotNone(wfs)
+        self.assertEqual(WF_DEF, wfs[0].definition)
+
+        mock.assert_called_once_with(
+            URL_TEMPLATE_SCOPE,
+            WF_DEF,
+            headers={'content-type': 'text/plain'}
+        )
+
     def test_list(self):
         mock = self.mock_http_get(content={'workflows': [WORKFLOW]})
 

mistralclient/utils.py

@@ -14,9 +14,12 @@
 #    limitations under the License.
 
 import json
-
+import os
 import yaml
 
+from six.moves.urllib import parse
+from six.moves.urllib import request
+
 from mistralclient import exceptions
 
 
@@ -51,3 +54,28 @@ def load_content(content):
 def load_file(path):
     with open(path, 'r') as f:
         return load_content(f.read())
+
+
+def get_contents_if_file(contents_or_file_name):
+    """Get the contents of a file.
+
+    If the value passed in is a file name or file URI, return the
+    contents. If not, or there is an error reading the file contents,
+    return the value passed in as the contents.
+
+    For example, a workflow definition will be returned if either the
+    workflow definition file name, or file URI are passed in, or the
+    actual workflow definition itself is passed in.
+    """
+    try:
+        if parse.urlparse(contents_or_file_name).scheme:
+            definition_url = contents_or_file_name
+        else:
+            path = os.path.abspath(contents_or_file_name)
+            definition_url = parse.urljoin(
+                'file:',
+                request.pathname2url(path)
+            )
+        return request.urlopen(definition_url).read().decode('utf8')
+    except Exception:
+        return contents_or_file_name

requirements.txt

@@ -7,3 +7,4 @@ python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0
 python-openstackclient>=2.1.0 # Apache-2.0
 PyYAML>=3.1.0 # MIT
 requests!=2.9.0,>=2.8.1 # Apache-2.0
+six>=1.9.0 # MIT