Initial Integration of Mistral into TripleO

This patch includes the base elements needed for integrating Mistral
and TripleO.  For a development workflow, clone the source code and
use the following steps:

 * Create additional actions in the /actions directory
 * Add an entry point in the setup.cfg file [entry_points]
 * See README for instructions on importing actions into Mistral

Change-Id: Iac675122e79c6f9b2f41ba455c15d6422c796929
Co-Authored-By: Dan Prince <dprince@redhat.com>
Co-Authored-By: Dougal Matthews <dmatthews@redhat.com>
Blueprint: mistral-deployment-library
Depends-On: Ibb98013d6ef283bd454f958bb036557d2b02997f

README.rst

@@ -8,3 +8,22 @@ A common library for TripleO workflows.
 * Documentation: http://docs.openstack.org/developer/tripleo-common
 * Source: http://git.openstack.org/cgit/openstack/tripleo-common
 * Bugs: http://bugs.launchpad.net/tripleo-common
+
+Action Development
+-------------------
+
+
+When developing new actions, you will checkout a copy of tripleo-common to an
+undercloud machine and add actions as needed.  To test the actions they need
+to be installed and selected services need to be restarted.  Use the following
+code below to accomplish these tasks.
+
+        sudo rm -Rf /usr/lib/python2.7/site-packages/tripleo_common*
+        sudo python setup.py install
+        sudo systemctl restart openstack-mistral-executor
+        sudo systemctl restart openstack-mistral-engine
+        # this loads the actions via entrypoints
+        sudo mistral-db-manage populate
+        # make sure the new actions got loaded
+        mistral action-list | grep tripleo
+

requirements.txt

@@ -12,3 +12,4 @@ oslo.utils>=3.5.0 # Apache-2.0
 python-glanceclient>=2.0.0 # Apache-2.0
 python-ironicclient>=1.1.0 # Apache-2.0
 six>=1.9.0 # MIT
+mistral!=2015.1.0,>=2.0.0 # Apache-2.0

setup.cfg

@@ -50,3 +50,9 @@ input_file = tripleo_common/locale/tripleo-common.pot
 keywords = _ gettext ngettext l_ lazy_gettext
 mapping_file = babel.cfg
 output_file = tripleo_common/locale/tripleo-common.pot
+
+[entry_points]
+mistral.actions =
+    tripleo.upload_default_templates = tripleo_common.actions.templates:UploadTemplatesAction
+    tripleo.create_container = tripleo_common.actions.plan:CreateContainerAction
+    tripleo.create_plan = tripleo_common.actions.plan:CreatePlanAction

tripleo_common/actions/base.py

@@ -0,0 +1,67 @@
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import logging
+
+from heatclient.v1 import client as heatclient
+from mistral.actions import base
+from mistral import context
+from mistral.utils.openstack import keystone as keystone_utils
+from mistralclient.api import client as mistral_client
+from swiftclient import client as swift_client
+
+
+LOG = logging.getLogger(__name__)
+
+
+class TripleOAction(base.Action):
+
+    def __init__(self):
+        super(TripleOAction, self).__init__()
+
+    def _get_object_client(self):
+        ctx = context.ctx()
+        obj_ep = keystone_utils.get_endpoint_for_project('swift')
+
+        kwargs = {
+            'preauthurl': obj_ep.url % {'tenant_id': ctx.project_id},
+            'preauthtoken': ctx.auth_token
+        }
+
+        return swift_client.Connection(**kwargs)
+
+    def _get_orchestration_client(self):
+        ctx = context.ctx()
+        heat_endpoint = keystone_utils.get_endpoint_for_project('heat')
+
+        endpoint_url = keystone_utils.format_url(
+            heat_endpoint.url,
+            {'tenant_id': ctx.project_id}
+        )
+
+        return heatclient.Client(
+            endpoint_url,
+            region_name=heat_endpoint.region,
+            token=ctx.auth_token,
+            username=ctx.user_name
+        )
+
+    def _get_workflow_client(self):
+        ctx = context.ctx()
+        mistral_endpoint = keystone_utils.get_endpoint_for_project('mistral')
+
+        mc = mistral_client.client(auth_token=ctx.auth_token,
+                                   mistral_url=mistral_endpoint.url)
+
+        return mc

tripleo_common/actions/plan.py

@@ -0,0 +1,67 @@
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import json
+import logging
+import yaml
+
+from tripleo_common.actions import base
+from tripleo_common.core import constants
+from tripleo_common.core import exception
+
+LOG = logging.getLogger(__name__)
+
+default_container_headers = {
+    constants.TRIPLEO_META_USAGE_KEY: 'plan'
+}
+
+
+class CreateContainerAction(base.TripleOAction):
+
+    def __init__(self, container):
+        super(CreateContainerAction, self).__init__()
+        self.container = container
+
+    def run(self):
+        oc = self._get_object_client()
+        # checks to see if a container with that name exists
+        if self.container in [container["name"] for container in
+                              oc.get_account()[1]]:
+            raise exception.ContainerAlreadyExistsError(name=self.container)
+        oc.put_container(self.container, headers=default_container_headers)
+
+
+class CreatePlanAction(base.TripleOAction):
+
+    def __init__(self, container):
+        super(CreatePlanAction, self).__init__()
+        self.container = container
+
+    def run(self):
+        oc = self._get_object_client()
+        env_data = {
+            'name': self.container,
+        }
+        env_vars = {}
+        # parses capabilities to get root_template, root_environment
+        mapfile = yaml.load(
+            oc.get_object(self.container, 'capabilities-map.yaml')[1])
+        if mapfile['root_template']:
+            env_vars['template'] = mapfile['root_template']
+        if mapfile['root_environment']:
+            env_vars['environments'] = [{'path': mapfile['root_environment']}]
+
+        env_data['variables'] = json.dumps(env_vars, sort_keys=True,)
+        # creates environment
+        self._get_workflow_client().environments.create(**env_data)

tripleo_common/actions/templates.py

@@ -0,0 +1,40 @@
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import logging
+import tempfile as tf
+
+from tripleo_common.actions import base
+from tripleo_common import constants
+from tripleo_common.utils import tarball
+
+LOG = logging.getLogger(__name__)
+
+
+class UploadTemplatesAction(base.TripleOAction):
+    """Upload default heat templates for TripleO.
+
+    """
+    def __init__(self, container=constants.DEFAULT_CONTAINER_NAME):
+        super(UploadTemplatesAction, self).__init__()
+        self.container = container
+
+    def run(self):
+        tht_base_path = constants.DEFAULT_TEMPLATES_PATH
+        with tf.NamedTemporaryFile() as tmp_tarball:
+            tarball.create_tarball(tht_base_path, tmp_tarball.name)
+            tarball.tarball_extract_to_swift_container(
+                self._get_object_client(),
+                tmp_tarball.name,
+                self.container)

tripleo_common/constants.py

@@ -25,3 +25,9 @@ UPDATE_RESOURCE_NAME = 'UpdateDeployment'
 
 #: The default timeout to pass to Heat stacks
 STACK_TIMEOUT_DEFAULT = 240
+
+#: The default name to use for a plan container
+DEFAULT_CONTAINER_NAME = 'overcloud'
+
+#: The path to the tripleo heat templates installed on the undercloud
+DEFAULT_TEMPLATES_PATH = '/usr/share/openstack-tripleo-heat-templates/'

tripleo_common/core/exception.py

@@ -71,8 +71,8 @@ class FileDoesNotExistError(TripleoCommonException):
     msg_fmt = _("A file with the name %(name)s does not exist.")
 
 
-class PlanAlreadyExistsError(TripleoCommonException):
-    msg_fmt = _("A plan with the name %(name)s already exists.")
+class ContainerAlreadyExistsError(TripleoCommonException):
+    msg_fmt = _("A container with the name %(name)s already exists.")
 
 
 class TooManyRootTemplatesError(TripleoCommonException):
@@ -90,3 +90,7 @@ class MappingFileNotFoundError(TripleoCommonException):
 
 class TooManyCapabilitiesMapFilesError(TripleoCommonException):
     msg_fmt = _("There cannot be more than one root template in a given plan.")
+
+
+class ServiceEndpointNotFoundError(TripleoCommonException):
+    msg_fmt = _("No endpoints found for [%(service_name)s, %(service_type)s].")

tripleo_common/core/plan_storage.py

@@ -44,7 +44,7 @@ class SwiftPlanStorageBackend(object):
                 headers=default_container_headers
             )
         else:
-            raise exception.PlanAlreadyExistsError(name=plan_name)
+            raise exception.ContainerAlreadyExistsError(name=plan_name)
 
     def delete(self, plan_name):
         """Deletes a plan and associated files

tripleo_common/tests/actions/test_plan.py

@@ -0,0 +1,129 @@
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import mock
+
+from tripleo_common.actions import plan
+from tripleo_common.core import exception
+from tripleo_common.tests import base
+
+MAPPING_YAML_CONTENTS = """root_template: /path/to/overcloud.yaml
+root_environment: /path/to/environment.yaml
+topics:
+  - title: Fake Single Environment Group Configuration
+    description:
+    environment_groups:
+      - title:
+        description: Random fake string of text
+        environments:
+          - file: /path/to/network-isolation.json
+            title: Default Configuration
+            description:
+
+  - title: Fake Multiple Environment Group Configuration
+    description:
+    environment_groups:
+      - title: Random Fake 1
+        description: Random fake string of text
+        environments:
+          - file: /path/to/ceph-storage-env.yaml
+            title: Fake1
+            description: Random fake string of text
+
+      - title: Random Fake 2
+        description:
+        environments:
+          - file: /path/to/poc-custom-env.yaml
+            title: Fake2
+            description:
+"""
+
+
+class CreateContainerActionTest(base.TestCase):
+
+    def setUp(self):
+        super(CreateContainerActionTest, self).setUp()
+        self.container_name = 'test-container'
+        self.expected_list = ['', [{'name': 'test1'}, {'name': 'test2'}]]
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction._get_object_client')
+    def test_run(self, get_obj_client_mock):
+
+        # Setup
+        swift = mock.MagicMock()
+        swift.get_account.return_value = self.expected_list
+        get_obj_client_mock.return_value = swift
+
+        # Test
+        action = plan.CreateContainerAction(self.container_name)
+        action.run()
+
+        # Verify
+        swift.put_container.assert_called_once_with(
+            self.container_name,
+            headers=plan.default_container_headers
+        )
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction._get_object_client')
+    def test_run_container_exists(self, get_obj_client_mock):
+
+        # Setup
+        swift = mock.MagicMock()
+        swift.get_account.return_value = [
+            '', [{'name': 'test-container'}, {'name': 'test2'}]]
+        get_obj_client_mock.return_value = swift
+
+        # Test
+        action = plan.CreateContainerAction(self.container_name)
+
+        self.assertRaises(exception.ContainerAlreadyExistsError, action.run)
+
+
+class CreatePlanActionTest(base.TestCase):
+
+    def setUp(self):
+        super(CreatePlanActionTest, self).setUp()
+        self.container_name = 'test-container'
+        self.capabilities_name = 'capabilities-map.yaml'
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction._get_object_client')
+    @mock.patch(
+        'tripleo_common.actions.base.TripleOAction._get_workflow_client')
+    def test_run(self, get_workflow_client_mock, get_obj_client_mock):
+
+        # setup swift
+        swift = mock.MagicMock()
+        swift.get_object.return_value = ({}, MAPPING_YAML_CONTENTS)
+        get_obj_client_mock.return_value = swift
+
+        # setup mistral
+        mistral = mock.MagicMock()
+        get_workflow_client_mock.return_value = mistral
+
+        # Test
+        action = plan.CreatePlanAction(self.container_name)
+        action.run()
+
+        # verify
+        swift.get_object.assert_called_once_with(
+            self.container_name,
+            self.capabilities_name
+        )
+
+        mistral.environments.create.assert_called_once_with(
+            name='test-container',
+            variables=('{"environments":'
+                       ' [{"path": "/path/to/environment.yaml"}], '
+                       '"template": "/path/to/overcloud.yaml"}')
+        )

tripleo_common/tests/actions/test_templates.py

@@ -0,0 +1,40 @@
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import mock
+
+from tripleo_common.actions import templates
+from tripleo_common import constants
+from tripleo_common.tests import base
+
+
+class UploadTemplatesActionTest(base.TestCase):
+
+    @mock.patch('tempfile.NamedTemporaryFile')
+    @mock.patch('tripleo_common.actions.base.TripleOAction._get_object_client')
+    @mock.patch('tripleo_common.utils.tarball.'
+                'tarball_extract_to_swift_container')
+    @mock.patch('tripleo_common.utils.tarball.create_tarball')
+    def test_run(self, mock_create_tar, mock_extract_tar, mock_get_swift,
+                 tempfile):
+
+        tempfile.return_value.__enter__.return_value.name = "test"
+
+        action = templates.UploadTemplatesAction(container='tar-container')
+        action.run()
+
+        mock_create_tar.assert_called_once_with(
+            constants.DEFAULT_TEMPLATES_PATH, 'test')
+        mock_extract_tar.assert_called_once_with(
+            mock_get_swift.return_value, 'test', 'tar-container')

tripleo_common/tests/core/test_plan_storage.py

@@ -88,7 +88,7 @@ class PlanStorageTest(base.TestCase):
 
         # attempt to create a 2nd plan should fail
         self.plan_store.list = mock.MagicMock(return_value=['overcloud'])
-        self.assertRaisesRegexp(exception.PlanAlreadyExistsError,
+        self.assertRaisesRegexp(exception.ContainerAlreadyExistsError,
                                 self.plan_name,
                                 self.plan_store.create,
                                 self.plan_name)

tripleo_common/tests/utils/test_templates.py

@@ -13,9 +13,11 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import mock
+
 from tripleo_common.tests import base
-from tripleo_common.utils.templates import find_root_template
-from tripleo_common.utils.templates import process_plan_data
+from tripleo_common.utils import templates
 
 PLAN_DATA = {
     '/path/to/overcloud.yaml': {
@@ -59,7 +61,7 @@ class UtilsTemplatesTest(base.TestCase):
 
     def setUp(self):
         super(UtilsTemplatesTest, self).setUp()
-        self.tpl, self.env, self.files = process_plan_data(PLAN_DATA)
+        self.tpl, self.env, self.files = templates.process_plan_data(PLAN_DATA)
         print(self.files)
 
     def test_find_root_template(self):
@@ -67,7 +69,7 @@ class UtilsTemplatesTest(base.TestCase):
         del PLAN_DATA['/path/to/overcloud.yaml']
 
         # without root, should return {}
-        self.assertEqual({}, find_root_template(PLAN_DATA))
+        self.assertEqual({}, templates.find_root_template(PLAN_DATA))
 
         # add root_template back to sample data
         root_template = {
@@ -77,7 +79,8 @@ class UtilsTemplatesTest(base.TestCase):
         }
         PLAN_DATA.update(root_template)
 
-        self.assertEqual(root_template, find_root_template(PLAN_DATA))
+        self.assertEqual(root_template,
+                         templates.find_root_template(PLAN_DATA))
 
     def test_template_found(self):
         self.assertEqual(self.tpl, 'heat_template_version: 2015-04-30')
@@ -86,3 +89,26 @@ class UtilsTemplatesTest(base.TestCase):
         self.assertEqual(self.files, {
             '/path/to/somefile.yaml': 'description: lorem ipsum',
         })
+
+    @mock.patch("requests.request")
+    def test_preprocess_templates(self, mock_request):
+
+        # Setup
+        envs = []
+        mock_request.return_value = mock.Mock(content="""{
+        "heat_template_version": "2016-04-08"
+        }""")
+
+        # Test a basic call to check the main code paths
+        result = templates.preprocess_templates(
+            "swift_base_url", "container", "template", envs, "auth_token")
+
+        # Verify the values we get out
+        self.assertEqual(result, {
+            'environment': {},
+            'files': {},
+            'stack_name': 'container',
+            'template': {
+                'heat_template_version': '2016-04-08'
+            }
+        })

tripleo_common/utils/tarball.py

@@ -0,0 +1,39 @@
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import logging
+
+from oslo_concurrency import processutils
+
+LOG = logging.getLogger(__name__)
+
+
+def create_tarball(directory, filename, options='-czf'):
+    """Create a tarball of a directory."""
+    LOG.debug('Creating tarball of %s at location %s' % (directory, filename))
+    processutils.execute('/usr/bin/tar', '-C', directory, options, filename,
+                         '--exclude', '.git', '--exclude', '.tox', '.')
+
+
+def tarball_extract_to_swift_container(object_client, filename, container):
+    LOG.debug('Uploading filename %s to Swift container %s' % (filename,
+                                                               container))
+    with open(filename, 'r') as f:
+        object_client.put_object(
+            container=container,
+            obj='',
+            contents=f,
+            query_string='extract-archive=tar.gz',
+            headers={'X-Detect-Content-Type': 'true'}
+        )

tripleo_common/utils/templates.py

@@ -14,8 +14,13 @@
 # under the License.
 import json
 import logging
+import os
+import requests
+import tempfile
 import yaml
 
+from heatclient.common import template_utils
+
 LOG = logging.getLogger(__name__)
 
 
@@ -91,6 +96,80 @@ def process_plan_data(plan_data):
     return template, environment, files
 
 
+def preprocess_templates(swift_base_url, container_name, template,
+                         environments, auth_token):
+    """Pre-processes and organizes plan files
+
+    This method processes heat templates and environments by collecting the
+    remote paths of the files in a given swift container and combining them
+    with given environment data and uses methods in python-heatclient to get
+    template contents and process the files with respect to order.  This
+    method also sets the stack_name returned in the results to the same name
+    as the given container.
+
+    :param swift_base_url: the endpoint url for swift
+    :param container_name: name of the swift container that holds heat
+                           templates for a deployment plan
+    :param template: the root template of a given plan
+    :param environments: environment files or yaml contents to be combined
+    :param auth_token: keystone authentication token for accessing heat and
+                       swift to retrieve file contents.
+    :return: dict of heat stack name, template, combined environment and files
+    """
+    template_object = os.path.join(swift_base_url, container_name, template)
+    env_paths = []
+    temp_files = []
+    LOG.debug('Template: %s' % template)
+    LOG.debug('Environments: %s' % environments)
+    try:
+        for env in environments:
+            if env.get('path'):
+                env_paths.append(os.path.join(swift_base_url, container_name,
+                                              env['path']))
+            elif env.get('data'):
+                handle, env_temp_file = tempfile.mkstemp()
+                with open(env_temp_file, 'w') as temp_file:
+                    temp_file.write(json.dumps(env['data']))
+                    os.close(handle)
+                temp_files.append(env_temp_file)
+                env_paths.append(env_temp_file)
+
+        def _env_path_is_object(env_path):
+            if env_path in temp_files:
+                LOG.debug('_env_path_is_object %s: False' % env_path)
+                return False
+            else:
+                LOG.debug('_env_path_is_object %s: True' % env_path)
+                return True
+
+        def _object_request(method, url, token=auth_token):
+            return requests.request(method, url,
+                                    headers={'X-Auth-Token': token}).content
+
+        template_files, template = template_utils.get_template_contents(
+            template_object=template_object,
+            object_request=_object_request)
+
+        env_files, env = (
+            template_utils.process_multiple_environments_and_files(
+                env_paths=env_paths,
+                env_path_is_object=_env_path_is_object,
+                object_request=_object_request))
+    finally:
+        # cleanup any local temp files
+        for f in temp_files:
+            os.remove(f)
+
+    files = dict(list(template_files.items()) + list(env_files.items()))
+
+    return {
+        'stack_name': container_name,
+        'template': template,
+        'environment': env,
+        'files': files
+    }
+
+
 def find_root_template(plan_files):
     return {k: v for (k, v) in plan_files.items()
             if v.get('meta', {}).get('file-type') == 'root-template'}

workbooks/plan_management.yaml

@@ -0,0 +1,30 @@
+---
+version: '2.0'
+name: tripleo.plan_management.v1
+description: TripleO Overcloud Deployment Workflows v1
+
+workflows:
+
+  create_deployment_plan:
+    type: direct
+    input:
+      - container
+    tasks:
+      create_plan:
+        action: tripleo.create_plan container=<% $.container %>
+
+  create_default_deployment_plan:
+    type: direct
+    input:
+      - container
+    tasks:
+      create_container:
+        action: tripleo.create_container container=<% $.container %>
+        on-success:
+          - upload_to_container
+      upload_to_container:
+        action: tripleo.upload_default_templates container=<% $.container %>
+        on-success:
+          - create_plan
+      create_plan:
+        action: tripleo.create_plan container=<% $.container %>