Merge "Create or update deployment plan from git repository"

releasenotes/notes/git-support-for-deployment-plans-cac4d3746689cbda.yaml

@@ -0,0 +1,13 @@
+---
+features:
+  - |
+    The create_deployment_plan workflow has been updated to provide support for
+    creating a deployment plan from a git repository of heat templates.  A tag
+    or branch can be specified in the repo url with an '@'.
+    Example: https://github.com/openstack/project.git@stable/newton
+deprecations:
+  - |
+    The tripleo.plan_management.v1.create_default_deployment_plan is deprecated
+    and will be removed in the Queens release.  The udpates to the
+    tripleo.plan_management.v1.create_deployment_plan ensures that it provides
+    the same functionality.

requirements.txt

@@ -5,6 +5,8 @@
 pbr>=2.0.0 # Apache-2.0
 Babel>=2.3.4 # BSD
 docker-py>=1.8.1 # Apache-2.0
+gitdb>=0.6.4 # BSD License (3 clause)
+GitPython>=1.0.1 # BSD License (3 clause)
 python-heatclient>=1.6.1 # Apache-2.0
 oslo.config>=3.22.0 # Apache-2.0
 oslo.log>=3.22.0 # Apache-2.0

setup.cfg

@@ -70,6 +70,8 @@ mistral.actions =
     tripleo.deployment.config = tripleo_common.actions.deployment:OrchestrationDeployAction
     tripleo.deployment.deploy = tripleo_common.actions.deployment:DeployStackAction
     tripleo.deployment.overcloudrc = tripleo_common.actions.deployment:OvercloudRcAction
+    tripleo.git.clean = tripleo_common.actions.vcs:GitCleanupAction
+    tripleo.git.clone = tripleo_common.actions.vcs:GitCloneAction
     tripleo.heat_capabilities.get = tripleo_common.actions.heat_capabilities:GetCapabilitiesAction
     tripleo.heat_capabilities.update = tripleo_common.actions.heat_capabilities:UpdateCapabilitiesAction
     tripleo.package_update.clear_breakpoints = tripleo_common.actions.package_update:ClearBreakpointsAction
@@ -92,9 +94,11 @@ mistral.actions =
     tripleo.scale.delete_node = tripleo_common.actions.scale:ScaleDownAction
     tripleo.swift.tempurl = tripleo_common.actions.swifthelper:SwiftTempUrlAction
     tripleo.templates.process = tripleo_common.actions.templates:ProcessTemplatesAction
-    tripleo.templates.upload_default = tripleo_common.actions.templates:UploadTemplatesAction
+    tripleo.templates.upload = tripleo_common.actions.templates:UploadTemplatesAction
     tripleo.validations.get_pubkey = tripleo_common.actions.validations:GetPubkeyAction
     tripleo.validations.enabled = tripleo_common.actions.validations:Enabled
     tripleo.validations.list_groups = tripleo_common.actions.validations:ListGroupsAction
     tripleo.validations.list_validations = tripleo_common.actions.validations:ListValidationsAction
     tripleo.validations.run_validation = tripleo_common.actions.validations:RunValidationAction
+    # deprecated for pike release, will be removed in queens
+    tripleo.templates.upload_default = tripleo_common.actions.templates:UploadTemplatesAction

tripleo_common/actions/templates.py

@@ -78,14 +78,15 @@ class J2SwiftLoader(jinja2.BaseLoader):
 
 class UploadTemplatesAction(base.TripleOAction):
     """Upload default heat templates for TripleO."""
-    def __init__(self, container=constants.DEFAULT_CONTAINER_NAME):
+    def __init__(self, container=constants.DEFAULT_CONTAINER_NAME,
+                 templates_path=constants.DEFAULT_TEMPLATES_PATH):
         super(UploadTemplatesAction, self).__init__()
         self.container = container
+        self.templates_path = templates_path
 
     def run(self):
-        tht_base_path = constants.DEFAULT_TEMPLATES_PATH
         with tf.NamedTemporaryFile() as tmp_tarball:
-            tarball.create_tarball(tht_base_path, tmp_tarball.name)
+            tarball.create_tarball(self.templates_path, tmp_tarball.name)
             tarball.tarball_extract_to_swift_container(
                 self.get_object_client(),
                 tmp_tarball.name,

tripleo_common/actions/vcs.py

@@ -0,0 +1,99 @@
+# Copyright 2017 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import glob
+import logging
+import shutil
+import tempfile
+
+from git import Repo
+import six
+
+from mistral.actions import base
+from mistral.workflow import utils as mistral_workflow_utils
+
+LOG = logging.getLogger(__name__)
+
+
+class GitCloneAction(base.Action):
+    """Clones a remote git repository
+
+    :param container: name of the container associated with the plan
+    :param url: url of git repository
+    :return: returns local path of cloned git repository
+    """
+
+    def __init__(self, container, url):
+        super(GitCloneAction, self).__init__()
+        self.container = container
+        self.url = url
+
+    def _checkout_reference(self, repo, ref):
+        return repo.git.checkout(repo.refs[ref])
+
+    def run(self):
+        # make a temp directory to contain the repo
+        local_dir_path = tempfile.mkdtemp(
+            suffix="_%s_import" % self.container)
+        url_bits = self.url.rsplit('@')
+        err_msg = None
+        try:
+            # create a bare repo
+            repo = Repo.clone_from(url_bits[0], local_dir_path)
+        except Exception:
+            err_msg = ("Error cloning remote repository: %s " % url_bits[0])
+            LOG.exception(err_msg)
+            return mistral_workflow_utils.Result(error=err_msg)
+
+        # if a tag value was given, checkout that tag
+        if len(url_bits) > 1:
+            try:
+                self._checkout_reference(repo, url_bits[-1])
+            except IndexError:
+                err_msg = ("Error finding %s reference "
+                           "from remote repository" % url_bits[-1])
+                LOG.exception(err_msg)
+            except Exception:
+                err_msg = ("Error checking out %s reference from remote "
+                           "repository %s" % (url_bits[-1], url_bits[0]))
+                LOG.exception(err_msg)
+
+        if err_msg:
+            return mistral_workflow_utils.Result(error=err_msg)
+
+        return local_dir_path
+
+
+class GitCleanupAction(base.Action):
+    """Removes temporary files associated with GitCloneAction operations
+
+    :param container: name of the container associated with the plan
+    :return: None if successful.  Returns error on failure to delete
+    associated temporary files
+    """
+    def __init__(self, container):
+        self.container = container
+
+    def run(self):
+        try:
+            temp_dir = tempfile.gettempdir()
+            target_path = '%s/*_%s_import' % (temp_dir, self.container)
+            path = glob.glob(target_path)[0]
+            shutil.rmtree(path)
+        except IndexError as idx_err:
+            LOG.exception("Directory not found: %s" % target_path)
+            return mistral_workflow_utils.Result(error=six.text_type(idx_err))
+        except OSError as os_err:
+            LOG.exception("Error removing directory: %s" % target_path)
+            return mistral_workflow_utils.Result(error=six.text_type(os_err))

tripleo_common/tests/actions/test_vcs.py

@@ -0,0 +1,138 @@
+# Copyright 2017 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import mock
+import os
+import shutil
+import tempfile
+import uuid
+
+import git
+from mistral.workflow import utils as mistral_workflow_utils
+
+from tripleo_common.actions import vcs
+from tripleo_common.tests import base
+
+
+class GitCloneActionTest(base.TestCase):
+
+    def setUp(self):
+        super(GitCloneActionTest, self).setUp()
+
+        self.temp_url = "/tmp/testdir"
+        self.git_url = "https://github.com/openstack/tripleo-common.git"
+        self.tag_ref = "some.test.ref"
+        self.container = "overcloudtest"
+
+    @mock.patch('tempfile.mkdtemp')
+    @mock.patch('git.Repo.clone_from')
+    def test_run(self, mock_repo_clone, mock_mkdtemp):
+
+        mock_mkdtemp.return_value = self.temp_url
+        action = vcs.GitCloneAction(self.container, self.git_url)
+        action.run()
+
+        mock_mkdtemp.assert_called()
+        mock_repo_clone.assert_called_with(self.git_url, self.temp_url)
+
+    @mock.patch('tempfile.mkdtemp')
+    @mock.patch('git.Repo.clone_from')
+    def test_run_repo_failure(self, mock_repo_clone, mock_mkdtemp):
+
+        mock_mkdtemp.return_value = self.temp_url
+        mock_repo_clone.side_effect = git.exc.GitCommandError
+        action = vcs.GitCloneAction(self.container, self.git_url)
+        result = action.run()
+
+        expected = mistral_workflow_utils.Result(
+            error="Error cloning remote repository: %s " % self.git_url
+        )
+
+        mock_mkdtemp.assert_called()
+        mock_repo_clone.assert_called_with(self.git_url, self.temp_url)
+        self.assertEqual(result, expected)
+
+    @mock.patch('tempfile.mkdtemp')
+    @mock.patch('git.Repo.clone_from')
+    @mock.patch(
+        'tripleo_common.actions.vcs.GitCloneAction._checkout_reference')
+    def test_run_ref_not_found(self, mock_checkout, mock_repo_clone,
+                               mock_mkdtemp):
+
+        mock_mkdtemp.return_value = self.temp_url
+        mock_checkout.side_effect = IndexError
+        action = vcs.GitCloneAction(
+            self.container,
+            "{url}@{tag}".format(url=self.git_url, tag=self.tag_ref)
+        )
+        result = action.run()
+
+        err_msg = ("Error finding %s reference from remote repository" %
+                   self.tag_ref)
+
+        expected = mistral_workflow_utils.Result(error=err_msg)
+
+        self.assertEqual(result, expected, "Error messages don't match.")
+
+        mock_mkdtemp.assert_called()
+        mock_repo_clone.assert_called_with(self.git_url, self.temp_url)
+
+    @mock.patch('tempfile.mkdtemp')
+    @mock.patch('git.Repo.clone_from')
+    @mock.patch(
+        'tripleo_common.actions.vcs.GitCloneAction._checkout_reference')
+    def test_run_ref_checkout_error(self, mock_checkout, mock_repo_clone,
+                                    mock_mkdtemp):
+
+        mock_mkdtemp.return_value = self.temp_url
+        mock_checkout.side_effect = git.cmd.GitCommandError
+        action = vcs.GitCloneAction(
+            self.container,
+            "{url}@{tag}".format(url=self.git_url, tag=self.tag_ref)
+        )
+        result = action.run()
+
+        err_msg = ("Error checking out %s reference from remote "
+                   "repository %s" % (self.tag_ref, self.git_url))
+
+        expected = mistral_workflow_utils.Result(error=err_msg)
+
+        self.assertEqual(result, expected, "Error messages don't match.")
+
+        mock_mkdtemp.assert_called()
+        mock_repo_clone.assert_called_with(self.git_url, self.temp_url)
+
+
+class GitCleanupActionTest(base.TestCase):
+
+    def setUp(self):
+        super(GitCleanupActionTest, self).setUp()
+        self.container = "overcloud"
+        self.temp_test_dir = tempfile.mkdtemp(
+            suffix="_%s_import" % self.container)
+
+    def tearDown(self):
+        super(GitCleanupActionTest, self).tearDown()
+        if os.path.exists(self.temp_test_dir):
+            shutil.rmtree(self.temp_test_dir)
+
+    def test_run(self):
+        action = vcs.GitCleanupAction(self.container)
+        action.run()
+        self.assertFalse(os.path.exists(self.temp_test_dir))
+
+    def test_run_with_error(self):
+        action = vcs.GitCleanupAction(str(uuid.uuid4()))
+        result = action.run()
+        self.assertIn("list index", str(result.error))

workbooks/plan_management.yaml

@@ -5,12 +5,119 @@ description: TripleO Overcloud Deployment Workflows v1
 
 workflows:
 
-  create_deployment_plan:
+  create_default_deployment_plan:
+    description: >
+      This workflow exists to maintain backwards compatibility in pike.  This
+      workflow will likely be removed in queens in favor of create_deployment_plan.
     input:
       - container
       - queue_name: tripleo
       - generate_passwords: true
     tasks:
+      call_create_deployment_plan:
+        workflow: tripleo.plan_management.v1.create_deployment_plan
+        on-success: set_status_success
+        on-error: call_create_deployment_plan_set_status_failed
+        input:
+          container: <% $.container %>
+          queue_name: <% $.queue_name %>
+          generate_passwords: <% $.queue_name %>
+          use_default_templates: true
+
+      set_status_success:
+        on-success: notify_zaqar
+        publish:
+          status: SUCCESS
+          message: <% task(call_create_deployment_plan).result %>
+
+      call_create_deployment_plan_set_status_failed:
+        on-success: notify_zaqar
+        publish:
+          status: FAILED
+          message: <% task(call_create_deployment_plan).result %>
+
+      notify_zaqar:
+        action: zaqar.queue_post
+        input:
+          queue_name: <% $.queue_name %>
+          messages:
+            body:
+              type: tripleo.plan_management.v1.create_default_deployment_plan
+              payload:
+                status: <% $.status %>
+                message: <% $.message or '' %>
+                execution: <% execution() %>
+        on-success:
+          - fail: <% $.get('status') = "FAILED" %>
+
+  create_deployment_plan:
+    description: >
+      This workflow provides the capability to create a deployment plan using
+      the default heat templates provided in a standard TripleO undercloud
+      deployment, heat templates contained in an external git repository, or a
+      swift container that already contains templates.
+    input:
+      - container
+      - source_url: null
+      - queue_name: tripleo
+      - generate_passwords: true
+      - use_default_templates: false
+
+    tasks:
+      container_required_check:
+        description: >
+          If using the default templates or importing templates from a git
+          repository, a new container needs to be created.  If using an existing
+          container containing templates, skip straight to create_plan
+        on-success:
+          - verify_container_doesnt_exist: <% $.use_default_templates or $.source_url %>
+          - create_plan: <% $.use_default_templates = false and $.source_url = null %>
+
+      verify_container_doesnt_exist:
+        action: swift.head_container container=<% $.container %>
+        on-success: notify_zaqar
+        on-error: verify_environment_doesnt_exist
+        publish:
+          status: FAILED
+          message: "Unable to create plan. The Swift container already exists"
+
+      verify_environment_doesnt_exist:
+        action: mistral.environments_get name=<% $.container %>
+        on-success: notify_zaqar
+        on-error: create_container
+        publish:
+          status: FAILED
+          message: "Unable to create plan. The Mistral environment already exists"
+
+      create_container:
+        action: tripleo.plan.create_container container=<% $.container %>
+        on-success: templates_source_check
+        on-error: create_container_set_status_failed
+
+      cleanup_temporary_files:
+        action: tripleo.git.clean container=<% $.container %>
+
+      templates_source_check:
+        on-success:
+          - upload_default_templates: <% $.use_default_templates = true %>
+          - clone_git_repo: <% $.source_url != null %>
+
+      clone_git_repo:
+        action: tripleo.git.clone url=<% $.source_url %>
+        on-success: upload_templates_directory
+        on-error: clone_git_repo_set_status_failed
+
+      upload_templates_directory:
+        action: tripleo.templates.upload container=<% $.container %> templates_path=<% task(clone_git_repo).result %>
+        on-success: create_plan
+        on-complete: cleanup_temporary_files
+        on-error: upload_templates_directory_set_status_failed
+
+      upload_default_templates:
+        action: tripleo.templates.upload container=<% $.container %>
+        on-success: create_plan
+        on-error: upload_to_container_set_status_failed
+
       create_plan:
         action: tripleo.plan.create container=<% $.container %>
         on-success:
@@ -34,6 +141,30 @@ workflows:
           status: SUCCESS
           message: <% task(create_plan).result %>
 
+      create_container_set_status_failed:
+        on-success: notify_zaqar
+        publish:
+          status: FAILED
+          message: <% task(create_container).result %>
+
+      clone_git_repo_set_status_failed:
+        on-success: notify_zaqar
+        publish:
+          status: FAILED
+          message: <% task(clone_git_repo).result %>
+
+      upload_templates_directory_set_status_failed:
+        on-success: notify_zaqar
+        publish:
+          status: FAILED
+          message: <% task(upload_templates_directory).result %>
+
+      upload_to_container_set_status_failed:
+        on-success: notify_zaqar
+        publish:
+          status: FAILED
+          message: <% task(upload_default_templates).result %>
+
       create_plan_set_status_failed:
         on-success: notify_zaqar
         publish:
@@ -70,9 +201,29 @@ workflows:
   update_deployment_plan:
     input:
       - container
+      - source_url: null
       - queue_name: tripleo
       - generate_passwords: true
     tasks:
+      templates_source_check:
+        on-success:
+          - update_plan: <% $.source_url = null %>
+          - clone_git_repo: <% $.source_url != null %>
+
+      clone_git_repo:
+        action: tripleo.git.clone url=<% $.source_url %>
+        on-success: upload_templates_directory
+        on-error: clone_git_repo_set_status_failed
+
+      upload_templates_directory:
+        action: tripleo.templates.upload container=<% $.container %> templates_path=<% task(clone_git_repo).result %>
+        on-success: update_plan
+        on-complete: cleanup_temporary_files
+        on-error: upload_templates_directory_set_status_failed
+
+      cleanup_temporary_files:
+        action: tripleo.git.clean container=<% $.container %>
+
       update_plan:
         action: tripleo.plan.update container=<% $.container %>
         on-success:
@@ -96,13 +247,24 @@ workflows:
           status: SUCCESS
           message: <% task(update_plan).result %>
 
+      clone_git_repo_set_status_failed:
+        on-success: notify_zaqar
+        publish:
+          status: FAILED
+          message: <% task(clone_git_repo).result %>
+
+      upload_templates_directory_set_status_failed:
+        on-success: notify_zaqar
+        publish:
+          status: FAILED
+          message: <% task(upload_templates_directory).result %>
+
       update_plan_set_status_failed:
         on-success: notify_zaqar
         publish:
           status: FAILED
           message: <% task(update_plan).result %>
 
-
       process_templates_set_status_failed:
         on-success: notify_zaqar
         publish:
@@ -130,107 +292,6 @@ workflows:
         on-success:
           - fail: <% $.get('status') = "FAILED" %>
 
-  create_default_deployment_plan:
-    input:
-      - container
-      - queue_name: tripleo
-      - generate_passwords: true
-    tasks:
-
-      verify_container_doesnt_exist:
-        action: swift.head_container container=<% $.container %>
-        on-success: notify_zaqar
-        on-error: verify_environment_doesnt_exist
-        publish:
-          status: FAILED
-          message: "Unable to create plan. The Swift container already exists"
-
-      verify_environment_doesnt_exist:
-        action: mistral.environments_get name=<% $.container %>
-        on-success: notify_zaqar
-        on-error: create_container
-        publish:
-          status: FAILED
-          message: "Unable to create plan. The Mistral environment already exists"
-
-      create_container:
-        action: tripleo.plan.create_container container=<% $.container %>
-        on-success: upload_to_container
-        on-error: container_set_status_failed
-
-      upload_to_container:
-        action: tripleo.templates.upload_default container=<% $.container %>
-        on-success: create_plan
-        on-error: upload_set_status_failed
-
-      create_plan:
-        action: tripleo.plan.create container=<% $.container %>
-        on-success:
-          - ensure_passwords_exist: <% $.generate_passwords = true %>
-          - plan_process_templates: <% $.generate_passwords != true %>
-        on-error: plan_set_status_failed
-
-      ensure_passwords_exist:
-        action: tripleo.parameters.generate_passwords container=<% $.container %>
-        on-success: plan_process_templates
-        on-error: ensure_passwords_exist_set_status_failed
-
-      plan_process_templates:
-        action: tripleo.templates.process container=<% $.container %>
-        on-success: plan_set_status_success
-        on-error: process_templates_set_status_failed
-
-      plan_set_status_success:
-        on-success: notify_zaqar
-        publish:
-          status: SUCCESS
-          message: <% task(create_plan).result %>
-
-      plan_set_status_failed:
-        on-success: notify_zaqar
-        publish:
-          status: FAILED
-          message: <% task(create_plan).result %>
-
-      ensure_passwords_exist_set_status_failed:
-        on-success: notify_zaqar
-        publish:
-          status: FAILED
-          message: <% task(ensure_passwords_exist).result %>
-
-      process_templates_set_status_failed:
-        on-success: notify_zaqar
-        publish:
-          status: FAILED
-          message: <% task(plan_process_templates).result %>
-
-      upload_set_status_failed:
-        on-success: notify_zaqar
-        publish:
-          status: FAILED
-          message: <% task(upload_to_container).result %>
-
-      container_set_status_failed:
-        on-success: notify_zaqar
-        publish:
-          status: FAILED
-          message: <% task(create_container).result %>
-
-      notify_zaqar:
-        action: zaqar.queue_post
-        retry: count=5 delay=1
-        input:
-          queue_name: <% $.queue_name %>
-          messages:
-            body:
-              type: tripleo.plan_management.v1.create_default_deployment_plan
-              payload:
-                status: <% $.status %>
-                message: <% $.message or '' %>
-                execution: <% execution() %>
-        on-success:
-          - fail: <% $.get('status') = "FAILED" %>
-
   get_passwords:
     description: Retrieves passwords for a given plan
     input: