diff --git a/releasenotes/notes/git-support-for-deployment-plans-cac4d3746689cbda.yaml b/releasenotes/notes/git-support-for-deployment-plans-cac4d3746689cbda.yaml new file mode 100644 index 000000000..34875b3c8 --- /dev/null +++ b/releasenotes/notes/git-support-for-deployment-plans-cac4d3746689cbda.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + The create_deployment_plan workflow has been updated to provide support for + creating a deployment plan from a git repository of heat templates. A tag + or branch can be specified in the repo url with an '@'. + Example: https://github.com/openstack/project.git@stable/newton +deprecations: + - | + The tripleo.plan_management.v1.create_default_deployment_plan is deprecated + and will be removed in the Queens release. The udpates to the + tripleo.plan_management.v1.create_deployment_plan ensures that it provides + the same functionality. diff --git a/requirements.txt b/requirements.txt index 5a810a054..38c3e4768 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,8 @@ pbr>=2.0.0 # Apache-2.0 Babel>=2.3.4 # BSD docker-py>=1.8.1 # Apache-2.0 +gitdb>=0.6.4 # BSD License (3 clause) +GitPython>=1.0.1 # BSD License (3 clause) python-heatclient>=1.6.1 # Apache-2.0 oslo.config>=3.22.0 # Apache-2.0 oslo.log>=3.22.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index e86ad0796..21daafd52 100644 --- a/setup.cfg +++ b/setup.cfg @@ -70,6 +70,8 @@ mistral.actions = tripleo.deployment.config = tripleo_common.actions.deployment:OrchestrationDeployAction tripleo.deployment.deploy = tripleo_common.actions.deployment:DeployStackAction tripleo.deployment.overcloudrc = tripleo_common.actions.deployment:OvercloudRcAction + tripleo.git.clean = tripleo_common.actions.vcs:GitCleanupAction + tripleo.git.clone = tripleo_common.actions.vcs:GitCloneAction tripleo.heat_capabilities.get = tripleo_common.actions.heat_capabilities:GetCapabilitiesAction tripleo.heat_capabilities.update = tripleo_common.actions.heat_capabilities:UpdateCapabilitiesAction tripleo.package_update.clear_breakpoints = tripleo_common.actions.package_update:ClearBreakpointsAction @@ -92,9 +94,11 @@ mistral.actions = tripleo.scale.delete_node = tripleo_common.actions.scale:ScaleDownAction tripleo.swift.tempurl = tripleo_common.actions.swifthelper:SwiftTempUrlAction tripleo.templates.process = tripleo_common.actions.templates:ProcessTemplatesAction - tripleo.templates.upload_default = tripleo_common.actions.templates:UploadTemplatesAction + tripleo.templates.upload = tripleo_common.actions.templates:UploadTemplatesAction tripleo.validations.get_pubkey = tripleo_common.actions.validations:GetPubkeyAction tripleo.validations.enabled = tripleo_common.actions.validations:Enabled tripleo.validations.list_groups = tripleo_common.actions.validations:ListGroupsAction tripleo.validations.list_validations = tripleo_common.actions.validations:ListValidationsAction tripleo.validations.run_validation = tripleo_common.actions.validations:RunValidationAction + # deprecated for pike release, will be removed in queens + tripleo.templates.upload_default = tripleo_common.actions.templates:UploadTemplatesAction diff --git a/tripleo_common/actions/templates.py b/tripleo_common/actions/templates.py index b3d096b06..4d9f6c57e 100644 --- a/tripleo_common/actions/templates.py +++ b/tripleo_common/actions/templates.py @@ -78,14 +78,15 @@ class J2SwiftLoader(jinja2.BaseLoader): class UploadTemplatesAction(base.TripleOAction): """Upload default heat templates for TripleO.""" - def __init__(self, container=constants.DEFAULT_CONTAINER_NAME): + def __init__(self, container=constants.DEFAULT_CONTAINER_NAME, + templates_path=constants.DEFAULT_TEMPLATES_PATH): super(UploadTemplatesAction, self).__init__() self.container = container + self.templates_path = templates_path def run(self): - tht_base_path = constants.DEFAULT_TEMPLATES_PATH with tf.NamedTemporaryFile() as tmp_tarball: - tarball.create_tarball(tht_base_path, tmp_tarball.name) + tarball.create_tarball(self.templates_path, tmp_tarball.name) tarball.tarball_extract_to_swift_container( self.get_object_client(), tmp_tarball.name, diff --git a/tripleo_common/actions/vcs.py b/tripleo_common/actions/vcs.py new file mode 100644 index 000000000..0a243af27 --- /dev/null +++ b/tripleo_common/actions/vcs.py @@ -0,0 +1,99 @@ +# Copyright 2017 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import glob +import logging +import shutil +import tempfile + +from git import Repo +import six + +from mistral.actions import base +from mistral.workflow import utils as mistral_workflow_utils + +LOG = logging.getLogger(__name__) + + +class GitCloneAction(base.Action): + """Clones a remote git repository + + :param container: name of the container associated with the plan + :param url: url of git repository + :return: returns local path of cloned git repository + """ + + def __init__(self, container, url): + super(GitCloneAction, self).__init__() + self.container = container + self.url = url + + def _checkout_reference(self, repo, ref): + return repo.git.checkout(repo.refs[ref]) + + def run(self): + # make a temp directory to contain the repo + local_dir_path = tempfile.mkdtemp( + suffix="_%s_import" % self.container) + url_bits = self.url.rsplit('@') + err_msg = None + try: + # create a bare repo + repo = Repo.clone_from(url_bits[0], local_dir_path) + except Exception: + err_msg = ("Error cloning remote repository: %s " % url_bits[0]) + LOG.exception(err_msg) + return mistral_workflow_utils.Result(error=err_msg) + + # if a tag value was given, checkout that tag + if len(url_bits) > 1: + try: + self._checkout_reference(repo, url_bits[-1]) + except IndexError: + err_msg = ("Error finding %s reference " + "from remote repository" % url_bits[-1]) + LOG.exception(err_msg) + except Exception: + err_msg = ("Error checking out %s reference from remote " + "repository %s" % (url_bits[-1], url_bits[0])) + LOG.exception(err_msg) + + if err_msg: + return mistral_workflow_utils.Result(error=err_msg) + + return local_dir_path + + +class GitCleanupAction(base.Action): + """Removes temporary files associated with GitCloneAction operations + + :param container: name of the container associated with the plan + :return: None if successful. Returns error on failure to delete + associated temporary files + """ + def __init__(self, container): + self.container = container + + def run(self): + try: + temp_dir = tempfile.gettempdir() + target_path = '%s/*_%s_import' % (temp_dir, self.container) + path = glob.glob(target_path)[0] + shutil.rmtree(path) + except IndexError as idx_err: + LOG.exception("Directory not found: %s" % target_path) + return mistral_workflow_utils.Result(error=six.text_type(idx_err)) + except OSError as os_err: + LOG.exception("Error removing directory: %s" % target_path) + return mistral_workflow_utils.Result(error=six.text_type(os_err)) diff --git a/tripleo_common/tests/actions/test_vcs.py b/tripleo_common/tests/actions/test_vcs.py new file mode 100644 index 000000000..9d0518218 --- /dev/null +++ b/tripleo_common/tests/actions/test_vcs.py @@ -0,0 +1,138 @@ +# Copyright 2017 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import mock +import os +import shutil +import tempfile +import uuid + +import git +from mistral.workflow import utils as mistral_workflow_utils + +from tripleo_common.actions import vcs +from tripleo_common.tests import base + + +class GitCloneActionTest(base.TestCase): + + def setUp(self): + super(GitCloneActionTest, self).setUp() + + self.temp_url = "/tmp/testdir" + self.git_url = "https://github.com/openstack/tripleo-common.git" + self.tag_ref = "some.test.ref" + self.container = "overcloudtest" + + @mock.patch('tempfile.mkdtemp') + @mock.patch('git.Repo.clone_from') + def test_run(self, mock_repo_clone, mock_mkdtemp): + + mock_mkdtemp.return_value = self.temp_url + action = vcs.GitCloneAction(self.container, self.git_url) + action.run() + + mock_mkdtemp.assert_called() + mock_repo_clone.assert_called_with(self.git_url, self.temp_url) + + @mock.patch('tempfile.mkdtemp') + @mock.patch('git.Repo.clone_from') + def test_run_repo_failure(self, mock_repo_clone, mock_mkdtemp): + + mock_mkdtemp.return_value = self.temp_url + mock_repo_clone.side_effect = git.exc.GitCommandError + action = vcs.GitCloneAction(self.container, self.git_url) + result = action.run() + + expected = mistral_workflow_utils.Result( + error="Error cloning remote repository: %s " % self.git_url + ) + + mock_mkdtemp.assert_called() + mock_repo_clone.assert_called_with(self.git_url, self.temp_url) + self.assertEqual(result, expected) + + @mock.patch('tempfile.mkdtemp') + @mock.patch('git.Repo.clone_from') + @mock.patch( + 'tripleo_common.actions.vcs.GitCloneAction._checkout_reference') + def test_run_ref_not_found(self, mock_checkout, mock_repo_clone, + mock_mkdtemp): + + mock_mkdtemp.return_value = self.temp_url + mock_checkout.side_effect = IndexError + action = vcs.GitCloneAction( + self.container, + "{url}@{tag}".format(url=self.git_url, tag=self.tag_ref) + ) + result = action.run() + + err_msg = ("Error finding %s reference from remote repository" % + self.tag_ref) + + expected = mistral_workflow_utils.Result(error=err_msg) + + self.assertEqual(result, expected, "Error messages don't match.") + + mock_mkdtemp.assert_called() + mock_repo_clone.assert_called_with(self.git_url, self.temp_url) + + @mock.patch('tempfile.mkdtemp') + @mock.patch('git.Repo.clone_from') + @mock.patch( + 'tripleo_common.actions.vcs.GitCloneAction._checkout_reference') + def test_run_ref_checkout_error(self, mock_checkout, mock_repo_clone, + mock_mkdtemp): + + mock_mkdtemp.return_value = self.temp_url + mock_checkout.side_effect = git.cmd.GitCommandError + action = vcs.GitCloneAction( + self.container, + "{url}@{tag}".format(url=self.git_url, tag=self.tag_ref) + ) + result = action.run() + + err_msg = ("Error checking out %s reference from remote " + "repository %s" % (self.tag_ref, self.git_url)) + + expected = mistral_workflow_utils.Result(error=err_msg) + + self.assertEqual(result, expected, "Error messages don't match.") + + mock_mkdtemp.assert_called() + mock_repo_clone.assert_called_with(self.git_url, self.temp_url) + + +class GitCleanupActionTest(base.TestCase): + + def setUp(self): + super(GitCleanupActionTest, self).setUp() + self.container = "overcloud" + self.temp_test_dir = tempfile.mkdtemp( + suffix="_%s_import" % self.container) + + def tearDown(self): + super(GitCleanupActionTest, self).tearDown() + if os.path.exists(self.temp_test_dir): + shutil.rmtree(self.temp_test_dir) + + def test_run(self): + action = vcs.GitCleanupAction(self.container) + action.run() + self.assertFalse(os.path.exists(self.temp_test_dir)) + + def test_run_with_error(self): + action = vcs.GitCleanupAction(str(uuid.uuid4())) + result = action.run() + self.assertIn("list index", str(result.error)) diff --git a/workbooks/plan_management.yaml b/workbooks/plan_management.yaml index 997f02fbd..6dd388523 100644 --- a/workbooks/plan_management.yaml +++ b/workbooks/plan_management.yaml @@ -5,12 +5,119 @@ description: TripleO Overcloud Deployment Workflows v1 workflows: - create_deployment_plan: + create_default_deployment_plan: + description: > + This workflow exists to maintain backwards compatibility in pike. This + workflow will likely be removed in queens in favor of create_deployment_plan. input: - container - queue_name: tripleo - generate_passwords: true tasks: + call_create_deployment_plan: + workflow: tripleo.plan_management.v1.create_deployment_plan + on-success: set_status_success + on-error: call_create_deployment_plan_set_status_failed + input: + container: <% $.container %> + queue_name: <% $.queue_name %> + generate_passwords: <% $.queue_name %> + use_default_templates: true + + set_status_success: + on-success: notify_zaqar + publish: + status: SUCCESS + message: <% task(call_create_deployment_plan).result %> + + call_create_deployment_plan_set_status_failed: + on-success: notify_zaqar + publish: + status: FAILED + message: <% task(call_create_deployment_plan).result %> + + notify_zaqar: + action: zaqar.queue_post + input: + queue_name: <% $.queue_name %> + messages: + body: + type: tripleo.plan_management.v1.create_default_deployment_plan + payload: + status: <% $.status %> + message: <% $.message or '' %> + execution: <% execution() %> + on-success: + - fail: <% $.get('status') = "FAILED" %> + + create_deployment_plan: + description: > + This workflow provides the capability to create a deployment plan using + the default heat templates provided in a standard TripleO undercloud + deployment, heat templates contained in an external git repository, or a + swift container that already contains templates. + input: + - container + - source_url: null + - queue_name: tripleo + - generate_passwords: true + - use_default_templates: false + + tasks: + container_required_check: + description: > + If using the default templates or importing templates from a git + repository, a new container needs to be created. If using an existing + container containing templates, skip straight to create_plan + on-success: + - verify_container_doesnt_exist: <% $.use_default_templates or $.source_url %> + - create_plan: <% $.use_default_templates = false and $.source_url = null %> + + verify_container_doesnt_exist: + action: swift.head_container container=<% $.container %> + on-success: notify_zaqar + on-error: verify_environment_doesnt_exist + publish: + status: FAILED + message: "Unable to create plan. The Swift container already exists" + + verify_environment_doesnt_exist: + action: mistral.environments_get name=<% $.container %> + on-success: notify_zaqar + on-error: create_container + publish: + status: FAILED + message: "Unable to create plan. The Mistral environment already exists" + + create_container: + action: tripleo.plan.create_container container=<% $.container %> + on-success: templates_source_check + on-error: create_container_set_status_failed + + cleanup_temporary_files: + action: tripleo.git.clean container=<% $.container %> + + templates_source_check: + on-success: + - upload_default_templates: <% $.use_default_templates = true %> + - clone_git_repo: <% $.source_url != null %> + + clone_git_repo: + action: tripleo.git.clone url=<% $.source_url %> + on-success: upload_templates_directory + on-error: clone_git_repo_set_status_failed + + upload_templates_directory: + action: tripleo.templates.upload container=<% $.container %> templates_path=<% task(clone_git_repo).result %> + on-success: create_plan + on-complete: cleanup_temporary_files + on-error: upload_templates_directory_set_status_failed + + upload_default_templates: + action: tripleo.templates.upload container=<% $.container %> + on-success: create_plan + on-error: upload_to_container_set_status_failed + create_plan: action: tripleo.plan.create container=<% $.container %> on-success: @@ -34,6 +141,30 @@ workflows: status: SUCCESS message: <% task(create_plan).result %> + create_container_set_status_failed: + on-success: notify_zaqar + publish: + status: FAILED + message: <% task(create_container).result %> + + clone_git_repo_set_status_failed: + on-success: notify_zaqar + publish: + status: FAILED + message: <% task(clone_git_repo).result %> + + upload_templates_directory_set_status_failed: + on-success: notify_zaqar + publish: + status: FAILED + message: <% task(upload_templates_directory).result %> + + upload_to_container_set_status_failed: + on-success: notify_zaqar + publish: + status: FAILED + message: <% task(upload_default_templates).result %> + create_plan_set_status_failed: on-success: notify_zaqar publish: @@ -70,9 +201,29 @@ workflows: update_deployment_plan: input: - container + - source_url: null - queue_name: tripleo - generate_passwords: true tasks: + templates_source_check: + on-success: + - update_plan: <% $.source_url = null %> + - clone_git_repo: <% $.source_url != null %> + + clone_git_repo: + action: tripleo.git.clone url=<% $.source_url %> + on-success: upload_templates_directory + on-error: clone_git_repo_set_status_failed + + upload_templates_directory: + action: tripleo.templates.upload container=<% $.container %> templates_path=<% task(clone_git_repo).result %> + on-success: update_plan + on-complete: cleanup_temporary_files + on-error: upload_templates_directory_set_status_failed + + cleanup_temporary_files: + action: tripleo.git.clean container=<% $.container %> + update_plan: action: tripleo.plan.update container=<% $.container %> on-success: @@ -96,13 +247,24 @@ workflows: status: SUCCESS message: <% task(update_plan).result %> + clone_git_repo_set_status_failed: + on-success: notify_zaqar + publish: + status: FAILED + message: <% task(clone_git_repo).result %> + + upload_templates_directory_set_status_failed: + on-success: notify_zaqar + publish: + status: FAILED + message: <% task(upload_templates_directory).result %> + update_plan_set_status_failed: on-success: notify_zaqar publish: status: FAILED message: <% task(update_plan).result %> - process_templates_set_status_failed: on-success: notify_zaqar publish: @@ -130,107 +292,6 @@ workflows: on-success: - fail: <% $.get('status') = "FAILED" %> - create_default_deployment_plan: - input: - - container - - queue_name: tripleo - - generate_passwords: true - tasks: - - verify_container_doesnt_exist: - action: swift.head_container container=<% $.container %> - on-success: notify_zaqar - on-error: verify_environment_doesnt_exist - publish: - status: FAILED - message: "Unable to create plan. The Swift container already exists" - - verify_environment_doesnt_exist: - action: mistral.environments_get name=<% $.container %> - on-success: notify_zaqar - on-error: create_container - publish: - status: FAILED - message: "Unable to create plan. The Mistral environment already exists" - - create_container: - action: tripleo.plan.create_container container=<% $.container %> - on-success: upload_to_container - on-error: container_set_status_failed - - upload_to_container: - action: tripleo.templates.upload_default container=<% $.container %> - on-success: create_plan - on-error: upload_set_status_failed - - create_plan: - action: tripleo.plan.create container=<% $.container %> - on-success: - - ensure_passwords_exist: <% $.generate_passwords = true %> - - plan_process_templates: <% $.generate_passwords != true %> - on-error: plan_set_status_failed - - ensure_passwords_exist: - action: tripleo.parameters.generate_passwords container=<% $.container %> - on-success: plan_process_templates - on-error: ensure_passwords_exist_set_status_failed - - plan_process_templates: - action: tripleo.templates.process container=<% $.container %> - on-success: plan_set_status_success - on-error: process_templates_set_status_failed - - plan_set_status_success: - on-success: notify_zaqar - publish: - status: SUCCESS - message: <% task(create_plan).result %> - - plan_set_status_failed: - on-success: notify_zaqar - publish: - status: FAILED - message: <% task(create_plan).result %> - - ensure_passwords_exist_set_status_failed: - on-success: notify_zaqar - publish: - status: FAILED - message: <% task(ensure_passwords_exist).result %> - - process_templates_set_status_failed: - on-success: notify_zaqar - publish: - status: FAILED - message: <% task(plan_process_templates).result %> - - upload_set_status_failed: - on-success: notify_zaqar - publish: - status: FAILED - message: <% task(upload_to_container).result %> - - container_set_status_failed: - on-success: notify_zaqar - publish: - status: FAILED - message: <% task(create_container).result %> - - notify_zaqar: - action: zaqar.queue_post - retry: count=5 delay=1 - input: - queue_name: <% $.queue_name %> - messages: - body: - type: tripleo.plan_management.v1.create_default_deployment_plan - payload: - status: <% $.status %> - message: <% $.message or '' %> - execution: <% execution() %> - on-success: - - fail: <% $.get('status') = "FAILED" %> - get_passwords: description: Retrieves passwords for a given plan input: