Create or update deployment plan from git repository

This patch refactors the plan_management workbook to reduce
redundancy in creating deployment plans and adds the ability
to use templates stored in an external git repo when creating
or updating a deployment plan.  The workbook has tasks added
to the workflows that cleanup temporary files created during
the git clone operations.

A tag or branch can be specified in the url with an '@'.
Example: https://github.com/openstack/project.git@stable/newton

Implements: blueprint git-deployment-plan
Change-Id: If069ebb33ee4cb7f81308c68849ba9f969752d25
This commit is contained in:
Ryan Brady 2017-01-05 23:19:13 -05:00 committed by Dougal Matthews
parent 456b7ddcd8
commit 6b2fefdeb7
7 changed files with 425 additions and 107 deletions

View File

@ -0,0 +1,13 @@
---
features:
- |
The create_deployment_plan workflow has been updated to provide support for
creating a deployment plan from a git repository of heat templates. A tag
or branch can be specified in the repo url with an '@'.
Example: https://github.com/openstack/project.git@stable/newton
deprecations:
- |
The tripleo.plan_management.v1.create_default_deployment_plan is deprecated
and will be removed in the Queens release. The udpates to the
tripleo.plan_management.v1.create_deployment_plan ensures that it provides
the same functionality.

View File

@ -5,6 +5,8 @@
pbr>=2.0.0 # Apache-2.0
Babel>=2.3.4 # BSD
docker-py>=1.8.1 # Apache-2.0
gitdb>=0.6.4 # BSD License (3 clause)
GitPython>=1.0.1 # BSD License (3 clause)
python-heatclient>=1.6.1 # Apache-2.0
oslo.config>=3.22.0 # Apache-2.0
oslo.log>=3.11.0 # Apache-2.0

View File

@ -69,6 +69,8 @@ mistral.actions =
tripleo.deployment.config = tripleo_common.actions.deployment:OrchestrationDeployAction
tripleo.deployment.deploy = tripleo_common.actions.deployment:DeployStackAction
tripleo.deployment.overcloudrc = tripleo_common.actions.deployment:OvercloudRcAction
tripleo.git.clean = tripleo_common.actions.vcs:GitCleanupAction
tripleo.git.clone = tripleo_common.actions.vcs:GitCloneAction
tripleo.heat_capabilities.get = tripleo_common.actions.heat_capabilities:GetCapabilitiesAction
tripleo.heat_capabilities.update = tripleo_common.actions.heat_capabilities:UpdateCapabilitiesAction
tripleo.package_update.clear_breakpoints = tripleo_common.actions.package_update:ClearBreakpointsAction
@ -91,9 +93,11 @@ mistral.actions =
tripleo.scale.delete_node = tripleo_common.actions.scale:ScaleDownAction
tripleo.swift.tempurl = tripleo_common.actions.swifthelper:SwiftTempUrlAction
tripleo.templates.process = tripleo_common.actions.templates:ProcessTemplatesAction
tripleo.templates.upload_default = tripleo_common.actions.templates:UploadTemplatesAction
tripleo.templates.upload = tripleo_common.actions.templates:UploadTemplatesAction
tripleo.validations.get_pubkey = tripleo_common.actions.validations:GetPubkeyAction
tripleo.validations.enabled = tripleo_common.actions.validations:Enabled
tripleo.validations.list_groups = tripleo_common.actions.validations:ListGroupsAction
tripleo.validations.list_validations = tripleo_common.actions.validations:ListValidationsAction
tripleo.validations.run_validation = tripleo_common.actions.validations:RunValidationAction
# deprecated for pike release, will be removed in queens
tripleo.templates.upload_default = tripleo_common.actions.templates:UploadTemplatesAction

View File

@ -78,14 +78,15 @@ class J2SwiftLoader(jinja2.BaseLoader):
class UploadTemplatesAction(base.TripleOAction):
"""Upload default heat templates for TripleO."""
def __init__(self, container=constants.DEFAULT_CONTAINER_NAME):
def __init__(self, container=constants.DEFAULT_CONTAINER_NAME,
templates_path=constants.DEFAULT_TEMPLATES_PATH):
super(UploadTemplatesAction, self).__init__()
self.container = container
self.templates_path = templates_path
def run(self):
tht_base_path = constants.DEFAULT_TEMPLATES_PATH
with tf.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tht_base_path, tmp_tarball.name)
tarball.create_tarball(self.templates_path, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(
self.get_object_client(),
tmp_tarball.name,

View File

@ -0,0 +1,99 @@
# Copyright 2017 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import glob
import logging
import shutil
import tempfile
from git import Repo
import six
from mistral.actions import base
from mistral.workflow import utils as mistral_workflow_utils
LOG = logging.getLogger(__name__)
class GitCloneAction(base.Action):
"""Clones a remote git repository
:param container: name of the container associated with the plan
:param url: url of git repository
:return: returns local path of cloned git repository
"""
def __init__(self, container, url):
super(GitCloneAction, self).__init__()
self.container = container
self.url = url
def _checkout_reference(self, repo, ref):
return repo.git.checkout(repo.refs[ref])
def run(self):
# make a temp directory to contain the repo
local_dir_path = tempfile.mkdtemp(
suffix="_%s_import" % self.container)
url_bits = self.url.rsplit('@')
err_msg = None
try:
# create a bare repo
repo = Repo.clone_from(url_bits[0], local_dir_path)
except Exception:
err_msg = ("Error cloning remote repository: %s " % url_bits[0])
LOG.exception(err_msg)
return mistral_workflow_utils.Result(error=err_msg)
# if a tag value was given, checkout that tag
if len(url_bits) > 1:
try:
self._checkout_reference(repo, url_bits[-1])
except IndexError:
err_msg = ("Error finding %s reference "
"from remote repository" % url_bits[-1])
LOG.exception(err_msg)
except Exception:
err_msg = ("Error checking out %s reference from remote "
"repository %s" % (url_bits[-1], url_bits[0]))
LOG.exception(err_msg)
if err_msg:
return mistral_workflow_utils.Result(error=err_msg)
return local_dir_path
class GitCleanupAction(base.Action):
"""Removes temporary files associated with GitCloneAction operations
:param container: name of the container associated with the plan
:return: None if successful. Returns error on failure to delete
associated temporary files
"""
def __init__(self, container):
self.container = container
def run(self):
try:
temp_dir = tempfile.gettempdir()
target_path = '%s/*_%s_import' % (temp_dir, self.container)
path = glob.glob(target_path)[0]
shutil.rmtree(path)
except IndexError as idx_err:
LOG.exception("Directory not found: %s" % target_path)
return mistral_workflow_utils.Result(error=six.text_type(idx_err))
except OSError as os_err:
LOG.exception("Error removing directory: %s" % target_path)
return mistral_workflow_utils.Result(error=six.text_type(os_err))

View File

@ -0,0 +1,138 @@
# Copyright 2017 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
import os
import shutil
import tempfile
import uuid
import git
from mistral.workflow import utils as mistral_workflow_utils
from tripleo_common.actions import vcs
from tripleo_common.tests import base
class GitCloneActionTest(base.TestCase):
def setUp(self):
super(GitCloneActionTest, self).setUp()
self.temp_url = "/tmp/testdir"
self.git_url = "https://github.com/openstack/tripleo-common.git"
self.tag_ref = "some.test.ref"
self.container = "overcloudtest"
@mock.patch('tempfile.mkdtemp')
@mock.patch('git.Repo.clone_from')
def test_run(self, mock_repo_clone, mock_mkdtemp):
mock_mkdtemp.return_value = self.temp_url
action = vcs.GitCloneAction(self.container, self.git_url)
action.run()
mock_mkdtemp.assert_called()
mock_repo_clone.assert_called_with(self.git_url, self.temp_url)
@mock.patch('tempfile.mkdtemp')
@mock.patch('git.Repo.clone_from')
def test_run_repo_failure(self, mock_repo_clone, mock_mkdtemp):
mock_mkdtemp.return_value = self.temp_url
mock_repo_clone.side_effect = git.exc.GitCommandError
action = vcs.GitCloneAction(self.container, self.git_url)
result = action.run()
expected = mistral_workflow_utils.Result(
error="Error cloning remote repository: %s " % self.git_url
)
mock_mkdtemp.assert_called()
mock_repo_clone.assert_called_with(self.git_url, self.temp_url)
self.assertEqual(result, expected)
@mock.patch('tempfile.mkdtemp')
@mock.patch('git.Repo.clone_from')
@mock.patch(
'tripleo_common.actions.vcs.GitCloneAction._checkout_reference')
def test_run_ref_not_found(self, mock_checkout, mock_repo_clone,
mock_mkdtemp):
mock_mkdtemp.return_value = self.temp_url
mock_checkout.side_effect = IndexError
action = vcs.GitCloneAction(
self.container,
"{url}@{tag}".format(url=self.git_url, tag=self.tag_ref)
)
result = action.run()
err_msg = ("Error finding %s reference from remote repository" %
self.tag_ref)
expected = mistral_workflow_utils.Result(error=err_msg)
self.assertEqual(result, expected, "Error messages don't match.")
mock_mkdtemp.assert_called()
mock_repo_clone.assert_called_with(self.git_url, self.temp_url)
@mock.patch('tempfile.mkdtemp')
@mock.patch('git.Repo.clone_from')
@mock.patch(
'tripleo_common.actions.vcs.GitCloneAction._checkout_reference')
def test_run_ref_checkout_error(self, mock_checkout, mock_repo_clone,
mock_mkdtemp):
mock_mkdtemp.return_value = self.temp_url
mock_checkout.side_effect = git.cmd.GitCommandError
action = vcs.GitCloneAction(
self.container,
"{url}@{tag}".format(url=self.git_url, tag=self.tag_ref)
)
result = action.run()
err_msg = ("Error checking out %s reference from remote "
"repository %s" % (self.tag_ref, self.git_url))
expected = mistral_workflow_utils.Result(error=err_msg)
self.assertEqual(result, expected, "Error messages don't match.")
mock_mkdtemp.assert_called()
mock_repo_clone.assert_called_with(self.git_url, self.temp_url)
class GitCleanupActionTest(base.TestCase):
def setUp(self):
super(GitCleanupActionTest, self).setUp()
self.container = "overcloud"
self.temp_test_dir = tempfile.mkdtemp(
suffix="_%s_import" % self.container)
def tearDown(self):
super(GitCleanupActionTest, self).tearDown()
if os.path.exists(self.temp_test_dir):
shutil.rmtree(self.temp_test_dir)
def test_run(self):
action = vcs.GitCleanupAction(self.container)
action.run()
self.assertFalse(os.path.exists(self.temp_test_dir))
def test_run_with_error(self):
action = vcs.GitCleanupAction(str(uuid.uuid4()))
result = action.run()
self.assertIn("list index", str(result.error))

View File

@ -5,12 +5,119 @@ description: TripleO Overcloud Deployment Workflows v1
workflows:
create_deployment_plan:
create_default_deployment_plan:
description: >
This workflow exists to maintain backwards compatibility in pike. This
workflow will likely be removed in queens in favor of create_deployment_plan.
input:
- container
- queue_name: tripleo
- generate_passwords: true
tasks:
call_create_deployment_plan:
workflow: tripleo.plan_management.v1.create_deployment_plan
on-success: set_status_success
on-error: call_create_deployment_plan_set_status_failed
input:
container: <% $.container %>
queue_name: <% $.queue_name %>
generate_passwords: <% $.queue_name %>
use_default_templates: true
set_status_success:
on-success: notify_zaqar
publish:
status: SUCCESS
message: <% task(call_create_deployment_plan).result %>
call_create_deployment_plan_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(call_create_deployment_plan).result %>
notify_zaqar:
action: zaqar.queue_post
input:
queue_name: <% $.queue_name %>
messages:
body:
type: tripleo.plan_management.v1.create_default_deployment_plan
payload:
status: <% $.status %>
message: <% $.message or '' %>
execution: <% execution() %>
on-success:
- fail: <% $.get('status') = "FAILED" %>
create_deployment_plan:
description: >
This workflow provides the capability to create a deployment plan using
the default heat templates provided in a standard TripleO undercloud
deployment, heat templates contained in an external git repository, or a
swift container that already contains templates.
input:
- container
- source_url: null
- queue_name: tripleo
- generate_passwords: true
- use_default_templates: false
tasks:
container_required_check:
description: >
If using the default templates or importing templates from a git
repository, a new container needs to be created. If using an existing
container containing templates, skip straight to create_plan
on-success:
- verify_container_doesnt_exist: <% $.use_default_templates or $.source_url %>
- create_plan: <% $.use_default_templates = false and $.source_url = null %>
verify_container_doesnt_exist:
action: swift.head_container container=<% $.container %>
on-success: notify_zaqar
on-error: verify_environment_doesnt_exist
publish:
status: FAILED
message: "Unable to create plan. The Swift container already exists"
verify_environment_doesnt_exist:
action: mistral.environments_get name=<% $.container %>
on-success: notify_zaqar
on-error: create_container
publish:
status: FAILED
message: "Unable to create plan. The Mistral environment already exists"
create_container:
action: tripleo.plan.create_container container=<% $.container %>
on-success: templates_source_check
on-error: create_container_set_status_failed
cleanup_temporary_files:
action: tripleo.git.clean container=<% $.container %>
templates_source_check:
on-success:
- upload_default_templates: <% $.use_default_templates = true %>
- clone_git_repo: <% $.source_url != null %>
clone_git_repo:
action: tripleo.git.clone url=<% $.source_url %>
on-success: upload_templates_directory
on-error: clone_git_repo_set_status_failed
upload_templates_directory:
action: tripleo.templates.upload container=<% $.container %> templates_path=<% task(clone_git_repo).result %>
on-success: create_plan
on-complete: cleanup_temporary_files
on-error: upload_templates_directory_set_status_failed
upload_default_templates:
action: tripleo.templates.upload container=<% $.container %>
on-success: create_plan
on-error: upload_to_container_set_status_failed
create_plan:
action: tripleo.plan.create container=<% $.container %>
on-success:
@ -34,6 +141,30 @@ workflows:
status: SUCCESS
message: <% task(create_plan).result %>
create_container_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(create_container).result %>
clone_git_repo_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(clone_git_repo).result %>
upload_templates_directory_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(upload_templates_directory).result %>
upload_to_container_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(upload_default_templates).result %>
create_plan_set_status_failed:
on-success: notify_zaqar
publish:
@ -70,9 +201,29 @@ workflows:
update_deployment_plan:
input:
- container
- source_url: null
- queue_name: tripleo
- generate_passwords: true
tasks:
templates_source_check:
on-success:
- update_plan: <% $.source_url = null %>
- clone_git_repo: <% $.source_url != null %>
clone_git_repo:
action: tripleo.git.clone url=<% $.source_url %>
on-success: upload_templates_directory
on-error: clone_git_repo_set_status_failed
upload_templates_directory:
action: tripleo.templates.upload container=<% $.container %> templates_path=<% task(clone_git_repo).result %>
on-success: update_plan
on-complete: cleanup_temporary_files
on-error: upload_templates_directory_set_status_failed
cleanup_temporary_files:
action: tripleo.git.clean container=<% $.container %>
update_plan:
action: tripleo.plan.update container=<% $.container %>
on-success:
@ -96,13 +247,24 @@ workflows:
status: SUCCESS
message: <% task(update_plan).result %>
clone_git_repo_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(clone_git_repo).result %>
upload_templates_directory_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(upload_templates_directory).result %>
update_plan_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(update_plan).result %>
process_templates_set_status_failed:
on-success: notify_zaqar
publish:
@ -130,107 +292,6 @@ workflows:
on-success:
- fail: <% $.get('status') = "FAILED" %>
create_default_deployment_plan:
input:
- container
- queue_name: tripleo
- generate_passwords: true
tasks:
verify_container_doesnt_exist:
action: swift.head_container container=<% $.container %>
on-success: notify_zaqar
on-error: verify_environment_doesnt_exist
publish:
status: FAILED
message: "Unable to create plan. The Swift container already exists"
verify_environment_doesnt_exist:
action: mistral.environments_get name=<% $.container %>
on-success: notify_zaqar
on-error: create_container
publish:
status: FAILED
message: "Unable to create plan. The Mistral environment already exists"
create_container:
action: tripleo.plan.create_container container=<% $.container %>
on-success: upload_to_container
on-error: container_set_status_failed
upload_to_container:
action: tripleo.templates.upload_default container=<% $.container %>
on-success: create_plan
on-error: upload_set_status_failed
create_plan:
action: tripleo.plan.create container=<% $.container %>
on-success:
- ensure_passwords_exist: <% $.generate_passwords = true %>
- plan_process_templates: <% $.generate_passwords != true %>
on-error: plan_set_status_failed
ensure_passwords_exist:
action: tripleo.parameters.generate_passwords container=<% $.container %>
on-success: plan_process_templates
on-error: ensure_passwords_exist_set_status_failed
plan_process_templates:
action: tripleo.templates.process container=<% $.container %>
on-success: plan_set_status_success
on-error: process_templates_set_status_failed
plan_set_status_success:
on-success: notify_zaqar
publish:
status: SUCCESS
message: <% task(create_plan).result %>
plan_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(create_plan).result %>
ensure_passwords_exist_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(ensure_passwords_exist).result %>
process_templates_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(plan_process_templates).result %>
upload_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(upload_to_container).result %>
container_set_status_failed:
on-success: notify_zaqar
publish:
status: FAILED
message: <% task(create_container).result %>
notify_zaqar:
action: zaqar.queue_post
retry: count=5 delay=1
input:
queue_name: <% $.queue_name %>
messages:
body:
type: tripleo.plan_management.v1.create_default_deployment_plan
payload:
status: <% $.status %>
message: <% $.message or '' %>
execution: <% execution() %>
on-success:
- fail: <% $.get('status') = "FAILED" %>
get_passwords:
description: Retrieves passwords for a given plan
input: