From 0a3d617590e31b5c7200203111219a45f2651b70 Mon Sep 17 00:00:00 2001 From: Mathieu Bultel Date: Wed, 26 Jul 2017 18:01:56 +0200 Subject: [PATCH] Execute minor update via ansible on mistral This review implement the minor update workflow which does: - a noop heat stack deployment to set and refresh the heat config output. - download the heat config and put them in a ansible playbooks - run the playbooks with the mistral action It adds actions for the config download and update deployment Adds the config download as a library for being call either, by the client or mistral Closes-Bug: #1715557 Change-Id: I199b35f865c0e68d28c5ddd82e5b8fe61abb5f33 (cherry picked from commit e9a4156dddd389d8f61183e75eac6b68571475ff) --- setup.cfg | 4 +- tripleo_common/actions/config.py | 79 +++++++++ tripleo_common/actions/package_update.py | 77 ++++----- tripleo_common/actions/plan.py | 59 +++++++ tripleo_common/constants.py | 3 + tripleo_common/tests/actions/test_config.py | 118 ++++++++++++++ .../tests/actions/test_package_update.py | 151 ++++++++---------- tripleo_common/tests/fake_config/__init__.py | 0 tripleo_common/tests/fake_config/fakes.py | 91 +++++++++++ tripleo_common/tests/utils/test_config.py | 143 +++++++++++++++++ tripleo_common/utils/config.py | 142 ++++++++++++++++ workbooks/package_update.yaml | 88 +++++++--- 12 files changed, 801 insertions(+), 154 deletions(-) create mode 100644 tripleo_common/actions/config.py create mode 100644 tripleo_common/tests/actions/test_config.py create mode 100644 tripleo_common/tests/fake_config/__init__.py create mode 100644 tripleo_common/tests/fake_config/fakes.py create mode 100644 tripleo_common/tests/utils/test_config.py create mode 100644 tripleo_common/utils/config.py diff --git a/setup.cfg b/setup.cfg index f9c698625..7694370db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -73,6 +73,8 @@ mistral.actions = tripleo.baremetal.validate_nodes = tripleo_common.actions.baremetal:ValidateNodes tripleo.baremetal.get_candidate_nodes = tripleo_common.actions.baremetal:GetCandidateNodes tripleo.baremetal.probe_node = tripleo_common.actions.baremetal:ProbeNode + tripleo.config.download_config = tripleo_common.actions.config:DownloadConfigAction + tripleo.config.get_overcloud_config = tripleo_common.actions.config:GetOvercloudConfig tripleo.deployment.config = tripleo_common.actions.deployment:OrchestrationDeployAction tripleo.deployment.deploy = tripleo_common.actions.deployment:DeployStackAction tripleo.deployment.overcloudrc = tripleo_common.actions.deployment:OvercloudRcAction @@ -86,7 +88,6 @@ mistral.actions = tripleo.git.clone = tripleo_common.actions.vcs:GitCloneAction tripleo.heat_capabilities.get = tripleo_common.actions.heat_capabilities:GetCapabilitiesAction tripleo.heat_capabilities.update = tripleo_common.actions.heat_capabilities:UpdateCapabilitiesAction - tripleo.package_update.clear_breakpoints = tripleo_common.actions.package_update:ClearBreakpointsAction tripleo.package_update.update_stack = tripleo_common.actions.package_update:UpdateStackAction tripleo.parameters.get = tripleo_common.actions.parameters:GetParametersAction tripleo.parameters.get_flatten = tripleo_common.actions.parameters:GetFlattenedParametersAction @@ -103,6 +104,7 @@ mistral.actions = tripleo.plan.delete = tripleo_common.actions.plan:DeletePlanAction tripleo.plan.list = tripleo_common.actions.plan:ListPlansAction tripleo.plan.export = tripleo_common.actions.plan:ExportPlanAction + tripleo.plan.update_from_dir = tripleo_common.actions.plan:UpdatePlanFromDirAction tripleo.logging_to_swift.format_messages = tripleo_common.actions.logging_to_swift:FormatMessagesAction tripleo.logging_to_swift.publish_ui_log_to_swift = tripleo_common.actions.logging_to_swift:PublishUILogToSwiftAction tripleo.logging_to_swift.prepare_log_download = tripleo_common.actions.logging_to_swift:PrepareLogDownloadAction diff --git a/tripleo_common/actions/config.py b/tripleo_common/actions/config.py new file mode 100644 index 000000000..d72cf2a53 --- /dev/null +++ b/tripleo_common/actions/config.py @@ -0,0 +1,79 @@ +# Copyright 2016 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import logging +import os +import shutil +import tempfile + +from tripleo_common.actions import templates +from tripleo_common import constants +from tripleo_common.utils import config as ooo_config +from tripleo_common.utils import swift as swiftutils +from tripleo_common.utils import tarball + +LOG = logging.getLogger(__name__) + + +class GetOvercloudConfig(templates.ProcessTemplatesAction): + """Get the Overcloud Config from the Heat outputs + + This action gets the Overcloud config from the Heat outputs and + write it to the disk to be call with Ansible. + + :param container: name of the Swift container / plan name + config_dir: directory where the config should be written + """ + + def __init__(self, container=constants.DEFAULT_CONTAINER_NAME, + config_dir=tempfile.gettempdir(), + container_config=constants.CONFIG_CONTAINER_NAME): + super(GetOvercloudConfig, self).__init__(container) + self.container = container + self.config_dir = config_dir + self.container_config = container_config + + def run(self, context): + heat = self.get_orchestration_client(context) + config = ooo_config.Config(heat) + config_path = config.download_config(self.container, self.config_dir) + + with tempfile.NamedTemporaryFile() as tmp_tarball: + tarball.create_tarball(config_path, tmp_tarball.name) + tarball.tarball_extract_to_swift_container( + self.get_object_client(context), + tmp_tarball.name, + self.container_config) + if os.path.exists(config_path): + shutil.rmtree(config_path) + + +class DownloadConfigAction(templates.ProcessTemplatesAction): + """Download the container config from swift + + This action downloads a container which contain the heat config output + + :param container: name of the Swift container / plan name + """ + + def __init__(self, container_config=constants.CONFIG_CONTAINER_NAME): + super(DownloadConfigAction, self).__init__(container_config) + self.container_config = container_config + + def run(self, context): + swift = self.get_object_client(context) + tmp_dir = tempfile.mkdtemp(prefix='tripleo-', + suffix='-config') + swiftutils.download_container(swift, self.container_config, tmp_dir) + return tmp_dir diff --git a/tripleo_common/actions/package_update.py b/tripleo_common/actions/package_update.py index f39a201e5..ed7b880f4 100644 --- a/tripleo_common/actions/package_update.py +++ b/tripleo_common/actions/package_update.py @@ -13,41 +13,26 @@ # License for the specific language governing permissions and limitations # under the License. import logging -import time from heatclient.common import template_utils from heatclient import exc as heat_exc from mistral_lib import actions from swiftclient import exceptions as swiftexceptions -from tripleo_common.actions import base from tripleo_common.actions import templates from tripleo_common import constants -from tripleo_common.update import PackageUpdateManager from tripleo_common.utils import plan as plan_utils LOG = logging.getLogger(__name__) -class ClearBreakpointsAction(base.TripleOAction): - def __init__(self, stack_id, refs): - super(ClearBreakpointsAction, self).__init__() - self.stack_id = stack_id - self.refs = refs - - def run(self, context): - heat = self.get_orchestration_client(context) - nova = self.get_compute_client(context) - update_manager = PackageUpdateManager( - heat, nova, self.stack_id, stack_fields={}) - update_manager.clear_breakpoints(self.refs) - - class UpdateStackAction(templates.ProcessTemplatesAction): - def __init__(self, timeout, container=constants.DEFAULT_CONTAINER_NAME): + def __init__(self, timeout, container_registry, + container=constants.DEFAULT_CONTAINER_NAME): super(UpdateStackAction, self).__init__(container) self.timeout_mins = timeout + self.container_registry = container_registry def run(self, context): # get the stack. Error if doesn't exist @@ -59,12 +44,6 @@ class UpdateStackAction(templates.ProcessTemplatesAction): LOG.exception(msg) return actions.Result(error=msg) - parameters = dict() - timestamp = int(time.time()) - parameters['DeployIdentifier'] = timestamp - parameters['UpdateIdentifier'] = timestamp - parameters['StackAction'] = 'UPDATE' - swift = self.get_object_client(context) try: @@ -75,14 +54,30 @@ class UpdateStackAction(templates.ProcessTemplatesAction): LOG.exception(err_msg) return actions.Result(error=err_msg) - try: - plan_utils.update_in_env(swift, env, 'parameter_defaults', - parameters) - except swiftexceptions.ClientException as err: - err_msg = ("Error updating environment for plan %s: %s" % ( - self.container, err)) - LOG.exception(err_msg) - return actions.Result(error=err_msg) + update_env = {} + if self.container_registry is not None: + update_env.update(self.container_registry) + + noop_env = { + 'resource_registry': { + 'OS::TripleO::DeploymentSteps': 'OS::Heat::None', + }, + } + + for output in stack.to_dict().get('outputs', {}): + if output['output_key'] == 'RoleData': + for role in output['output_value']: + role_env = { + "OS::TripleO::Tasks::%sPreConfig" % role: + 'OS::Heat::None', + "OS::TripleO::Tasks::%sPostConfig" % role: + 'OS::Heat::None', + } + noop_env['resource_registry'].update(role_env) + update_env.update(noop_env) + template_utils.deep_update(env, update_env) + plan_utils.update_in_env(swift, env, 'parameter_defaults', + self.container_registry['parameter_defaults']) # process all plan files and create or update a stack processed_data = super(UpdateStackAction, self).run(context) @@ -94,24 +89,6 @@ class UpdateStackAction(templates.ProcessTemplatesAction): stack_args = processed_data.copy() - env = stack_args.get('environment', {}) - template_utils.deep_update(env, { - 'resource_registry': { - 'resources': { - '*': { - '*': { - constants.UPDATE_RESOURCE_NAME: { - 'hooks': 'pre-update'} - } - } - } - } - }) - stack_args['environment'] = env - - stack_args['timeout_mins'] = self.timeout_mins - stack_args['existing'] = 'true' - LOG.info("Performing Heat stack update") LOG.info('updating stack: %s', stack.stack_name) return heat.stacks.update(stack.id, **stack_args) diff --git a/tripleo_common/actions/plan.py b/tripleo_common/actions/plan.py index a9ed4c490..077642d56 100644 --- a/tripleo_common/actions/plan.py +++ b/tripleo_common/actions/plan.py @@ -30,6 +30,7 @@ from tripleo_common import constants from tripleo_common import exception from tripleo_common.utils import plan as plan_utils from tripleo_common.utils import swift as swiftutils +from tripleo_common.utils import tarball from tripleo_common.utils.validations import pattern_validator @@ -243,3 +244,61 @@ class ExportPlanAction(base.TripleOAction): return actions.Result(error=msg) finally: shutil.rmtree(tmp_dir) + + +class UpdatePlanFromDirAction(base.TripleOAction): + """Updates a plan and associated files + + Updates a plan by comparing the current files with the new ones + provided: + Updates only new files from the plan + Add new files from the plan + + :param container: name of the Swift container / plan name + """ + + def __init__(self, container=constants.DEFAULT_CONTAINER_NAME, + templates_dir=constants.DEFAULT_TEMPLATES_PATH): + super(UpdatePlanFromDirAction, self).__init__() + self.container = container + self.templates_dir = templates_dir + + def run(self, context): + try: + swift = self.get_object_client(context) + # Upload template dir to tmp container + container_tmp = '%s-tmp' % self.container + with tempfile.NamedTemporaryFile() as tmp_tarball: + tarball.create_tarball(self.templates_dir, tmp_tarball.name) + tarball.tarball_extract_to_swift_container( + swift, + tmp_tarball.name, + container_tmp) + # Get all new templates: + new_templates = swift.get_object(container_tmp, + '')[1].splitlines() + old_templates = swift.get_object(self.container, + '')[1].splitlines() + # Update the old container + for new in new_templates: + # if doesn't exist, push it: + if new not in old_templates: + swift.put_object( + self.container, + new, + swift.get_object(container_tmp, new)[1]) + else: + content_new = swift.get_object(container_tmp, new) + content_old = swift.get_object(self.container, new) + if (not content_new == content_old and + constants.PLAN_ENVIRONMENT not in new): + swift.put_object( + self.container, + new, + swift.get_object(container_tmp, new)[1]) + except swiftexceptions.ClientException as err: + msg = "Error attempting an operation on container: %s" % err + return actions.Result(error=msg) + except Exception as err: + msg = "Error while updating plan: %s" % err + return actions.Result(error=msg) diff --git a/tripleo_common/constants.py b/tripleo_common/constants.py index faab98016..f45440529 100644 --- a/tripleo_common/constants.py +++ b/tripleo_common/constants.py @@ -41,6 +41,9 @@ STACK_TIMEOUT_DEFAULT = 240 #: The default name to use for a plan container DEFAULT_CONTAINER_NAME = 'overcloud' +#: The default name to use for the config files of the container +CONFIG_CONTAINER_NAME = 'overcloud-config' + #: The default key to use for updating parameters in plan environment. DEFAULT_PLAN_ENV_KEY = 'parameter_defaults' diff --git a/tripleo_common/tests/actions/test_config.py b/tripleo_common/tests/actions/test_config.py new file mode 100644 index 000000000..a0b49a159 --- /dev/null +++ b/tripleo_common/tests/actions/test_config.py @@ -0,0 +1,118 @@ +# Copyright 2016 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import mock + +from tripleo_common.actions import config +from tripleo_common.tests import base + +RESOURCES_YAML_CONTENTS = """heat_template_version: 2016-04-08 +resources: + Controller: + type: OS::Heat::ResourceGroup + NotRoleContoller: + type: OS::Dummy::DummyGroup +""" + + +class GetOvercloudConfigActionTest(base.TestCase): + + def setUp(self,): + super(GetOvercloudConfigActionTest, self).setUp() + self.plan = 'overcloud' + self.delete_after = 3600 + self.config_container = 'config-overcloud' + + # setup swift + self.template_files = ( + 'some-name.yaml', + 'some-other-name.yaml', + 'yet-some-other-name.yaml', + 'finally-another-name.yaml' + ) + self.swift = mock.MagicMock() + self.swift.get_container.return_value = ( + {'x-container-meta-usage-tripleo': 'plan'}, [ + {'name': tf} for tf in self.template_files + ] + ) + self.swift.get_object.return_value = ({}, RESOURCES_YAML_CONTENTS) + swift_patcher = mock.patch( + 'tripleo_common.actions.base.TripleOAction.get_object_client', + return_value=self.swift) + swift_patcher.start() + self.addCleanup(swift_patcher.stop) + + self.ctx = mock.MagicMock() + + @mock.patch('tripleo_common.actions.base.TripleOAction.' + 'get_orchestration_client') + @mock.patch('tripleo_common.utils.config.Config.download_config') + @mock.patch('tripleo_common.utils.tarball.create_tarball') + def test_run(self, mock_create_tarball, + mock_config, + mock_orchestration_client): + heat = mock.MagicMock() + heat.stacks.get.return_value = mock.MagicMock( + stack_name='stack', id='stack_id') + mock_orchestration_client.return_value = heat + mock_config.return_value = '/tmp/fake-path' + + action = config.GetOvercloudConfig(self.plan, '/tmp', + self.config_container) + action.run(self.ctx) + + self.swift.put_object.assert_called_once() + mock_create_tarball.assert_called_once() + + +class DownloadConfigActionTest(base.TestCase): + + def setUp(self,): + super(DownloadConfigActionTest, self).setUp() + self.plan = 'overcloud' + self.delete_after = 3600 + self.config_container = 'config-overcloud' + + # setup swift + self.template_files = ( + 'some-name.yaml', + 'some-other-name.yaml', + 'yet-some-other-name.yaml', + 'finally-another-name.yaml' + ) + self.swift = mock.MagicMock() + self.swift.get_container.return_value = ( + {'x-container-meta-usage-tripleo': 'plan'}, [ + {'name': tf} for tf in self.template_files + ] + ) + self.swift.get_object.return_value = ({}, RESOURCES_YAML_CONTENTS) + swift_patcher = mock.patch( + 'tripleo_common.actions.base.TripleOAction.get_object_client', + return_value=self.swift) + swift_patcher.start() + self.addCleanup(swift_patcher.stop) + + self.ctx = mock.MagicMock() + + @mock.patch('tripleo_common.utils.swift.download_container') + @mock.patch('tempfile.mkdtemp') + def test_run(self, mock_mkdtemp, + mock_swiftutils): + action = config.DownloadConfigAction(self.config_container) + action.run(self.ctx) + mock_swiftutils.assert_called_once_with(self.swift, + self.config_container, + mock_mkdtemp()) diff --git a/tripleo_common/tests/actions/test_package_update.py b/tripleo_common/tests/actions/test_package_update.py index b6bb16ad0..69f63f044 100644 --- a/tripleo_common/tests/actions/test_package_update.py +++ b/tripleo_common/tests/actions/test_package_update.py @@ -15,41 +15,9 @@ import mock from tripleo_common.actions import package_update -from tripleo_common import constants from tripleo_common.tests import base -class ClearBreakpointsActionTest(base.TestCase): - - def setUp(self,): - super(ClearBreakpointsActionTest, self).setUp() - self.stack_id = 'stack_id' - self.refs = 'refs' - - @mock.patch('tripleo_common.actions.package_update.PackageUpdateManager') - @mock.patch('tripleo_common.actions.base.TripleOAction.' - 'get_orchestration_client') - @mock.patch('tripleo_common.actions.base.TripleOAction.' - 'get_compute_client') - def test_run(self, mock_compute_client, - mock_orchestration_client, - mock_update_manager): - mock_ctx = mock.MagicMock() - action = package_update.ClearBreakpointsAction(self.stack_id, - self.refs) - result = action.run(mock_ctx) - self.assertIsNone(result) - mock_compute_client.assert_called_once() - mock_orchestration_client.assert_called_once() - mock_update_manager.assert_called_once_with( - mock_orchestration_client(), - mock_compute_client(), - self.stack_id, - stack_fields={}) - mock_update_manager().clear_breakpoints.assert_called_once_with( - self.refs) - - class UpdateStackActionTest(base.TestCase): def setUp(self,): @@ -63,10 +31,14 @@ class UpdateStackActionTest(base.TestCase): 'get_orchestration_client') @mock.patch('tripleo_common.actions.base.TripleOAction.' 'get_compute_client') - @mock.patch('tripleo_common.actions.package_update.time') @mock.patch('heatclient.common.template_utils.get_template_contents') - def test_run(self, mock_template_contents, - mock_time, + @mock.patch('tripleo_common.utils.plan.get_env') + @mock.patch('tripleo_common.utils.plan.update_in_env') + @mock.patch('heatclient.common.template_utils.deep_update') + def test_run(self, mock_deepupdate, + mock_updateinenv, + mock_getenv, + mock_template_contents, mock_compute_client, mock_orchestration_client, mock_object_client, @@ -82,59 +54,68 @@ class UpdateStackActionTest(base.TestCase): 'heat_template_version': '2016-04-30' }) mock_swift = mock.MagicMock() - mock_env = """environments: -- path: environments/test.yaml -name: container -parameter_defaults: - random_data: a_value -temp_environment: temp_environment -template: template -""" - mock_swift.get_object.return_value = ({}, mock_env) + env = { + 'parameters': { + 'ControllerCount': 1, + 'ComputeCount': 1, + 'ObjectStorageCount': 0, + 'BlockStorageCount': 0, + 'CephStorageCount': 0, + }, + 'stack_name': 'overcloud', + 'stack_status': "CREATE_COMPLETE", + 'outputs': [ + {'output_key': 'RoleConfig', + 'output_value': { + 'foo_config': 'foo'}}, + {'output_key': 'RoleData', + 'output_value': { + 'FakeCompute': { + 'config_settings': {'nova::compute::fake' + 'libvirt_virt_type': 'qemu'}, + 'global_config_settings': {}, + 'logging_groups': ['root', 'neutron', 'nova'], + 'logging_sources': [{'path': '/var/log/fake.log', + 'type': 'tail'}], + 'monitoring_subscriptions': ['nova-compute'], + 'service_config_settings': None, + 'service_metadata_settings': None, + 'service_names': ['nova_compute', 'fake_service'], + 'step_config': ['include ::tripleo::profile::fake', + 'include ::timezone'], + 'upgrade_batch_tasks': [], + 'upgrade_tasks': [{'name': 'Stop fake service', + 'service': 'name=fo state=stopped', + 'tags': 'step1', + 'when': 'existingcondition'}, + {'name': 'Stop nova-compute', + 'service': 'name=nova-compute ' + 'state=stopped', + 'tags': 'step1', + 'when': ['existing', 'list']}] + }}}]} + + update_env = {'resource_registry': + {'OS::TripleO::DeploymentSteps': 'OS::Heat::None'}} + + mock_getenv.return_value = env + fake_registry = {'parameter_defaults': [ + {'DockerKeystoneImage': '192.168.24.1:8787/' + 'keystone-docker:latest', + 'DockerHeatApiImage:': '192.168.24.1:8787/' + 'heat-api-docker:latest'}]} + update_env.update(fake_registry) + mock_swift.get_object.return_value = ({}, env) mock_object_client.return_value = mock_swift - # freeze time at datetime.datetime(2016, 9, 8, 16, 24, 24) - mock_time.time.return_value = 1473366264 - - mock_templates_run.return_value = { - 'StackAction': 'UPDATE', - 'DeployIdentifier': 1473366264, - 'UpdateIdentifier': 1473366264 - } - - action = package_update.UpdateStackAction(self.timeout, + action = package_update.UpdateStackAction(self.timeout, fake_registry, container=self.container) action.run(mock_ctx) - - # verify parameters are as expected - updated_mock_env = """environments: -- path: environments/test.yaml -name: container -parameter_defaults: - DeployIdentifier: 1473366264 - StackAction: UPDATE - UpdateIdentifier: 1473366264 - random_data: a_value -temp_environment: temp_environment -template: template -""" - mock_swift.put_object.assert_called_once_with( - self.container, constants.PLAN_ENVIRONMENT, updated_mock_env + mock_updateinenv.assert_called_once_with( + mock_swift, env, 'parameter_defaults', + fake_registry['parameter_defaults'] ) - heat.stacks.update.assert_called_once_with( - 'stack_id', - StackAction='UPDATE', - DeployIdentifier=1473366264, - UpdateIdentifier=1473366264, - existing='true', - timeout_mins=1, - environment={ - 'resource_registry': { - 'resources': { - '*': { - '*': {'UpdateDeployment': {'hooks': 'pre-update'}} - } - } - } - }) + mock_deepupdate.assert_called_once_with(env, update_env) + + heat.stacks.update.assert_called_once_with('stack_id') diff --git a/tripleo_common/tests/fake_config/__init__.py b/tripleo_common/tests/fake_config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tripleo_common/tests/fake_config/fakes.py b/tripleo_common/tests/fake_config/fakes.py new file mode 100644 index 000000000..c6b3a751d --- /dev/null +++ b/tripleo_common/tests/fake_config/fakes.py @@ -0,0 +1,91 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import mock + + +FAKE_STACK = { + 'parameters': { + 'ControllerCount': 1, + 'ComputeCount': 1, + 'ObjectStorageCount': 0, + 'BlockStorageCount': 0, + 'CephStorageCount': 0, + }, + 'stack_name': 'overcloud', + 'stack_status': "CREATE_COMPLETE", + 'outputs': [ + {'output_key': 'RoleConfig', + 'output_value': { + 'foo_config': 'foo'}}, + {'output_key': 'RoleData', + 'output_value': { + 'FakeCompute': { + 'config_settings': {'nova::compute::libvirt::services::' + 'libvirt_virt_type': 'qemu'}, + 'global_config_settings': {}, + 'logging_groups': ['root', 'neutron', 'nova'], + 'logging_sources': [{'path': '/var/log/nova/nova-compute.log', + 'type': 'tail'}], + 'monitoring_subscriptions': ['overcloud-nova-compute'], + 'service_config_settings': {'horizon': {'neutron::' + 'plugins': ['ovs']} + }, + 'service_metadata_settings': None, + 'service_names': ['nova_compute', 'fake_service'], + 'step_config': ['include ::tripleo::profile::base::sshd', + 'include ::timezone'], + 'upgrade_batch_tasks': [], + 'upgrade_tasks': [{'name': 'Stop fake service', + 'service': 'name=fake state=stopped', + 'tags': 'step1', + 'when': 'existingcondition'}, + {'name': 'Stop nova-compute service', + 'service': 'name=openstack-nova-compute ' + 'state=stopped', + 'tags': 'step1', + 'when': ['existing', 'list']}] + }, + 'FakeController': { + 'config_settings': {'tripleo::haproxy::user': 'admin'}, + 'global_config_settings': {}, + 'logging_groups': ['root', 'keystone', 'neutron'], + 'logging_sources': [{'path': '/var/log/keystone/keystone.log', + 'type': 'tail'}], + 'monitoring_subscriptions': ['overcloud-keystone'], + 'service_config_settings': {'horizon': {'neutron::' + 'plugins': ['ovs']} + }, + 'service_metadata_settings': None, + 'service_names': ['pacemaker', 'fake_service'], + 'step_config': ['include ::tripleo::profile::base::sshd', + 'include ::timezone'], + 'upgrade_batch_tasks': [], + 'upgrade_tasks': [{'name': 'Stop fake service', + 'service': 'name=fake state=stopped', + 'tags': 'step1'}]}}}]} + + +def create_to_dict_mock(**kwargs): + mock_with_to_dict = mock.Mock() + mock_with_to_dict.configure_mock(**kwargs) + mock_with_to_dict.to_dict.return_value = kwargs + return mock_with_to_dict + + +def create_tht_stack(**kwargs): + stack = FAKE_STACK.copy() + stack.update(kwargs) + return create_to_dict_mock(**stack) diff --git a/tripleo_common/tests/utils/test_config.py b/tripleo_common/tests/utils/test_config.py new file mode 100644 index 000000000..169021c1f --- /dev/null +++ b/tripleo_common/tests/utils/test_config.py @@ -0,0 +1,143 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures +import mock +import os + +from mock import call +from mock import patch + +from tripleo_common.tests import base +from tripleo_common.tests.fake_config import fakes +from tripleo_common.utils import config as ooo_config + + +class TestConfig(base.TestCase): + + def setUp(self): + super(TestConfig, self).setUp() + + @patch.object(ooo_config.Config, '_mkdir') + @patch.object(ooo_config.Config, '_open_file') + @mock.patch('tempfile.mkdtemp', autospec=True) + def test_overcloud_config_generate_config(self, + mock_tmpdir, + mock_open, + mock_mkdir): + config_type_list = ['config_settings', 'global_config_settings', + 'logging_sources', 'monitoring_subscriptions', + 'service_config_settings', + 'service_metadata_settings', + 'service_names', + 'upgrade_batch_tasks', 'upgrade_tasks'] + fake_role = [role for role in + fakes.FAKE_STACK['outputs'][1]['output_value']] + + heat = mock.MagicMock() + heat.stacks.get.return_value = fakes.create_tht_stack() + self.config = ooo_config.Config(heat) + mock_tmpdir.return_value = "/tmp/tht" + self.config.download_config('overcloud', '/tmp', config_type_list) + + expected_mkdir_calls = [call('/tmp/tht/%s' % r) for r in fake_role] + mock_mkdir.assert_has_calls(expected_mkdir_calls, any_order=True) + expected_calls = [] + for config in config_type_list: + for role in fake_role: + if config == 'step_config': + expected_calls += [call('/tmp/tht/%s/%s.pp' % + (role, config))] + else: + expected_calls += [call('/tmp/tht/%s/%s.yaml' % + (role, config))] + mock_open.assert_has_calls(expected_calls, any_order=True) + + @patch.object(ooo_config.Config, '_mkdir') + @patch.object(ooo_config.Config, '_open_file') + @mock.patch('tempfile.mkdtemp', autospec=True) + def test_overcloud_config_one_config_type(self, + mock_tmpdir, + mock_open, + mock_mkdir): + + expected_config_type = 'config_settings' + fake_role = [role for role in + fakes.FAKE_STACK['outputs'][1]['output_value']] + + heat = mock.MagicMock() + heat.stacks.get.return_value = fakes.create_tht_stack() + self.config = ooo_config.Config(heat) + mock_tmpdir.return_value = "/tmp/tht" + self.config.download_config('overcloud', '/tmp', ['config_settings']) + expected_mkdir_calls = [call('/tmp/tht/%s' % r) for r in fake_role] + expected_calls = [call('/tmp/tht/%s/%s.yaml' + % (r, expected_config_type)) + for r in fake_role] + mock_mkdir.assert_has_calls(expected_mkdir_calls, any_order=True) + mock_open.assert_has_calls(expected_calls, any_order=True) + + @mock.patch('os.mkdir') + @mock.patch('six.moves.builtins.open') + @mock.patch('tempfile.mkdtemp', autospec=True) + def test_overcloud_config_wrong_config_type(self, mock_tmpdir, + mock_open, mock_mkdir): + args = {'name': 'overcloud', 'config_dir': '/tmp', + 'config_type': ['bad_config']} + heat = mock.MagicMock() + heat.stacks.get.return_value = fakes.create_tht_stack() + self.config = ooo_config.Config(heat) + mock_tmpdir.return_value = "/tmp/tht" + self.assertRaises( + KeyError, + self.config.download_config, *args) + + @mock.patch('tripleo_common.utils.config.Config.get_role_data', + autospec=True) + def test_overcloud_config_upgrade_tasks(self, mock_get_role_data): + + heat = mock.MagicMock() + heat.stacks.get.return_value = fakes.create_tht_stack() + self.config = ooo_config.Config(heat) + self.tmp_dir = self.useFixture(fixtures.TempDir()).path + fake_role = [role for role in + fakes.FAKE_STACK['outputs'][1]['output_value']] + expected_tasks = {'FakeController': [{'name': 'Stop fake service', + 'service': 'name=fake ' + 'state=stopped', + 'tags': 'step1', + 'when': 'step|int == 1'}], + 'FakeCompute': [{'name': 'Stop fake service', + 'service': + 'name=fake state=stopped', + 'tags': 'step1', + 'when': ['existingcondition', + 'step|int == 1']}, + {'name': 'Stop nova-' + 'compute service', + 'service': + 'name=openstack-nova-' + 'compute state=stopped', + 'tags': 'step1', + 'when': ['existing', + 'list', 'step|int == 1']}]} + mock_get_role_data.return_value = fake_role + + for role in fake_role: + filedir = os.path.join(self.tmp_dir, role) + os.makedirs(filedir) + filepath = os.path.join(filedir, "upgrade_tasks_playbook.yaml") + playbook_tasks = self.config._write_playbook_get_tasks( + fakes.FAKE_STACK['outputs'][1]['output_value'][role] + ['upgrade_tasks'], role, filepath) + self.assertTrue(os.path.isfile(filepath)) + self.assertEqual(expected_tasks[role], playbook_tasks) diff --git a/tripleo_common/utils/config.py b/tripleo_common/utils/config.py new file mode 100644 index 000000000..f6ad581be --- /dev/null +++ b/tripleo_common/utils/config.py @@ -0,0 +1,142 @@ +# Copyright 2016 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import logging +import os +import re +import six +import tempfile +import yaml + + +class Config(object): + + def __init__(self, orchestration_client): + self.log = logging.getLogger(__name__ + ".Config") + self.client = orchestration_client + + def get_role_data(self, stack): + role_data = {} + for output in stack.to_dict().get('outputs', {}): + if output['output_key'] == 'RoleData': + for role in output['output_value']: + role_data[role] = output['output_value'][role] + return role_data + + def get_role_config(self, stack): + role_data = {} + for output in stack.to_dict().get('outputs', {}): + if output['output_key'] == 'RoleConfig': + for role in output['output_value']: + role_data[role] = output['output_value'][role] + return role_data + + @staticmethod + def _open_file(path): + return os.fdopen(os.open(path, + os.O_WRONLY | os.O_CREAT, 0o600), + 'w') + + def _step_tags_to_when(self, sorted_tasks): + for task in sorted_tasks: + tag = task.get('tags', '') + match = re.search('step([0-9]+)', tag) + if match: + step = match.group(1) + whenexpr = task.get('when', None) + if whenexpr: + # Handle when: foo and a list of when conditionals + if not isinstance(whenexpr, list): + whenexpr = [whenexpr] + for w in whenexpr: + when_exists = re.search('step|int == [0-9]', w) + if when_exists: + break + if when_exists: + # Skip to the next task, + # there is an existing 'step|int == N' + continue + whenexpr.append("step|int == %s" % step) + task['when'] = whenexpr + else: + task.update({"when": "step|int == %s" % step}) + + def _write_playbook_get_tasks(self, tasks, role, filepath): + playbook = [] + sorted_tasks = sorted(tasks, key=lambda x: x.get('tags', None)) + self._step_tags_to_when(sorted_tasks) + playbook.append({'name': '%s playbook' % role, + 'hosts': role, + 'tasks': sorted_tasks}) + with self._open_file(filepath) as conf_file: + yaml.safe_dump(playbook, conf_file, default_flow_style=False) + return sorted_tasks + + def _mkdir(self, dirname): + if not os.path.exists(dirname): + try: + os.mkdir(dirname, 0o700) + except OSError as e: + message = 'Failed to create: %s, error: %s' % (dirname, + str(e)) + raise OSError(message) + + def download_config(self, name, config_dir, config_type=None): + # Get the stack object + stack = self.client.stacks.get(name) + # Create config directory + self._mkdir(config_dir) + tmp_path = tempfile.mkdtemp(prefix='tripleo-', + suffix='-config', + dir=config_dir) + self.log.info("Generating configuration under the directory: " + "%s" % tmp_path) + # Get role data: + role_data = self.get_role_data(stack) + for role_name, role in six.iteritems(role_data): + role_path = os.path.join(tmp_path, role_name) + self._mkdir(role_path) + for config in config_type or role.keys(): + if config == 'step_config': + filepath = os.path.join(role_path, 'step_config.pp') + with self._open_file(filepath) as step_config: + step_config.write('\n'.join(step for step in + role[config] + if step is not None)) + else: + if 'upgrade_tasks' in config: + filepath = os.path.join(role_path, '%s_playbook.yaml' % + config) + data = self._write_playbook_get_tasks( + role[config], role_name, filepath) + else: + try: + data = role[config] + except KeyError as e: + message = 'Invalid key: %s, error: %s' % (config, + str(e)) + raise KeyError(message) + filepath = os.path.join(role_path, '%s.yaml' % config) + with self._open_file(filepath) as conf_file: + yaml.safe_dump(data, + conf_file, + default_flow_style=False) + role_config = self.get_role_config(stack) + for config_name, config in six.iteritems(role_config): + conf_path = os.path.join(tmp_path, config_name + ".yaml") + with self._open_file(conf_path) as conf_file: + conf_file.write(config) + self.log.info("The TripleO configuration has been successfully " + "generated into: %s" % tmp_path) + return tmp_path diff --git a/workbooks/package_update.yaml b/workbooks/package_update.yaml index 8f536e3bb..5fb5ea5eb 100644 --- a/workbooks/package_update.yaml +++ b/workbooks/package_update.yaml @@ -11,15 +11,34 @@ workflows: input: - container + - container_registry - timeout: 240 - queue_name: tripleo + - skip_deploy_identifier: False + - config_dir: '/tmp/' tags: - tripleo-common-managed tasks: + update_plan: + action: tripleo.plan.update_from_dir + input: + container: <% $.container %> + on-success: update + on-error: set_update_failed + update: - action: tripleo.package_update.update_stack container=<% $.container %> timeout=<% $.timeout %> + action: tripleo.package_update.update_stack container=<% $.container %> timeout=<% $.timeout %> container_registry=<% $.container_registry %> + input: + timeout: <% $.timeout %> + container: <% $.container %> + container_registry: <% $.container_registry %> + on-success: get_config + on-error: set_update_failed + + get_config: + action: tripleo.config.get_overcloud_config on-success: send_message on-error: set_update_failed @@ -41,42 +60,75 @@ workflows: message: <% $.get('message', '') %> execution: <% execution() %> on-success: - - fail: <% $.get('status') = "FAILED" %> + - fail: <% $.get('get_config') = "FAILED" %> - # Clear an update breakpoint - clear_breakpoints: - description: Clear any pending breakpoints and continue with update + update_nodes: + description: Take a container and perform an update nodes by nodes input: - - stack_id - - refs + - node_user: heat-admin + - nodes + - playbook + - inventory_file - queue_name: tripleo tags: - tripleo-common-managed tasks: - clear: - action: tripleo.package_update.clear_breakpoints stack_id=<% $.stack_id %> refs=<% $.refs %> - on-success: send_message - on-error: set_clear_breakpoints_failed + download_config: + action: tripleo.config.download_config + on-success: get_private_key + publish: + tmp_path: <% task(download_config).result %> + on-error: node_update_failed - set_clear_breakpoints_failed: - on-success: send_message + get_private_key: + action: tripleo.validations.get_privkey + publish: + private_key: <% task(get_private_key).result %> + on-success: node_update + + node_update: + action: tripleo.ansible-playbook + input: + inventory: <% $.inventory_file %> + playbook: <% $.tmp_path %>/<% $.playbook %> + remote_user: <% $.node_user %> + become: true + become_user: root + verbosity: 0 + ssh_private_key: <% $.private_key %> + ssh_extra_args: '-o StrictHostKeyChecking=no' + limit_hosts: <% $.nodes %> + on-success: node_update_passed + on-error: node_update_failed + publish: + output: <% task(node_update).result %> + + node_update_passed: + on-success: notify_zaqar + publish: + status: SUCCESS + message: Updated nodes - <% $.nodes %> + + node_update_failed: + on-success: notify_zaqar publish: status: FAILED - message: <% task(clear).result %> + message: Failed to update nodes - <% $.nodes %>, please see the logs. - send_message: + notify_zaqar: action: zaqar.queue_post + retry: count=5 delay=1 input: queue_name: <% $.queue_name %> messages: body: - type: tripleo.package_update.v1.clear_breakpoints + type: tripleo.package_update.v1.update_nodes payload: - status: <% $.get('status', 'SUCCESS') %> - message: <% $.get('message', '') %> + status: <% $.status %> + message: <% task(node_update).result %> execution: <% execution() %> on-success: - fail: <% $.get('status') = "FAILED" %>