Execute minor update via ansible on mistral

This review implement the minor update workflow
which does:
- a noop heat stack deployment to set and refresh
  the heat config output.
- download the heat config and put them in a ansible
  playbooks
- run the playbooks with the mistral action

It adds actions for the config download and update
deployment
Adds the config download as a library for being call
either, by the client or mistral

Closes-Bug: #1715557
Change-Id: I199b35f865c0e68d28c5ddd82e5b8fe61abb5f33

setup.cfg

@@ -73,6 +73,8 @@ mistral.actions =
     tripleo.baremetal.validate_nodes = tripleo_common.actions.baremetal:ValidateNodes
     tripleo.baremetal.get_candidate_nodes = tripleo_common.actions.baremetal:GetCandidateNodes
     tripleo.baremetal.probe_node = tripleo_common.actions.baremetal:ProbeNode
+    tripleo.config.download_config = tripleo_common.actions.config:DownloadConfigAction
+    tripleo.config.get_overcloud_config = tripleo_common.actions.config:GetOvercloudConfig
     tripleo.deployment.config = tripleo_common.actions.deployment:OrchestrationDeployAction
     tripleo.deployment.deploy = tripleo_common.actions.deployment:DeployStackAction
     tripleo.deployment.overcloudrc = tripleo_common.actions.deployment:OvercloudRcAction
@@ -86,7 +88,6 @@ mistral.actions =
     tripleo.git.clone = tripleo_common.actions.vcs:GitCloneAction
     tripleo.heat_capabilities.get = tripleo_common.actions.heat_capabilities:GetCapabilitiesAction
     tripleo.heat_capabilities.update = tripleo_common.actions.heat_capabilities:UpdateCapabilitiesAction
-    tripleo.package_update.clear_breakpoints = tripleo_common.actions.package_update:ClearBreakpointsAction
     tripleo.package_update.update_stack = tripleo_common.actions.package_update:UpdateStackAction
     tripleo.parameters.get = tripleo_common.actions.parameters:GetParametersAction
     tripleo.parameters.get_flatten = tripleo_common.actions.parameters:GetFlattenedParametersAction
@@ -103,6 +104,7 @@ mistral.actions =
     tripleo.plan.delete = tripleo_common.actions.plan:DeletePlanAction
     tripleo.plan.list = tripleo_common.actions.plan:ListPlansAction
     tripleo.plan.export = tripleo_common.actions.plan:ExportPlanAction
+    tripleo.plan.update_from_dir = tripleo_common.actions.plan:UpdatePlanFromDirAction
     tripleo.logging_to_swift.format_messages = tripleo_common.actions.logging_to_swift:FormatMessagesAction
     tripleo.logging_to_swift.publish_ui_log_to_swift = tripleo_common.actions.logging_to_swift:PublishUILogToSwiftAction
     tripleo.logging_to_swift.prepare_log_download = tripleo_common.actions.logging_to_swift:PrepareLogDownloadAction

tripleo_common/actions/config.py

@@ -0,0 +1,79 @@
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import logging
+import os
+import shutil
+import tempfile
+
+from tripleo_common.actions import templates
+from tripleo_common import constants
+from tripleo_common.utils import config as ooo_config
+from tripleo_common.utils import swift as swiftutils
+from tripleo_common.utils import tarball
+
+LOG = logging.getLogger(__name__)
+
+
+class GetOvercloudConfig(templates.ProcessTemplatesAction):
+    """Get the Overcloud Config from the Heat outputs
+
+    This action gets the Overcloud config from the Heat outputs and
+    write it to the disk to be call with Ansible.
+
+    :param container: name of the Swift container / plan name
+     config_dir: directory where the config should be written
+    """
+
+    def __init__(self, container=constants.DEFAULT_CONTAINER_NAME,
+                 config_dir=tempfile.gettempdir(),
+                 container_config=constants.CONFIG_CONTAINER_NAME):
+        super(GetOvercloudConfig, self).__init__(container)
+        self.container = container
+        self.config_dir = config_dir
+        self.container_config = container_config
+
+    def run(self, context):
+        heat = self.get_orchestration_client(context)
+        config = ooo_config.Config(heat)
+        config_path = config.download_config(self.container, self.config_dir)
+
+        with tempfile.NamedTemporaryFile() as tmp_tarball:
+            tarball.create_tarball(config_path, tmp_tarball.name)
+            tarball.tarball_extract_to_swift_container(
+                self.get_object_client(context),
+                tmp_tarball.name,
+                self.container_config)
+        if os.path.exists(config_path):
+            shutil.rmtree(config_path)
+
+
+class DownloadConfigAction(templates.ProcessTemplatesAction):
+    """Download the container config from swift
+
+    This action downloads a container which contain the heat config output
+
+    :param container: name of the Swift container / plan name
+    """
+
+    def __init__(self, container_config=constants.CONFIG_CONTAINER_NAME):
+        super(DownloadConfigAction, self).__init__(container_config)
+        self.container_config = container_config
+
+    def run(self, context):
+        swift = self.get_object_client(context)
+        tmp_dir = tempfile.mkdtemp(prefix='tripleo-',
+                                   suffix='-config')
+        swiftutils.download_container(swift, self.container_config, tmp_dir)
+        return tmp_dir

tripleo_common/actions/package_update.py

@@ -13,41 +13,26 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 import logging
-import time
 
 from heatclient.common import template_utils
 from heatclient import exc as heat_exc
 from mistral_lib import actions
 from swiftclient import exceptions as swiftexceptions
 
-from tripleo_common.actions import base
 from tripleo_common.actions import templates
 from tripleo_common import constants
-from tripleo_common.update import PackageUpdateManager
 from tripleo_common.utils import plan as plan_utils
 
 LOG = logging.getLogger(__name__)
 
 
-class ClearBreakpointsAction(base.TripleOAction):
-    def __init__(self, stack_id, refs):
-        super(ClearBreakpointsAction, self).__init__()
-        self.stack_id = stack_id
-        self.refs = refs
-
-    def run(self, context):
-        heat = self.get_orchestration_client(context)
-        nova = self.get_compute_client(context)
-        update_manager = PackageUpdateManager(
-            heat, nova, self.stack_id, stack_fields={})
-        update_manager.clear_breakpoints(self.refs)
-
-
 class UpdateStackAction(templates.ProcessTemplatesAction):
 
-    def __init__(self, timeout, container=constants.DEFAULT_CONTAINER_NAME):
+    def __init__(self, timeout, container_registry,
+                 container=constants.DEFAULT_CONTAINER_NAME):
         super(UpdateStackAction, self).__init__(container)
         self.timeout_mins = timeout
+        self.container_registry = container_registry
 
     def run(self, context):
         # get the stack. Error if doesn't exist
@@ -59,12 +44,6 @@ class UpdateStackAction(templates.ProcessTemplatesAction):
             LOG.exception(msg)
             return actions.Result(error=msg)
 
-        parameters = dict()
-        timestamp = int(time.time())
-        parameters['DeployIdentifier'] = timestamp
-        parameters['UpdateIdentifier'] = timestamp
-        parameters['StackAction'] = 'UPDATE'
-
         swift = self.get_object_client(context)
 
         try:
@@ -75,14 +54,30 @@ class UpdateStackAction(templates.ProcessTemplatesAction):
             LOG.exception(err_msg)
             return actions.Result(error=err_msg)
 
-        try:
+        update_env = {}
-            plan_utils.update_in_env(swift, env, 'parameter_defaults',
+        if self.container_registry is not None:
-                                     parameters)
+            update_env.update(self.container_registry)
-        except swiftexceptions.ClientException as err:
+
-            err_msg = ("Error updating environment for plan %s: %s" % (
+        noop_env = {
-                self.container, err))
+            'resource_registry': {
-            LOG.exception(err_msg)
+                'OS::TripleO::DeploymentSteps': 'OS::Heat::None',
-            return actions.Result(error=err_msg)
+            },
+        }
+
+        for output in stack.to_dict().get('outputs', {}):
+            if output['output_key'] == 'RoleData':
+                for role in output['output_value']:
+                    role_env = {
+                        "OS::TripleO::Tasks::%sPreConfig" % role:
+                        'OS::Heat::None',
+                        "OS::TripleO::Tasks::%sPostConfig" % role:
+                        'OS::Heat::None',
+                    }
+                    noop_env['resource_registry'].update(role_env)
+        update_env.update(noop_env)
+        template_utils.deep_update(env, update_env)
+        plan_utils.update_in_env(swift, env, 'parameter_defaults',
+                                 self.container_registry['parameter_defaults'])
 
         # process all plan files and create or update a stack
         processed_data = super(UpdateStackAction, self).run(context)
@@ -94,24 +89,6 @@ class UpdateStackAction(templates.ProcessTemplatesAction):
 
         stack_args = processed_data.copy()
 
-        env = stack_args.get('environment', {})
-        template_utils.deep_update(env, {
-            'resource_registry': {
-                'resources': {
-                    '*': {
-                        '*': {
-                            constants.UPDATE_RESOURCE_NAME: {
-                                'hooks': 'pre-update'}
-                        }
-                    }
-                }
-            }
-        })
-        stack_args['environment'] = env
-
-        stack_args['timeout_mins'] = self.timeout_mins
-        stack_args['existing'] = 'true'
-
         LOG.info("Performing Heat stack update")
         LOG.info('updating stack: %s', stack.stack_name)
         return heat.stacks.update(stack.id, **stack_args)

tripleo_common/actions/plan.py

@@ -30,6 +30,7 @@ from tripleo_common import constants
 from tripleo_common import exception
 from tripleo_common.utils import plan as plan_utils
 from tripleo_common.utils import swift as swiftutils
+from tripleo_common.utils import tarball
 from tripleo_common.utils.validations import pattern_validator
 
 
@@ -243,3 +244,61 @@ class ExportPlanAction(base.TripleOAction):
             return actions.Result(error=msg)
         finally:
             shutil.rmtree(tmp_dir)
+
+
+class UpdatePlanFromDirAction(base.TripleOAction):
+    """Updates a plan and associated files
+
+    Updates a plan by comparing the current files with the new ones
+    provided:
+        Updates only new files from the plan
+        Add new files from the plan
+
+    :param container: name of the Swift container / plan name
+    """
+
+    def __init__(self, container=constants.DEFAULT_CONTAINER_NAME,
+                 templates_dir=constants.DEFAULT_TEMPLATES_PATH):
+        super(UpdatePlanFromDirAction, self).__init__()
+        self.container = container
+        self.templates_dir = templates_dir
+
+    def run(self, context):
+        try:
+            swift = self.get_object_client(context)
+            # Upload template dir to tmp container
+            container_tmp = '%s-tmp' % self.container
+            with tempfile.NamedTemporaryFile() as tmp_tarball:
+                tarball.create_tarball(self.templates_dir, tmp_tarball.name)
+                tarball.tarball_extract_to_swift_container(
+                    swift,
+                    tmp_tarball.name,
+                    container_tmp)
+            # Get all new templates:
+            new_templates = swift.get_object(container_tmp,
+                                             '')[1].splitlines()
+            old_templates = swift.get_object(self.container,
+                                             '')[1].splitlines()
+            # Update the old container
+            for new in new_templates:
+                # if doesn't exist, push it:
+                if new not in old_templates:
+                    swift.put_object(
+                        self.container,
+                        new,
+                        swift.get_object(container_tmp, new)[1])
+                else:
+                    content_new = swift.get_object(container_tmp, new)
+                    content_old = swift.get_object(self.container, new)
+                    if (not content_new == content_old and
+                       constants.PLAN_ENVIRONMENT not in new):
+                        swift.put_object(
+                            self.container,
+                            new,
+                            swift.get_object(container_tmp, new)[1])
+        except swiftexceptions.ClientException as err:
+            msg = "Error attempting an operation on container: %s" % err
+            return actions.Result(error=msg)
+        except Exception as err:
+            msg = "Error while updating plan: %s" % err
+            return actions.Result(error=msg)

tripleo_common/constants.py

@@ -41,6 +41,9 @@ STACK_TIMEOUT_DEFAULT = 240
 #: The default name to use for a plan container
 DEFAULT_CONTAINER_NAME = 'overcloud'
 
+#: The default name to use for the config files of the container
+CONFIG_CONTAINER_NAME = 'overcloud-config'
+
 #: The default key to use for updating parameters in plan environment.
 DEFAULT_PLAN_ENV_KEY = 'parameter_defaults'
 

tripleo_common/tests/actions/test_config.py

@@ -0,0 +1,118 @@
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import mock
+
+from tripleo_common.actions import config
+from tripleo_common.tests import base
+
+RESOURCES_YAML_CONTENTS = """heat_template_version: 2016-04-08
+resources:
+  Controller:
+    type: OS::Heat::ResourceGroup
+  NotRoleContoller:
+    type: OS::Dummy::DummyGroup
+"""
+
+
+class GetOvercloudConfigActionTest(base.TestCase):
+
+    def setUp(self,):
+        super(GetOvercloudConfigActionTest, self).setUp()
+        self.plan = 'overcloud'
+        self.delete_after = 3600
+        self.config_container = 'config-overcloud'
+
+        # setup swift
+        self.template_files = (
+            'some-name.yaml',
+            'some-other-name.yaml',
+            'yet-some-other-name.yaml',
+            'finally-another-name.yaml'
+        )
+        self.swift = mock.MagicMock()
+        self.swift.get_container.return_value = (
+            {'x-container-meta-usage-tripleo': 'plan'}, [
+                {'name': tf} for tf in self.template_files
+            ]
+        )
+        self.swift.get_object.return_value = ({}, RESOURCES_YAML_CONTENTS)
+        swift_patcher = mock.patch(
+            'tripleo_common.actions.base.TripleOAction.get_object_client',
+            return_value=self.swift)
+        swift_patcher.start()
+        self.addCleanup(swift_patcher.stop)
+
+        self.ctx = mock.MagicMock()
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_orchestration_client')
+    @mock.patch('tripleo_common.utils.config.Config.download_config')
+    @mock.patch('tripleo_common.utils.tarball.create_tarball')
+    def test_run(self, mock_create_tarball,
+                 mock_config,
+                 mock_orchestration_client):
+        heat = mock.MagicMock()
+        heat.stacks.get.return_value = mock.MagicMock(
+            stack_name='stack', id='stack_id')
+        mock_orchestration_client.return_value = heat
+        mock_config.return_value = '/tmp/fake-path'
+
+        action = config.GetOvercloudConfig(self.plan, '/tmp',
+                                           self.config_container)
+        action.run(self.ctx)
+
+        self.swift.put_object.assert_called_once()
+        mock_create_tarball.assert_called_once()
+
+
+class DownloadConfigActionTest(base.TestCase):
+
+    def setUp(self,):
+        super(DownloadConfigActionTest, self).setUp()
+        self.plan = 'overcloud'
+        self.delete_after = 3600
+        self.config_container = 'config-overcloud'
+
+        # setup swift
+        self.template_files = (
+            'some-name.yaml',
+            'some-other-name.yaml',
+            'yet-some-other-name.yaml',
+            'finally-another-name.yaml'
+        )
+        self.swift = mock.MagicMock()
+        self.swift.get_container.return_value = (
+            {'x-container-meta-usage-tripleo': 'plan'}, [
+                {'name': tf} for tf in self.template_files
+            ]
+        )
+        self.swift.get_object.return_value = ({}, RESOURCES_YAML_CONTENTS)
+        swift_patcher = mock.patch(
+            'tripleo_common.actions.base.TripleOAction.get_object_client',
+            return_value=self.swift)
+        swift_patcher.start()
+        self.addCleanup(swift_patcher.stop)
+
+        self.ctx = mock.MagicMock()
+
+    @mock.patch('tripleo_common.utils.swift.download_container')
+    @mock.patch('tempfile.mkdtemp')
+    def test_run(self, mock_mkdtemp,
+                 mock_swiftutils):
+        action = config.DownloadConfigAction(self.config_container)
+        action.run(self.ctx)
+        mock_swiftutils.assert_called_once_with(self.swift,
+                                                self.config_container,
+                                                mock_mkdtemp())

tripleo_common/tests/actions/test_package_update.py

@@ -15,41 +15,9 @@
 import mock
 
 from tripleo_common.actions import package_update
-from tripleo_common import constants
 from tripleo_common.tests import base
 
 
-class ClearBreakpointsActionTest(base.TestCase):
-
-    def setUp(self,):
-        super(ClearBreakpointsActionTest, self).setUp()
-        self.stack_id = 'stack_id'
-        self.refs = 'refs'
-
-    @mock.patch('tripleo_common.actions.package_update.PackageUpdateManager')
-    @mock.patch('tripleo_common.actions.base.TripleOAction.'
-                'get_orchestration_client')
-    @mock.patch('tripleo_common.actions.base.TripleOAction.'
-                'get_compute_client')
-    def test_run(self, mock_compute_client,
-                 mock_orchestration_client,
-                 mock_update_manager):
-        mock_ctx = mock.MagicMock()
-        action = package_update.ClearBreakpointsAction(self.stack_id,
-                                                       self.refs)
-        result = action.run(mock_ctx)
-        self.assertIsNone(result)
-        mock_compute_client.assert_called_once()
-        mock_orchestration_client.assert_called_once()
-        mock_update_manager.assert_called_once_with(
-            mock_orchestration_client(),
-            mock_compute_client(),
-            self.stack_id,
-            stack_fields={})
-        mock_update_manager().clear_breakpoints.assert_called_once_with(
-            self.refs)
-
-
 class UpdateStackActionTest(base.TestCase):
 
     def setUp(self,):
@@ -63,10 +31,14 @@ class UpdateStackActionTest(base.TestCase):
                 'get_orchestration_client')
     @mock.patch('tripleo_common.actions.base.TripleOAction.'
                 'get_compute_client')
-    @mock.patch('tripleo_common.actions.package_update.time')
     @mock.patch('heatclient.common.template_utils.get_template_contents')
-    def test_run(self, mock_template_contents,
+    @mock.patch('tripleo_common.utils.plan.get_env')
-                 mock_time,
+    @mock.patch('tripleo_common.utils.plan.update_in_env')
+    @mock.patch('heatclient.common.template_utils.deep_update')
+    def test_run(self, mock_deepupdate,
+                 mock_updateinenv,
+                 mock_getenv,
+                 mock_template_contents,
                  mock_compute_client,
                  mock_orchestration_client,
                  mock_object_client,
@@ -82,59 +54,68 @@ class UpdateStackActionTest(base.TestCase):
             'heat_template_version': '2016-04-30'
         })
         mock_swift = mock.MagicMock()
-        mock_env = """environments:
+        env = {
-- path: environments/test.yaml
+            'parameters': {
-name: container
+                'ControllerCount': 1,
-parameter_defaults:
+                'ComputeCount': 1,
-  random_data: a_value
+                'ObjectStorageCount': 0,
-temp_environment: temp_environment
+                'BlockStorageCount': 0,
-template: template
+                'CephStorageCount': 0,
-"""
+            },
-        mock_swift.get_object.return_value = ({}, mock_env)
+            'stack_name': 'overcloud',
+            'stack_status': "CREATE_COMPLETE",
+            'outputs': [
+                {'output_key': 'RoleConfig',
+                 'output_value': {
+                     'foo_config': 'foo'}},
+                {'output_key': 'RoleData',
+                 'output_value': {
+                     'FakeCompute': {
+                         'config_settings': {'nova::compute::fake'
+                                             'libvirt_virt_type': 'qemu'},
+                         'global_config_settings': {},
+                         'logging_groups': ['root', 'neutron', 'nova'],
+                         'logging_sources': [{'path': '/var/log/fake.log',
+                                             'type': 'tail'}],
+                         'monitoring_subscriptions': ['nova-compute'],
+                         'service_config_settings': None,
+                         'service_metadata_settings': None,
+                         'service_names': ['nova_compute', 'fake_service'],
+                         'step_config': ['include ::tripleo::profile::fake',
+                                         'include ::timezone'],
+                         'upgrade_batch_tasks': [],
+                         'upgrade_tasks': [{'name': 'Stop fake service',
+                                            'service': 'name=fo state=stopped',
+                                            'tags': 'step1',
+                                            'when': 'existingcondition'},
+                                           {'name': 'Stop nova-compute',
+                                            'service': 'name=nova-compute '
+                                                       'state=stopped',
+                                            'tags': 'step1',
+                                            'when': ['existing', 'list']}]
+                         }}}]}
+
+        update_env = {'resource_registry':
+                      {'OS::TripleO::DeploymentSteps': 'OS::Heat::None'}}
+
+        mock_getenv.return_value = env
+        fake_registry = {'parameter_defaults': [
+                         {'DockerKeystoneImage': '192.168.24.1:8787/'
+                                                 'keystone-docker:latest',
+                          'DockerHeatApiImage:': '192.168.24.1:8787/'
+                                                 'heat-api-docker:latest'}]}
+        update_env.update(fake_registry)
+        mock_swift.get_object.return_value = ({}, env)
         mock_object_client.return_value = mock_swift
 
-        # freeze time at datetime.datetime(2016, 9, 8, 16, 24, 24)
+        action = package_update.UpdateStackAction(self.timeout, fake_registry,
-        mock_time.time.return_value = 1473366264
-
-        mock_templates_run.return_value = {
-            'StackAction': 'UPDATE',
-            'DeployIdentifier': 1473366264,
-            'UpdateIdentifier': 1473366264
-        }
-
-        action = package_update.UpdateStackAction(self.timeout,
                                                   container=self.container)
         action.run(mock_ctx)
-
+        mock_updateinenv.assert_called_once_with(
-        # verify parameters are as expected
+            mock_swift, env, 'parameter_defaults',
-        updated_mock_env = """environments:
+            fake_registry['parameter_defaults']
-- path: environments/test.yaml
-name: container
-parameter_defaults:
-  DeployIdentifier: 1473366264
-  StackAction: UPDATE
-  UpdateIdentifier: 1473366264
-  random_data: a_value
-temp_environment: temp_environment
-template: template
-"""
-        mock_swift.put_object.assert_called_once_with(
-            self.container, constants.PLAN_ENVIRONMENT, updated_mock_env
         )
 
-        heat.stacks.update.assert_called_once_with(
+        mock_deepupdate.assert_called_once_with(env, update_env)
-            'stack_id',
+
-            StackAction='UPDATE',
+        heat.stacks.update.assert_called_once_with('stack_id')
-            DeployIdentifier=1473366264,
-            UpdateIdentifier=1473366264,
-            existing='true',
-            timeout_mins=1,
-            environment={
-                'resource_registry': {
-                    'resources': {
-                        '*': {
-                            '*': {'UpdateDeployment': {'hooks': 'pre-update'}}
-                        }
-                    }
-                }
-            })

tripleo_common/tests/fake_config/fakes.py

@@ -0,0 +1,91 @@
+#   Copyright 2015 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+import mock
+
+
+FAKE_STACK = {
+    'parameters': {
+        'ControllerCount': 1,
+        'ComputeCount': 1,
+        'ObjectStorageCount': 0,
+        'BlockStorageCount': 0,
+        'CephStorageCount': 0,
+    },
+    'stack_name': 'overcloud',
+    'stack_status': "CREATE_COMPLETE",
+    'outputs': [
+        {'output_key': 'RoleConfig',
+         'output_value': {
+             'foo_config': 'foo'}},
+        {'output_key': 'RoleData',
+         'output_value': {
+             'FakeCompute': {
+                 'config_settings': {'nova::compute::libvirt::services::'
+                                     'libvirt_virt_type': 'qemu'},
+                 'global_config_settings': {},
+                 'logging_groups': ['root', 'neutron', 'nova'],
+                 'logging_sources': [{'path': '/var/log/nova/nova-compute.log',
+                                     'type': 'tail'}],
+                 'monitoring_subscriptions': ['overcloud-nova-compute'],
+                 'service_config_settings': {'horizon': {'neutron::'
+                                                         'plugins': ['ovs']}
+                                             },
+                 'service_metadata_settings': None,
+                 'service_names': ['nova_compute', 'fake_service'],
+                 'step_config': ['include ::tripleo::profile::base::sshd',
+                                 'include ::timezone'],
+                 'upgrade_batch_tasks': [],
+                 'upgrade_tasks': [{'name': 'Stop fake service',
+                                    'service': 'name=fake state=stopped',
+                                    'tags': 'step1',
+                                    'when': 'existingcondition'},
+                                   {'name': 'Stop nova-compute service',
+                                    'service': 'name=openstack-nova-compute '
+                                               'state=stopped',
+                                    'tags': 'step1',
+                                    'when': ['existing', 'list']}]
+                 },
+             'FakeController': {
+                 'config_settings': {'tripleo::haproxy::user': 'admin'},
+                 'global_config_settings': {},
+                 'logging_groups': ['root', 'keystone', 'neutron'],
+                 'logging_sources': [{'path': '/var/log/keystone/keystone.log',
+                                     'type': 'tail'}],
+                 'monitoring_subscriptions': ['overcloud-keystone'],
+                 'service_config_settings': {'horizon': {'neutron::'
+                                                         'plugins': ['ovs']}
+                                             },
+                 'service_metadata_settings': None,
+                 'service_names': ['pacemaker', 'fake_service'],
+                 'step_config': ['include ::tripleo::profile::base::sshd',
+                                 'include ::timezone'],
+                 'upgrade_batch_tasks': [],
+                 'upgrade_tasks': [{'name': 'Stop fake service',
+                                    'service': 'name=fake state=stopped',
+                                    'tags': 'step1'}]}}}]}
+
+
+def create_to_dict_mock(**kwargs):
+    mock_with_to_dict = mock.Mock()
+    mock_with_to_dict.configure_mock(**kwargs)
+    mock_with_to_dict.to_dict.return_value = kwargs
+    return mock_with_to_dict
+
+
+def create_tht_stack(**kwargs):
+    stack = FAKE_STACK.copy()
+    stack.update(kwargs)
+    return create_to_dict_mock(**stack)

tripleo_common/tests/utils/test_config.py

@@ -0,0 +1,143 @@
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+import fixtures
+import mock
+import os
+
+from mock import call
+from mock import patch
+
+from tripleo_common.tests import base
+from tripleo_common.tests.fake_config import fakes
+from tripleo_common.utils import config as ooo_config
+
+
+class TestConfig(base.TestCase):
+
+    def setUp(self):
+        super(TestConfig, self).setUp()
+
+    @patch.object(ooo_config.Config, '_mkdir')
+    @patch.object(ooo_config.Config, '_open_file')
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    def test_overcloud_config_generate_config(self,
+                                              mock_tmpdir,
+                                              mock_open,
+                                              mock_mkdir):
+        config_type_list = ['config_settings', 'global_config_settings',
+                            'logging_sources', 'monitoring_subscriptions',
+                            'service_config_settings',
+                            'service_metadata_settings',
+                            'service_names',
+                            'upgrade_batch_tasks', 'upgrade_tasks']
+        fake_role = [role for role in
+                     fakes.FAKE_STACK['outputs'][1]['output_value']]
+
+        heat = mock.MagicMock()
+        heat.stacks.get.return_value = fakes.create_tht_stack()
+        self.config = ooo_config.Config(heat)
+        mock_tmpdir.return_value = "/tmp/tht"
+        self.config.download_config('overcloud', '/tmp', config_type_list)
+
+        expected_mkdir_calls = [call('/tmp/tht/%s' % r) for r in fake_role]
+        mock_mkdir.assert_has_calls(expected_mkdir_calls, any_order=True)
+        expected_calls = []
+        for config in config_type_list:
+            for role in fake_role:
+                if config == 'step_config':
+                    expected_calls += [call('/tmp/tht/%s/%s.pp' %
+                                            (role, config))]
+                else:
+                    expected_calls += [call('/tmp/tht/%s/%s.yaml' %
+                                            (role, config))]
+        mock_open.assert_has_calls(expected_calls, any_order=True)
+
+    @patch.object(ooo_config.Config, '_mkdir')
+    @patch.object(ooo_config.Config, '_open_file')
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    def test_overcloud_config_one_config_type(self,
+                                              mock_tmpdir,
+                                              mock_open,
+                                              mock_mkdir):
+
+        expected_config_type = 'config_settings'
+        fake_role = [role for role in
+                     fakes.FAKE_STACK['outputs'][1]['output_value']]
+
+        heat = mock.MagicMock()
+        heat.stacks.get.return_value = fakes.create_tht_stack()
+        self.config = ooo_config.Config(heat)
+        mock_tmpdir.return_value = "/tmp/tht"
+        self.config.download_config('overcloud', '/tmp', ['config_settings'])
+        expected_mkdir_calls = [call('/tmp/tht/%s' % r) for r in fake_role]
+        expected_calls = [call('/tmp/tht/%s/%s.yaml'
+                          % (r, expected_config_type))
+                          for r in fake_role]
+        mock_mkdir.assert_has_calls(expected_mkdir_calls, any_order=True)
+        mock_open.assert_has_calls(expected_calls, any_order=True)
+
+    @mock.patch('os.mkdir')
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    def test_overcloud_config_wrong_config_type(self, mock_tmpdir,
+                                                mock_open, mock_mkdir):
+        args = {'name': 'overcloud', 'config_dir': '/tmp',
+                'config_type': ['bad_config']}
+        heat = mock.MagicMock()
+        heat.stacks.get.return_value = fakes.create_tht_stack()
+        self.config = ooo_config.Config(heat)
+        mock_tmpdir.return_value = "/tmp/tht"
+        self.assertRaises(
+            KeyError,
+            self.config.download_config, *args)
+
+    @mock.patch('tripleo_common.utils.config.Config.get_role_data',
+                autospec=True)
+    def test_overcloud_config_upgrade_tasks(self, mock_get_role_data):
+
+        heat = mock.MagicMock()
+        heat.stacks.get.return_value = fakes.create_tht_stack()
+        self.config = ooo_config.Config(heat)
+        self.tmp_dir = self.useFixture(fixtures.TempDir()).path
+        fake_role = [role for role in
+                     fakes.FAKE_STACK['outputs'][1]['output_value']]
+        expected_tasks = {'FakeController': [{'name': 'Stop fake service',
+                                              'service': 'name=fake '
+                                              'state=stopped',
+                                              'tags': 'step1',
+                                              'when': 'step|int == 1'}],
+                          'FakeCompute': [{'name': 'Stop fake service',
+                                           'service':
+                                           'name=fake state=stopped',
+                                           'tags': 'step1',
+                                           'when': ['existingcondition',
+                                                    'step|int == 1']},
+                                          {'name': 'Stop nova-'
+                                           'compute service',
+                                           'service':
+                                           'name=openstack-nova-'
+                                           'compute state=stopped',
+                                           'tags': 'step1',
+                                           'when': ['existing',
+                                                    'list', 'step|int == 1']}]}
+        mock_get_role_data.return_value = fake_role
+
+        for role in fake_role:
+            filedir = os.path.join(self.tmp_dir, role)
+            os.makedirs(filedir)
+            filepath = os.path.join(filedir, "upgrade_tasks_playbook.yaml")
+            playbook_tasks = self.config._write_playbook_get_tasks(
+                fakes.FAKE_STACK['outputs'][1]['output_value'][role]
+                ['upgrade_tasks'], role, filepath)
+            self.assertTrue(os.path.isfile(filepath))
+            self.assertEqual(expected_tasks[role], playbook_tasks)

tripleo_common/utils/config.py

@@ -0,0 +1,142 @@
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import logging
+import os
+import re
+import six
+import tempfile
+import yaml
+
+
+class Config(object):
+
+    def __init__(self, orchestration_client):
+        self.log = logging.getLogger(__name__ + ".Config")
+        self.client = orchestration_client
+
+    def get_role_data(self, stack):
+        role_data = {}
+        for output in stack.to_dict().get('outputs', {}):
+            if output['output_key'] == 'RoleData':
+                for role in output['output_value']:
+                    role_data[role] = output['output_value'][role]
+        return role_data
+
+    def get_role_config(self, stack):
+        role_data = {}
+        for output in stack.to_dict().get('outputs', {}):
+            if output['output_key'] == 'RoleConfig':
+                for role in output['output_value']:
+                    role_data[role] = output['output_value'][role]
+        return role_data
+
+    @staticmethod
+    def _open_file(path):
+        return os.fdopen(os.open(path,
+                                 os.O_WRONLY | os.O_CREAT, 0o600),
+                         'w')
+
+    def _step_tags_to_when(self, sorted_tasks):
+        for task in sorted_tasks:
+            tag = task.get('tags', '')
+            match = re.search('step([0-9]+)', tag)
+            if match:
+                step = match.group(1)
+                whenexpr = task.get('when', None)
+                if whenexpr:
+                    # Handle when: foo and a list of when conditionals
+                    if not isinstance(whenexpr, list):
+                        whenexpr = [whenexpr]
+                    for w in whenexpr:
+                        when_exists = re.search('step|int == [0-9]', w)
+                        if when_exists:
+                            break
+                    if when_exists:
+                        # Skip to the next task,
+                        # there is an existing 'step|int == N'
+                        continue
+                    whenexpr.append("step|int == %s" % step)
+                    task['when'] = whenexpr
+                else:
+                    task.update({"when": "step|int == %s" % step})
+
+    def _write_playbook_get_tasks(self, tasks, role, filepath):
+        playbook = []
+        sorted_tasks = sorted(tasks, key=lambda x: x.get('tags', None))
+        self._step_tags_to_when(sorted_tasks)
+        playbook.append({'name': '%s playbook' % role,
+                         'hosts': role,
+                         'tasks': sorted_tasks})
+        with self._open_file(filepath) as conf_file:
+            yaml.safe_dump(playbook, conf_file, default_flow_style=False)
+        return sorted_tasks
+
+    def _mkdir(self, dirname):
+        if not os.path.exists(dirname):
+            try:
+                os.mkdir(dirname, 0o700)
+            except OSError as e:
+                message = 'Failed to create: %s, error: %s' % (dirname,
+                                                               str(e))
+                raise OSError(message)
+
+    def download_config(self, name, config_dir, config_type=None):
+        # Get the stack object
+        stack = self.client.stacks.get(name)
+        # Create config directory
+        self._mkdir(config_dir)
+        tmp_path = tempfile.mkdtemp(prefix='tripleo-',
+                                    suffix='-config',
+                                    dir=config_dir)
+        self.log.info("Generating configuration under the directory: "
+                      "%s" % tmp_path)
+        # Get role data:
+        role_data = self.get_role_data(stack)
+        for role_name, role in six.iteritems(role_data):
+            role_path = os.path.join(tmp_path, role_name)
+            self._mkdir(role_path)
+            for config in config_type or role.keys():
+                if config == 'step_config':
+                    filepath = os.path.join(role_path, 'step_config.pp')
+                    with self._open_file(filepath) as step_config:
+                        step_config.write('\n'.join(step for step in
+                                                    role[config]
+                                                    if step is not None))
+                else:
+                    if 'upgrade_tasks' in config:
+                        filepath = os.path.join(role_path, '%s_playbook.yaml' %
+                                                config)
+                        data = self._write_playbook_get_tasks(
+                            role[config], role_name, filepath)
+                    else:
+                        try:
+                            data = role[config]
+                        except KeyError as e:
+                            message = 'Invalid key: %s, error: %s' % (config,
+                                                                      str(e))
+                            raise KeyError(message)
+                    filepath = os.path.join(role_path, '%s.yaml' % config)
+                    with self._open_file(filepath) as conf_file:
+                        yaml.safe_dump(data,
+                                       conf_file,
+                                       default_flow_style=False)
+        role_config = self.get_role_config(stack)
+        for config_name, config in six.iteritems(role_config):
+            conf_path = os.path.join(tmp_path, config_name + ".yaml")
+            with self._open_file(conf_path) as conf_file:
+                conf_file.write(config)
+        self.log.info("The TripleO configuration has been successfully "
+                      "generated into: %s" % tmp_path)
+        return tmp_path

workbooks/package_update.yaml

@@ -11,15 +11,34 @@ workflows:
 
     input:
       - container
+      - container_registry
       - timeout: 240
       - queue_name: tripleo
+      - skip_deploy_identifier: False
+      - config_dir: '/tmp/'
 
     tags:
       - tripleo-common-managed
 
     tasks:
+      update_plan:
+        action: tripleo.plan.update_from_dir
+        input:
+          container: <% $.container %>
+        on-success: update
+        on-error: set_update_failed
+
       update:
-        action: tripleo.package_update.update_stack container=<% $.container %> timeout=<% $.timeout %>
+        action: tripleo.package_update.update_stack container=<% $.container %> timeout=<% $.timeout %> container_registry=<% $.container_registry %>
+        input:
+          timeout: <% $.timeout %>
+          container: <% $.container %>
+          container_registry: <% $.container_registry %>
+        on-success: get_config
+        on-error: set_update_failed
+
+      get_config:
+        action: tripleo.config.get_overcloud_config
         on-success: send_message
         on-error: set_update_failed
 
@@ -41,42 +60,75 @@ workflows:
                 message: <% $.get('message', '') %>
                 execution: <% execution() %>
         on-success:
-          - fail: <% $.get('status') = "FAILED" %>
+          - fail: <% $.get('get_config') = "FAILED" %>
 
-  # Clear an update breakpoint
+  update_nodes:
-  clear_breakpoints:
+    description: Take a container and perform an update nodes by nodes
-    description: Clear any pending breakpoints and continue with update
 
     input:
-      - stack_id
+      - node_user: heat-admin
-      - refs
+      - nodes
+      - playbook
+      - inventory_file
       - queue_name: tripleo
 
     tags:
       - tripleo-common-managed
 
     tasks:
-      clear:
+      download_config:
-        action: tripleo.package_update.clear_breakpoints stack_id=<% $.stack_id %> refs=<% $.refs %>
+        action: tripleo.config.download_config
-        on-success: send_message
+        on-success: get_private_key
-        on-error: set_clear_breakpoints_failed
+        publish:
+          tmp_path: <% task(download_config).result %>
+        on-error: node_update_failed
 
-      set_clear_breakpoints_failed:
+      get_private_key:
-        on-success: send_message
+        action: tripleo.validations.get_privkey
+        publish:
+          private_key: <% task(get_private_key).result %>
+        on-success: node_update
+
+      node_update:
+        action: tripleo.ansible-playbook
+        input:
+          inventory: <% $.inventory_file %>
+          playbook: <% $.tmp_path %>/<% $.playbook %>
+          remote_user: <% $.node_user %>
+          become: true
+          become_user: root
+          verbosity: 0
+          ssh_private_key: <% $.private_key %>
+          ssh_extra_args: '-o StrictHostKeyChecking=no'
+          limit_hosts:  <% $.nodes %>
+        on-success: node_update_passed
+        on-error: node_update_failed
+        publish:
+          output: <% task(node_update).result %>
+
+      node_update_passed:
+        on-success: notify_zaqar
+        publish:
+          status: SUCCESS
+          message: Updated nodes - <% $.nodes %>
+
+      node_update_failed:
+        on-success: notify_zaqar
         publish:
           status: FAILED
-          message: <% task(clear).result %>
+          message: Failed to update nodes - <% $.nodes %>, please see the logs.
 
-      send_message:
+      notify_zaqar:
         action: zaqar.queue_post
+        retry: count=5 delay=1
         input:
           queue_name: <% $.queue_name %>
           messages:
             body:
-              type: tripleo.package_update.v1.clear_breakpoints
+              type: tripleo.package_update.v1.update_nodes
               payload:
-                status: <% $.get('status', 'SUCCESS') %>
+                status: <% $.status %>
-                message: <% $.get('message', '') %>
+                message: <% task(node_update).result %>
                 execution: <% execution() %>
         on-success:
           - fail: <% $.get('status') = "FAILED" %>