Dump all deployment role_data files to a config dir

Add new overcloud cli command in order to dump
all the configs from the overcloud contains in role_data.
All different type of configs are dump in a separate file
per role ex:
ls config/tmpD2nDOd-config/
config_settings-Compute.yaml
global_config_settings-Compute.yaml
logging_sources-Compute.yaml
monitoring_subscriptions-Compute.yaml
service_config_settings-Compute.yaml
service_metadata_settings-Compute.yaml
service_names-Compute.yaml
step_config-Compute.yaml
upgrade_batch_tasks-Compute.yaml
upgrade_tasks-Compute.yaml

The upgrade_tasks is in yaml format and can be directly
call with ansible-playbook command, example:

$ cat config/tmpD2nDOd-config/upgrade_tasks-Compute.yaml
- hosts: Compute
  name: Compute playbook
  tasks:
  - name: Stop snmp service
    service: name=snmpd state=stopped
    tags: step1
  - name: Stop nova-compute service
    service: name=openstack-nova-compute state=stopped
    tags: step1

This patch to allow operators to run manually
the upgrade tasks via ansible on the Undercloud.

Change-Id: I7d89ffde8a19b30aaf59efb11e41d32cfa644b83
Closes-bug: #1665585

setup.cfg

@@ -64,6 +64,7 @@ openstack.tripleoclient.v1 =
     baremetal_configure_ready_state = tripleoclient.v1.baremetal:ConfigureReadyState
     baremetal_configure_boot = tripleoclient.v1.baremetal:ConfigureBaremetalBoot
     overcloud_netenv_validate = tripleoclient.v1.overcloud_netenv_validate:ValidateOvercloudNetenv
+    overcloud_download_config = tripleoclient.v1.overcloud_config:DownloadConfig
     overcloud_container_image_upload = tripleoclient.v1.container_image:UploadImage
     overcloud_container_image_build = tripleoclient.v1.container_image:BuildImage
     overcloud_delete = tripleoclient.v1.overcloud_delete:DeleteOvercloud

tripleoclient/tests/v1/overcloud_config/fakes.py

@@ -0,0 +1,90 @@
+#   Copyright 2015 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+import mock
+
+
+FAKE_STACK = {
+    'parameters': {
+        'ControllerCount': 1,
+        'ComputeCount': 1,
+        'ObjectStorageCount': 0,
+        'BlockStorageCount': 0,
+        'CephStorageCount': 0,
+    },
+    'stack_name': 'overcloud',
+    'stack_status': "CREATE_COMPLETE",
+    'outputs': [{
+        'output_key': 'RoleData',
+        'output_value': {
+            'FakeCompute': {
+                'config_settings': {'nova::compute::libvirt::services::'
+                                    'libvirt_virt_type': 'qemu'},
+                'global_config_settings': {},
+                'logging_groups': ['root', 'neutron', 'nova'],
+                'logging_sources': [{'path': '/var/log/nova/nova-compute.log',
+                                    'type': 'tail'}],
+                'monitoring_subscriptions': ['overcloud-nova-compute'],
+                'service_config_settings': {'horizon': {'neutron::'
+                                                        'plugins': ['ovs']}
+                                            },
+                'service_metadata_settings': None,
+                'service_names': ['nova_compute', 'fake_service'],
+                'step_config': ['include ::tripleo::profile::base::sshd',
+                                'include ::timezone'],
+                'upgrade_batch_tasks': [],
+                'upgrade_tasks': [{'name': 'Stop fake service',
+                                   'service': 'name=fake state=stopped',
+                                   'tags': 'step1'},
+                                  {'name': 'Stop nova-compute service',
+                                   'service': 'name=openstack-nova-compute '
+                                              'state=stopped',
+                                   'tags': 'step1'}]
+                },
+            'FakeController': {
+                'config_settings': {'tripleo::haproxy::user': 'admin'},
+                'global_config_settings': {},
+                'logging_groups': ['root', 'keystone', 'neutron'],
+                'logging_sources': [{'path': '/var/log/keystone/keystone.log',
+                                    'type': 'tail'}],
+                'monitoring_subscriptions': ['overcloud-keystone'],
+                'service_config_settings': {'horizon': {'neutron::'
+                                                        'plugins': ['ovs']}
+                                            },
+                'service_metadata_settings': None,
+                'service_names': ['pacemaker', 'fake_service'],
+                'step_config': ['include ::tripleo::profile::base::sshd',
+                                'include ::timezone'],
+                'upgrade_batch_tasks': [],
+                'upgrade_tasks': [{'name': 'Stop fake service',
+                                   'service': 'name=fake state=stopped',
+                                   'tags': 'step1'}]
+                }
+        }
+    }]
+}
+
+
+def create_to_dict_mock(**kwargs):
+    mock_with_to_dict = mock.Mock()
+    mock_with_to_dict.configure_mock(**kwargs)
+    mock_with_to_dict.to_dict.return_value = kwargs
+    return mock_with_to_dict
+
+
+def create_tht_stack(**kwargs):
+    stack = FAKE_STACK.copy()
+    stack.update(kwargs)
+    return create_to_dict_mock(**stack)

tripleoclient/tests/v1/overcloud_config/test_overcloud_config.py

@@ -0,0 +1,179 @@
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+import mock
+
+from mock import call
+from osc_lib.tests import utils
+
+from tripleoclient.tests.v1.overcloud_config import fakes
+from tripleoclient.v1 import overcloud_config
+
+
+class TestOvercloudConfig(utils.TestCommand):
+
+    def setUp(self):
+        super(TestOvercloudConfig, self).setUp()
+
+        self.cmd = overcloud_config.DownloadConfig(self.app, None)
+        self.app.client_manager.workflow_engine = mock.Mock()
+        self.app.client_manager.orchestration = mock.Mock()
+        self.workflow = self.app.client_manager.workflow_engine
+
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    def test_overcloud_config_generate_config(self, mock_tmpdir,  mock_open):
+
+        arglist = ['--name', 'overcloud', '--config-dir', '/tmp']
+        verifylist = [
+            ('name', 'overcloud'),
+            ('config_dir', '/tmp')
+        ]
+        config_type_list = ['config_settings', 'global_config_settings',
+                            'logging_sources', 'monitoring_subscriptions',
+                            'service_config_settings',
+                            'service_metadata_settings',
+                            'service_names', 'step_config',
+                            'upgrade_batch_tasks', 'upgrade_tasks']
+        fake_role = [role for role in
+                     fakes.FAKE_STACK['outputs'][0]['output_value']]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        clients = self.app.client_manager
+        orchestration_client = clients.orchestration
+        orchestration_client.stacks.get.return_value = fakes.create_tht_stack()
+        mock_tmpdir.return_value = "/tmp/tht"
+        self.cmd.take_action(parsed_args)
+        for config in config_type_list:
+            for role in fake_role:
+                if 'step_config' in config:
+                    expected_calls = [call('/tmp/tht/%s-%s.pp' %
+                                           (config, role), 'w')]
+                else:
+                    expected_calls = [call('/tmp/tht/%s-%s.yaml' %
+                                           (config, role), 'w')]
+                mock_open.assert_has_calls(expected_calls, any_order=True)
+
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    def test_overcloud_config_one_config_type(self, mock_tmpdir,  mock_open):
+
+        arglist = ['--name', 'overcloud', '--config-dir', '/tmp',
+                   '--config-type', ['config_settings']]
+        verifylist = [
+            ('name', 'overcloud'),
+            ('config_dir', '/tmp'),
+            ('config_type', ['config_settings'])
+        ]
+        config_type_list = ['config_settings', 'global_config_settings',
+                            'logging_sources', 'monitoring_subscriptions',
+                            'service_config_settings',
+                            'service_metadata_settings',
+                            'service_names', 'step_config',
+                            'upgrade_batch_tasks', 'upgrade_tasks']
+        expected_config_type = 'config_settings'
+        fake_role = [role for role in
+                     fakes.FAKE_STACK['outputs'][0]['output_value']]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        clients = self.app.client_manager
+        orchestration_client = clients.orchestration
+        orchestration_client.stacks.get.return_value = fakes.create_tht_stack()
+        mock_tmpdir.return_value = "/tmp/tht"
+        self.cmd.take_action(parsed_args)
+        for config in config_type_list:
+            if config == expected_config_type:
+                for role in fake_role:
+                    expected_calls = [call('/tmp/tht/%s-%s.yaml'
+                                           % (config, role), 'w')]
+                    mock_open.assert_has_calls(expected_calls, any_order=True)
+            else:
+                for role in fake_role:
+                    unexpected_calls = [call('/tmp/tht/%s-%s.yaml'
+                                             % (config, role), 'w')]
+                    try:
+                        mock_open.assert_has_calls(unexpected_calls,
+                                                   any_order=True)
+                    except AssertionError:
+                        pass
+
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    def test_overcloud_config_wrong_config_type(self, mock_tmpdir, mock_open):
+
+        arglist = [
+            '--name', 'overcloud',
+            '--config-dir',
+            '/tmp', '--config-type', ['bad_config']]
+        verifylist = [
+            ('name', 'overcloud'),
+            ('config_dir', '/tmp'),
+            ('config_type', ['bad_config'])
+        ]
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+        clients = self.app.client_manager
+        orchestration_client = clients.orchestration
+        orchestration_client.stacks.get.return_value = fakes.create_tht_stack()
+        self.assertRaises(
+            KeyError,
+            self.cmd.take_action, parsed_args)
+
+    @mock.patch('tripleoclient.utils.get_role_data', autospec=True)
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    def test_overcloud_config_upgrade_tasks(self, mock_tmpdir,
+                                            mock_open,
+                                            mock_get_role_data):
+
+        clients = self.app.client_manager
+        orchestration_client = clients.orchestration
+        orchestration_client.stacks.get.return_value = fakes.create_tht_stack()
+        mock_tmpdir.return_value = "/tmp/tht"
+        fake_role = [role for role in
+                     fakes.FAKE_STACK['outputs'][0]['output_value']]
+        fake_playbook = {'FakeController': [{'hosts': 'FakeController',
+                                             'name': 'FakeController playbook',
+                                             'tasks': [{'name': 'Stop fake '
+                                                                'service',
+                                                        'service':
+                                                        'name=fake '
+                                                        'state=stopped',
+                                                        'tags': 'step1'}]
+                                             }],
+                         'FakeCompute': [{'hosts': 'FakeCompute',
+                                          'name': 'FakeCompute playbook',
+                                          'tasks': [{'name': 'Stop fake '
+                                                             'service',
+                                                     'service':
+                                                     'name=fake state=stopped',
+                                                     'tags': 'step1'},
+                                                    {'name': 'Stop nova-'
+                                                             'compute service',
+                                                     'service':
+                                                     'name=openstack-nova-'
+                                                     'compute state=stopped',
+                                                     'tags': 'step1'}]
+                                          }]
+                         }
+        mock_get_role_data.return_value = fake_role
+
+        for role in fake_role:
+            playbook = self.cmd._convert_playbook(fakes.
+                                                  FAKE_STACK['outputs']
+                                                            [0]
+                                                            ['output_value']
+                                                            [role]
+                                                            ['upgrade_tasks'],
+                                                  role)
+            self.assertEqual(fake_playbook[role], playbook)

tripleoclient/utils.py

@@ -341,6 +341,15 @@ def get_endpoint_map(stack):
     return endpoint_map
 
 
+def get_role_data(stack):
+    role_data = {}
+    for output in stack.to_dict().get('outputs', {}):
+        if output['output_key'] == 'RoleData':
+            for role in output['output_value']:
+                role_data[role] = output['output_value'][role]
+    return role_data
+
+
 def get_endpoint(key, stack):
     endpoint_map = get_endpoint_map(stack)
     if endpoint_map:

tripleoclient/v1/overcloud_config.py

@@ -0,0 +1,115 @@
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+import logging
+import os
+import tempfile
+import yaml
+
+from osc_lib.command import command
+from osc_lib.i18n import _
+
+from tripleoclient import utils
+
+
+class DownloadConfig(command.Command):
+    """Download Overcloud Config"""
+
+    log = logging.getLogger(__name__ + ".DownloadConfig")
+
+    def get_parser(self, prog_name):
+        parser = super(DownloadConfig, self).get_parser(prog_name)
+        parser.add_argument(
+            '--name',
+            dest='name',
+            default='overcloud',
+            help=_('The name of the plan, which is used for the object '
+                   'storage container, workflow environment and orchestration '
+                   'stack names.'),
+        )
+        parser.add_argument(
+            '--config-dir',
+            dest='config_dir',
+            default=os.path.expanduser("~"),
+            help=_('The directory where the configuration files will be '
+                   'pushed'),
+        )
+        parser.add_argument(
+            '--config-type',
+            dest='config_type',
+            type=list,
+            default=['config_settings', 'global_config_settings',
+                     'logging_sources', 'monitoring_subscriptions',
+                     'service_config_settings', 'service_metadata_settings',
+                     'service_names', 'step_config', 'upgrade_batch_tasks',
+                     'upgrade_tasks'],
+            help=_('Type of object config to be extract from the deployment'),
+        )
+        return parser
+
+    def _convert_playbook(self, tasks, role):
+        playbook = []
+        sorted_tasks = sorted(tasks, key=lambda x: x.get('tags', None))
+        playbook.append({'name': '%s playbook' % role,
+                         'hosts': role,
+                         'tasks': sorted_tasks})
+        return playbook
+
+    def take_action(self, parsed_args):
+        self.log.debug("take_action(%s)" % parsed_args)
+        clients = self.app.client_manager
+
+        name = parsed_args.name
+        configs = parsed_args.config_type
+        config_dir = parsed_args.config_dir
+        if not os.path.exists(config_dir):
+            try:
+                os.mkdir(config_dir)
+            except OSError as e:
+                message = 'Failed to create: %s, error: %s' % (config_dir,
+                                                               str(e))
+                raise OSError(message)
+        stack = utils.get_stack(clients.orchestration, name)
+        tmp_path = tempfile.mkdtemp(prefix='tripleo-',
+                                    suffix='-config',
+                                    dir=config_dir)
+        self.log.info("Generating configuration under the directory: "
+                      "%s" % tmp_path)
+        role_data = utils.get_role_data(stack)
+        for role in role_data:
+            for config in configs:
+                if 'step_config' in config:
+                        with open('%s/%s-%s.pp' % (tmp_path,
+                                                   config,
+                                                   role), 'w') as step_config:
+                            step_config.write('\n'.join(step for step in
+                                                        role_data[role][config]
+                                                        if step is not None))
+                else:
+                    if 'upgrade_tasks' in config:
+                        data = self._convert_playbook(role_data[role][config],
+                                                      role)
+                    else:
+                        try:
+                            data = role_data[role][config]
+                        except KeyError as e:
+                            message = 'Invalide key: %s, error: %s' % (config,
+                                                                       str(e))
+                            raise KeyError(message)
+                    with open('%s/%s-%s.yaml' % (tmp_path,
+                                                 config,
+                                                 role), 'w') as conf_file:
+                        yaml.safe_dump(data,
+                                       conf_file,
+                                       default_flow_style=False)
+        print("The TripleO configuration has been successfully generated "
+              "into: {0}".format(tmp_path))