Add openstack overcloud export ceph

Add a new command to export Ceph information from one or
more Heat stacks to be used as input of another stack.
Creates a valid YAML file with the CephExternalMultiConfig
parameter populated.

Also have export's export_password use yaml.safe_load in
place of the deprecated yaml.load.

Closes-Bug: #1895034
Change-Id: Ibdf9115e92c6b476b99d6df785b0c7e9f23991de

releasenotes/notes/openstack-overcloud-export-ceph-f36421e1685db302.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - A new command "openstack overcloud export ceph" is added. The command is
+    used to export the Ceph deployment data from one stack for use in another
+    stack with storage services which use that Ceph cluster when using the
+    multi-stack deployment feature.

setup.cfg

@@ -48,6 +48,7 @@ openstack.tripleoclient.v2 =
     overcloud_credentials = tripleoclient.v1.overcloud_credentials:OvercloudCredentials
     overcloud_deploy = tripleoclient.v1.overcloud_deploy:DeployOvercloud
     overcloud_export = tripleoclient.v1.overcloud_export:ExportOvercloud
+    overcloud_export_ceph = tripleoclient.v1.overcloud_export_ceph:ExportOvercloudCeph
     overcloud_status = tripleoclient.v1.overcloud_deploy:GetDeploymentStatus
     overcloud_image_build = tripleoclient.v1.overcloud_image:BuildOvercloudImage
     overcloud_image_upload = tripleoclient.v1.overcloud_image:UploadOvercloudImage

tripleoclient/export.py

@@ -42,7 +42,7 @@ def export_passwords(swift, stack, excludes=True):
                   "file from swift: %s", str(e))
         sys.exit(1)
 
-    data = yaml.load(content)
+    data = yaml.safe_load(content)
     # The "passwords" key in plan-environment.yaml are generated passwords,
     # they are not necessarily the actual password values used during the
     # deployment.
@@ -141,3 +141,64 @@ def export_stack(heat, stack, should_filter=False,
                 "No data returned to export %s from." % param)
 
     return data
+
+
+def export_storage_ips(stack, config_download_dir=constants.DEFAULT_WORK_DIR):
+    inventory_file = "ceph-ansible/inventory.yml"
+    file = os.path.join(config_download_dir, stack, inventory_file)
+    with open(file, 'r') as ff:
+        try:
+            inventory_data = yaml.safe_load(ff)
+        except Exception as e:
+            LOG.error(
+                _('Could not read file %s') % file)
+            LOG.error(e)
+    mon_ips = []
+    for mon_role in inventory_data['mons']['children'].keys():
+        for hostname in inventory_data[mon_role]['hosts']:
+            ip = inventory_data[mon_role]['hosts'][hostname]['storage_ip']
+            mon_ips.append(ip)
+
+    return mon_ips
+
+
+def export_ceph(stack, cephx,
+                config_download_dir=constants.DEFAULT_WORK_DIR,
+                mon_ips=[]):
+    # Return a map of ceph data for a list item in CephExternalMultiConfig
+    # by parsing files within the config_download_dir of a certain stack
+
+    if len(mon_ips) == 0:
+        mon_ips = export_storage_ips(stack, config_download_dir)
+
+    # Use ceph-ansible group_vars/all.yml to get remaining values
+    ceph_ansible_all = "ceph-ansible/group_vars/all.yml"
+    file = os.path.join(config_download_dir, stack, ceph_ansible_all)
+    with open(file, 'r') as ff:
+        try:
+            ceph_data = yaml.safe_load(ff)
+        except Exception as e:
+            LOG.error(
+                _('Could not read file %s') % file)
+            LOG.error(e)
+
+    for key in ceph_data['keys']:
+        if key['name'] == 'client.' + str(cephx):
+            cephx_keys = [key]
+
+    ceph_conf_overrides = {}
+    ceph_conf_overrides['client'] = {}
+    ceph_conf_overrides['client']['keyring'] = '/etc/ceph/' \
+                                               + ceph_data['cluster'] \
+                                               + '.client.' + cephx \
+                                               + '.keyring'
+    # Combine extracted data into one map to return
+    data = {}
+    data['external_cluster_mon_ips'] = str(','.join(mon_ips))
+    data['keys'] = cephx_keys
+    data['ceph_conf_overrides'] = ceph_conf_overrides
+    data['cluster'] = ceph_data['cluster']
+    data['fsid'] = ceph_data['fsid']
+    data['dashboard_enabled'] = False
+
+    return data

tripleoclient/tests/test_export.py

@@ -42,6 +42,31 @@ class TestExport(TestCase):
         self.mock_stack.to_dict.return_value = dict(outputs=outputs)
         self.mock_open = mock.mock_open(read_data='{"an_key":"an_value"}')
 
+        ceph_inv = {
+            'DistributedComputeHCI': {
+                'hosts': {
+                    'dcn0-distributedcomputehci-0': {
+                        'storage_ip': '192.168.24.42'
+                    }
+                }
+            },
+            'mons': {
+                'children': {
+                    'DistributedComputeHCI': {}
+                }
+            }
+        }
+        self.mock_open_ceph_inv = mock.mock_open(read_data=str(ceph_inv))
+
+        ceph_all = {
+            'cluster': 'dcn0',
+            'fsid': 'a5a22d37-e01f-4fa0-a440-c72585c7487f',
+            'keys': [
+                {'name': 'client.openstack'}
+            ]
+        }
+        self.mock_open_ceph_all = mock.mock_open(read_data=str(ceph_all))
+
     @mock.patch('tripleoclient.utils.get_stack')
     def test_export_stack(self, mock_get_stack):
         heat = mock.Mock()
@@ -151,3 +176,34 @@ class TestExport(TestCase):
         mock_passwords['passwords'].pop('CephRgwKey')
 
         self.assertEqual(mock_passwords['passwords'], data)
+
+    def test_export_storage_ips(self):
+        with mock.patch('six.moves.builtins.open', self.mock_open_ceph_inv):
+            storage_ips = export.export_storage_ips('dcn0',
+                                                    config_download_dir='/foo')
+        self.assertEqual(storage_ips, ['192.168.24.42'])
+        self.mock_open_ceph_inv.assert_called_once_with(
+            '/foo/dcn0/ceph-ansible/inventory.yml', 'r')
+
+    def test_export_ceph(self):
+        expected = {
+            'external_cluster_mon_ips': '192.168.24.42',
+            'keys': [
+                {'name': 'client.openstack'}
+            ],
+            'ceph_conf_overrides': {
+                'client': {
+                    'keyring': '/etc/ceph/dcn0.client.openstack.keyring'
+                }
+            },
+            'cluster': 'dcn0',
+            'fsid': 'a5a22d37-e01f-4fa0-a440-c72585c7487f',
+            'dashboard_enabled': False
+        }
+        with mock.patch('six.moves.builtins.open', self.mock_open_ceph_all):
+            data = export.export_ceph('dcn0', 'openstack',
+                                      config_download_dir='/foo',
+                                      mon_ips=['192.168.24.42'])
+        self.assertEqual(data, expected)
+        self.mock_open_ceph_all.assert_called_once_with(
+            '/foo/dcn0/ceph-ansible/group_vars/all.yml', 'r')

tripleoclient/tests/v1/test_overcloud_export_ceph.py

@@ -0,0 +1,66 @@
+#   Copyright 2020 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+import os
+
+import mock
+
+from osc_lib.tests import utils
+
+from tripleoclient.v1 import overcloud_export_ceph
+
+
+class TestOvercloudExportCeph(utils.TestCommand):
+
+    def setUp(self):
+        super(TestOvercloudExportCeph, self).setUp()
+
+        self.cmd = overcloud_export_ceph.ExportOvercloudCeph(self.app, None)
+        self.tripleoclient = mock.Mock()
+        self.app.client_manager.tripleoclient = self.tripleoclient
+        self.mock_open = mock.mock_open()
+
+    @mock.patch('os.path.exists')
+    @mock.patch('yaml.safe_dump')
+    @mock.patch('tripleoclient.export.export_ceph')
+    def test_export_ceph(self, mock_export_ceph,
+                         mock_safe_dump,
+                         mock_exists):
+        argslist = ['--stack', 'dcn0']
+        verifylist = [('stack', 'dcn0')]
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        mock_exists.return_value = False
+        expected = {
+            'external_cluster_mon_ips': '192.168.24.42',
+            'keys': [
+                {'name': 'client.openstack'}
+            ],
+            'ceph_conf_overrides': {
+                'client': {
+                    'keyring': '/etc/ceph/dcn0.client.openstack.keyring'
+                }
+            },
+            'cluster': 'dcn0',
+            'fsid': 'a5a22d37-e01f-4fa0-a440-c72585c7487f',
+            'dashboard_enabled': False
+        }
+        data = {}
+        data['parameter_defaults'] = {}
+        data['parameter_defaults']['CephExternalMultiConfig'] = [expected]
+        mock_export_ceph.return_value = expected
+
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            self.cmd.take_action(parsed_args)
+        path = os.path.join(os.environ.get('HOME'), 'config-download')
+        mock_export_ceph.assert_called_once_with('dcn0', 'openstack', path)
+        self.assertEqual(data, mock_safe_dump.call_args[0][0])

tripleoclient/v1/overcloud_export_ceph.py

@@ -0,0 +1,115 @@
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+from datetime import datetime
+import logging
+import os.path
+import yaml
+
+from osc_lib.i18n import _
+from osc_lib import utils
+
+from tripleoclient import command
+from tripleoclient import export
+
+
+class ExportOvercloudCeph(command.Command):
+    """Export Ceph information used as import of another stack
+
+    Export Ceph information from one or more stacks to be used
+    as input of another stack. Creates a valid YAML file with
+    the CephExternalMultiConfig parameter populated.
+    """
+
+    log = logging.getLogger(__name__ + ".ExportOvercloudCeph")
+    now = datetime.now().strftime('%Y%m%d%H%M%S')
+
+    def get_parser(self, prog_name):
+        parser = super(ExportOvercloudCeph, self).get_parser(prog_name)
+        parser.add_argument('--stack',
+                            dest='stack',
+                            metavar='<stack>',
+                            help=_('Name of the overcloud stack(s) '
+                                   'to export Ceph information from. '
+                                   'If a comma delimited list of stacks is '
+                                   'passed, Ceph information for all stacks '
+                                   'will be exported into a single file. '
+                                   '(default=Env: OVERCLOUD_STACK_NAME) '),
+                            default=utils.env('OVERCLOUD_STACK_NAME',
+                                              default='overcloud'))
+        parser.add_argument('--cephx-key-client-name', '-k',
+                            dest='cephx',
+                            metavar='<cephx>',
+                            help=_('Name of the cephx client key to export. '
+                                   '(default=openstack)'),
+                            default='openstack')
+        parser.add_argument('--output-file', '-o', metavar='<output file>',
+                            help=_('Name of the output file for the Ceph '
+                                   'data export. Defaults to '
+                                   '"ceph-export-<STACK>.yaml" if one '
+                                   'stack is provided. Defaults to '
+                                   '"ceph-export-<N>-stacks.yaml" '
+                                   'if N stacks are provided.'))
+        parser.add_argument('--force-overwrite', '-f', action='store_true',
+                            default=False,
+                            help=_('Overwrite output file if it exists.'))
+        parser.add_argument('--config-download-dir',
+                            action='store',
+                            help=_('Directory to search for config-download '
+                                   'export data. Defaults to '
+                                   '$HOME/config-download'))
+
+        return parser
+
+    def take_action(self, parsed_args):
+        self.log.debug("take_action(%s)" % parsed_args)
+
+        stacks = parsed_args.stack.split(',')
+        stack_count = len(stacks)
+        if stack_count == 1:
+            name = parsed_args.stack
+        else:
+            name = str(stack_count) + '-stacks'
+        output_file = parsed_args.output_file or \
+            'ceph-export-%s.yaml' % name
+
+        self.log.info('Running at %s with parameters %s',
+                      self.now,
+                      parsed_args)
+
+        if os.path.exists(output_file) and not parsed_args.force_overwrite:
+            raise Exception(
+                "File '%s' already exists, not exporting." % output_file)
+
+        if not parsed_args.config_download_dir:
+            config_download_dir = os.path.join(os.environ.get('HOME'),
+                                               'config-download')
+        else:
+            config_download_dir = parsed_args.config_download_dir
+
+        # extract ceph data for each stack into the cephs list
+        cephs = []
+        for stack in stacks:
+            self.log.info('Exporting Ceph data from stack %s at %s',
+                          stack, self.now)
+            cephs.append(export.export_ceph(stack,
+                                            parsed_args.cephx,
+                                            config_download_dir))
+        data = {}
+        data['parameter_defaults'] = {}
+        data['parameter_defaults']['CephExternalMultiConfig'] = cephs
+        # write the exported data
+        with open(output_file, 'w') as f:
+            yaml.safe_dump(data, f, default_flow_style=False)
+
+        print("Ceph information from %s stack(s) exported to %s." %
+              (len(cephs), output_file))