Merge "Add openstack overcloud export ceph"

This commit is contained in:
Zuul 2020-09-25 00:14:04 +00:00 committed by Gerrit Code Review
commit ef397d894f
6 changed files with 306 additions and 1 deletions

View File

@ -0,0 +1,6 @@
---
features:
- A new command "openstack overcloud export ceph" is added. The command is
used to export the Ceph deployment data from one stack for use in another
stack with storage services which use that Ceph cluster when using the
multi-stack deployment feature.

View File

@ -48,6 +48,7 @@ openstack.tripleoclient.v2 =
overcloud_credentials = tripleoclient.v1.overcloud_credentials:OvercloudCredentials
overcloud_deploy = tripleoclient.v1.overcloud_deploy:DeployOvercloud
overcloud_export = tripleoclient.v1.overcloud_export:ExportOvercloud
overcloud_export_ceph = tripleoclient.v1.overcloud_export_ceph:ExportOvercloudCeph
overcloud_status = tripleoclient.v1.overcloud_deploy:GetDeploymentStatus
overcloud_image_build = tripleoclient.v1.overcloud_image:BuildOvercloudImage
overcloud_image_upload = tripleoclient.v1.overcloud_image:UploadOvercloudImage

View File

@ -42,7 +42,7 @@ def export_passwords(swift, stack, excludes=True):
"file from swift: %s", str(e))
sys.exit(1)
data = yaml.load(content)
data = yaml.safe_load(content)
# The "passwords" key in plan-environment.yaml are generated passwords,
# they are not necessarily the actual password values used during the
# deployment.
@ -141,3 +141,64 @@ def export_stack(heat, stack, should_filter=False,
"No data returned to export %s from." % param)
return data
def export_storage_ips(stack, config_download_dir=constants.DEFAULT_WORK_DIR):
inventory_file = "ceph-ansible/inventory.yml"
file = os.path.join(config_download_dir, stack, inventory_file)
with open(file, 'r') as ff:
try:
inventory_data = yaml.safe_load(ff)
except Exception as e:
LOG.error(
_('Could not read file %s') % file)
LOG.error(e)
mon_ips = []
for mon_role in inventory_data['mons']['children'].keys():
for hostname in inventory_data[mon_role]['hosts']:
ip = inventory_data[mon_role]['hosts'][hostname]['storage_ip']
mon_ips.append(ip)
return mon_ips
def export_ceph(stack, cephx,
config_download_dir=constants.DEFAULT_WORK_DIR,
mon_ips=[]):
# Return a map of ceph data for a list item in CephExternalMultiConfig
# by parsing files within the config_download_dir of a certain stack
if len(mon_ips) == 0:
mon_ips = export_storage_ips(stack, config_download_dir)
# Use ceph-ansible group_vars/all.yml to get remaining values
ceph_ansible_all = "ceph-ansible/group_vars/all.yml"
file = os.path.join(config_download_dir, stack, ceph_ansible_all)
with open(file, 'r') as ff:
try:
ceph_data = yaml.safe_load(ff)
except Exception as e:
LOG.error(
_('Could not read file %s') % file)
LOG.error(e)
for key in ceph_data['keys']:
if key['name'] == 'client.' + str(cephx):
cephx_keys = [key]
ceph_conf_overrides = {}
ceph_conf_overrides['client'] = {}
ceph_conf_overrides['client']['keyring'] = '/etc/ceph/' \
+ ceph_data['cluster'] \
+ '.client.' + cephx \
+ '.keyring'
# Combine extracted data into one map to return
data = {}
data['external_cluster_mon_ips'] = str(','.join(mon_ips))
data['keys'] = cephx_keys
data['ceph_conf_overrides'] = ceph_conf_overrides
data['cluster'] = ceph_data['cluster']
data['fsid'] = ceph_data['fsid']
data['dashboard_enabled'] = False
return data

View File

@ -42,6 +42,31 @@ class TestExport(TestCase):
self.mock_stack.to_dict.return_value = dict(outputs=outputs)
self.mock_open = mock.mock_open(read_data='{"an_key":"an_value"}')
ceph_inv = {
'DistributedComputeHCI': {
'hosts': {
'dcn0-distributedcomputehci-0': {
'storage_ip': '192.168.24.42'
}
}
},
'mons': {
'children': {
'DistributedComputeHCI': {}
}
}
}
self.mock_open_ceph_inv = mock.mock_open(read_data=str(ceph_inv))
ceph_all = {
'cluster': 'dcn0',
'fsid': 'a5a22d37-e01f-4fa0-a440-c72585c7487f',
'keys': [
{'name': 'client.openstack'}
]
}
self.mock_open_ceph_all = mock.mock_open(read_data=str(ceph_all))
@mock.patch('tripleoclient.utils.get_stack')
def test_export_stack(self, mock_get_stack):
heat = mock.Mock()
@ -151,3 +176,34 @@ class TestExport(TestCase):
mock_passwords['passwords'].pop('CephRgwKey')
self.assertEqual(mock_passwords['passwords'], data)
def test_export_storage_ips(self):
with mock.patch('six.moves.builtins.open', self.mock_open_ceph_inv):
storage_ips = export.export_storage_ips('dcn0',
config_download_dir='/foo')
self.assertEqual(storage_ips, ['192.168.24.42'])
self.mock_open_ceph_inv.assert_called_once_with(
'/foo/dcn0/ceph-ansible/inventory.yml', 'r')
def test_export_ceph(self):
expected = {
'external_cluster_mon_ips': '192.168.24.42',
'keys': [
{'name': 'client.openstack'}
],
'ceph_conf_overrides': {
'client': {
'keyring': '/etc/ceph/dcn0.client.openstack.keyring'
}
},
'cluster': 'dcn0',
'fsid': 'a5a22d37-e01f-4fa0-a440-c72585c7487f',
'dashboard_enabled': False
}
with mock.patch('six.moves.builtins.open', self.mock_open_ceph_all):
data = export.export_ceph('dcn0', 'openstack',
config_download_dir='/foo',
mon_ips=['192.168.24.42'])
self.assertEqual(data, expected)
self.mock_open_ceph_all.assert_called_once_with(
'/foo/dcn0/ceph-ansible/group_vars/all.yml', 'r')

View File

@ -0,0 +1,66 @@
# Copyright 2020 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import mock
from osc_lib.tests import utils
from tripleoclient.v1 import overcloud_export_ceph
class TestOvercloudExportCeph(utils.TestCommand):
def setUp(self):
super(TestOvercloudExportCeph, self).setUp()
self.cmd = overcloud_export_ceph.ExportOvercloudCeph(self.app, None)
self.tripleoclient = mock.Mock()
self.app.client_manager.tripleoclient = self.tripleoclient
self.mock_open = mock.mock_open()
@mock.patch('os.path.exists')
@mock.patch('yaml.safe_dump')
@mock.patch('tripleoclient.export.export_ceph')
def test_export_ceph(self, mock_export_ceph,
mock_safe_dump,
mock_exists):
argslist = ['--stack', 'dcn0']
verifylist = [('stack', 'dcn0')]
parsed_args = self.check_parser(self.cmd, argslist, verifylist)
mock_exists.return_value = False
expected = {
'external_cluster_mon_ips': '192.168.24.42',
'keys': [
{'name': 'client.openstack'}
],
'ceph_conf_overrides': {
'client': {
'keyring': '/etc/ceph/dcn0.client.openstack.keyring'
}
},
'cluster': 'dcn0',
'fsid': 'a5a22d37-e01f-4fa0-a440-c72585c7487f',
'dashboard_enabled': False
}
data = {}
data['parameter_defaults'] = {}
data['parameter_defaults']['CephExternalMultiConfig'] = [expected]
mock_export_ceph.return_value = expected
with mock.patch('six.moves.builtins.open', self.mock_open):
self.cmd.take_action(parsed_args)
path = os.path.join(os.environ.get('HOME'), 'config-download')
mock_export_ceph.assert_called_once_with('dcn0', 'openstack', path)
self.assertEqual(data, mock_safe_dump.call_args[0][0])

View File

@ -0,0 +1,115 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from datetime import datetime
import logging
import os.path
import yaml
from osc_lib.i18n import _
from osc_lib import utils
from tripleoclient import command
from tripleoclient import export
class ExportOvercloudCeph(command.Command):
"""Export Ceph information used as import of another stack
Export Ceph information from one or more stacks to be used
as input of another stack. Creates a valid YAML file with
the CephExternalMultiConfig parameter populated.
"""
log = logging.getLogger(__name__ + ".ExportOvercloudCeph")
now = datetime.now().strftime('%Y%m%d%H%M%S')
def get_parser(self, prog_name):
parser = super(ExportOvercloudCeph, self).get_parser(prog_name)
parser.add_argument('--stack',
dest='stack',
metavar='<stack>',
help=_('Name of the overcloud stack(s) '
'to export Ceph information from. '
'If a comma delimited list of stacks is '
'passed, Ceph information for all stacks '
'will be exported into a single file. '
'(default=Env: OVERCLOUD_STACK_NAME) '),
default=utils.env('OVERCLOUD_STACK_NAME',
default='overcloud'))
parser.add_argument('--cephx-key-client-name', '-k',
dest='cephx',
metavar='<cephx>',
help=_('Name of the cephx client key to export. '
'(default=openstack)'),
default='openstack')
parser.add_argument('--output-file', '-o', metavar='<output file>',
help=_('Name of the output file for the Ceph '
'data export. Defaults to '
'"ceph-export-<STACK>.yaml" if one '
'stack is provided. Defaults to '
'"ceph-export-<N>-stacks.yaml" '
'if N stacks are provided.'))
parser.add_argument('--force-overwrite', '-f', action='store_true',
default=False,
help=_('Overwrite output file if it exists.'))
parser.add_argument('--config-download-dir',
action='store',
help=_('Directory to search for config-download '
'export data. Defaults to '
'$HOME/config-download'))
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
stacks = parsed_args.stack.split(',')
stack_count = len(stacks)
if stack_count == 1:
name = parsed_args.stack
else:
name = str(stack_count) + '-stacks'
output_file = parsed_args.output_file or \
'ceph-export-%s.yaml' % name
self.log.info('Running at %s with parameters %s',
self.now,
parsed_args)
if os.path.exists(output_file) and not parsed_args.force_overwrite:
raise Exception(
"File '%s' already exists, not exporting." % output_file)
if not parsed_args.config_download_dir:
config_download_dir = os.path.join(os.environ.get('HOME'),
'config-download')
else:
config_download_dir = parsed_args.config_download_dir
# extract ceph data for each stack into the cephs list
cephs = []
for stack in stacks:
self.log.info('Exporting Ceph data from stack %s at %s',
stack, self.now)
cephs.append(export.export_ceph(stack,
parsed_args.cephx,
config_download_dir))
data = {}
data['parameter_defaults'] = {}
data['parameter_defaults']['CephExternalMultiConfig'] = cephs
# write the exported data
with open(output_file, 'w') as f:
yaml.safe_dump(data, f, default_flow_style=False)
print("Ceph information from %s stack(s) exported to %s." %
(len(cephs), output_file))