From 975f7717029b56c7a2e6113d82e166e7c63fcba2 Mon Sep 17 00:00:00 2001 From: James Slagle Date: Mon, 4 Nov 2019 15:59:10 -0500 Subject: [PATCH] Add openstack overcloud export Add a new command to export the data from a control plane stack for multi-stack. The created export file can be used as input into a compute stack for the multi-stack feature. Also refactors the overcloud cell export command to use the same common code. Closes-Bug: #1850636 Change-Id: I6e145d7a54dcd306c4633ebd4d7471a19a967a86 --- ...ack-overcloud-export-293c8f0f6ab13e91.yaml | 5 + setup.cfg | 1 + tripleoclient/constants.py | 4 + tripleoclient/export.py | 122 +++++++++++++++ tripleoclient/tests/test_export.py | 139 ++++++++++++++++++ .../tests/v1/test_overcloud_export.py | 132 +++++++++++++++++ tripleoclient/v1/overcloud_cell.py | 94 ++---------- tripleoclient/v1/overcloud_export.py | 100 +++++++++++++ 8 files changed, 513 insertions(+), 84 deletions(-) create mode 100644 releasenotes/notes/openstack-overcloud-export-293c8f0f6ab13e91.yaml create mode 100644 tripleoclient/export.py create mode 100644 tripleoclient/tests/test_export.py create mode 100644 tripleoclient/tests/v1/test_overcloud_export.py create mode 100644 tripleoclient/v1/overcloud_export.py diff --git a/releasenotes/notes/openstack-overcloud-export-293c8f0f6ab13e91.yaml b/releasenotes/notes/openstack-overcloud-export-293c8f0f6ab13e91.yaml new file mode 100644 index 000000000..b48264977 --- /dev/null +++ b/releasenotes/notes/openstack-overcloud-export-293c8f0f6ab13e91.yaml @@ -0,0 +1,5 @@ +--- +features: + - A new command "openstack overcloud export" is added. The command is used to + export the data from a control stack for use in a compute stack for the + multi-stack feature. diff --git a/setup.cfg b/setup.cfg index 255ab2902..74bff8eda 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,6 +62,7 @@ openstack.tripleoclient.v1 = overcloud_delete = tripleoclient.v1.overcloud_delete:DeleteOvercloud overcloud_credentials = tripleoclient.v1.overcloud_credentials:OvercloudCredentials overcloud_deploy = tripleoclient.v1.overcloud_deploy:DeployOvercloud + overcloud_export = tripleoclient.v1.overcloud_export:ExportOvercloud overcloud_failures = tripleoclient.v1.overcloud_deploy:GetDeploymentFailures overcloud_status = tripleoclient.v1.overcloud_deploy:GetDeploymentStatus overcloud_image_build = tripleoclient.v1.overcloud_image:BuildOvercloudImage diff --git a/tripleoclient/constants.py b/tripleoclient/constants.py index 924538712..b5b6ef417 100644 --- a/tripleoclient/constants.py +++ b/tripleoclient/constants.py @@ -135,3 +135,7 @@ CLOUDS_YAML_DIR = os.path.join('.config', 'openstack') UNUSED_PARAMETER_EXCLUDES_RE = ['^(Docker|Container).*Image$', '^SwiftFetchDir(Get|Put)Tempurl$', '^PythonInterpreter$'] + +EXPORT_PASSWORD_EXCLUDE_PATTERNS = [ + 'ceph.*' +] diff --git a/tripleoclient/export.py b/tripleoclient/export.py new file mode 100644 index 000000000..181e9a2e7 --- /dev/null +++ b/tripleoclient/export.py @@ -0,0 +1,122 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +import json +import logging +import os +import re +import sys +import yaml + +from osc_lib.i18n import _ + +from tripleoclient import constants +from tripleoclient import utils as oooutils + + +LOG = logging.getLogger(__name__ + ".utils") + + +def export_passwords(swift, stack, excludes=True): + # Export the passwords from swift + obj = 'plan-environment.yaml' + container = stack + try: + resp_headers, content = swift.get_object(container, obj) + except Exception as e: + LOG.error("An error happened while exporting the password " + "file from swift: %s", str(e)) + sys.exit(1) + + data = yaml.load(content)["passwords"] + if excludes: + excluded_passwords = [] + for k in data: + for pattern in constants.EXPORT_PASSWORD_EXCLUDE_PATTERNS: + if re.match(pattern, k, re.I): + excluded_passwords.append(k) + [data.pop(e) for e in excluded_passwords] + return data + + +def export_stack(heat, stack, should_filter=False, + config_download_dir='/var/lib/mistral/overcloud'): + + # data to export + # parameter: Parameter to be exported + # file: IF file specified it is taken as source instead of heat + # output.File is relative to /stack. + # filter: in case only specific settings should be + # exported from parameter data. + export_data = { + "EndpointMap": { + "parameter": "EndpointMapOverride", + }, + "HostsEntry": { + "parameter": "ExtraHostFileEntries", + }, + "GlobalConfig": { + "parameter": "GlobalConfigExtraMapData", + }, + "AllNodesConfig": { + "file": "group_vars/overcloud.json", + "parameter": "AllNodesExtraMapData", + "filter": ["oslo_messaging_notify_short_bootstrap_node_name", + "oslo_messaging_notify_node_names", + "oslo_messaging_rpc_node_names", + "memcached_node_ips", + "ovn_dbs_vip", + "redis_vip"]}, + } + + data = {} + heat_stack = oooutils.get_stack(heat, stack) + + for export_key, export_param in export_data.items(): + param = export_param["parameter"] + if "file" in export_param: + # get file data + file = os.path.join(config_download_dir, + export_param["file"]) + with open(file, 'r') as ff: + try: + export_data = json.load(ff) + except Exception as e: + LOG.error( + _('Could not read file %s') % file) + LOG.error(e) + + else: + # get stack data + export_data = oooutils.get_stack_output_item( + heat_stack, export_key) + + if export_data: + # When we export information from a cell controller stack + # we don't want to filter. + if "filter" in export_param and should_filter: + for filter_key in export_param["filter"]: + if filter_key in export_data: + element = {filter_key: export_data[filter_key]} + data.setdefault(param, {}).update(element) + else: + data[param] = export_data + + else: + raise Exception( + "No data returned to export %s from." % param) + + return data diff --git a/tripleoclient/tests/test_export.py b/tripleoclient/tests/test_export.py new file mode 100644 index 000000000..9d396db22 --- /dev/null +++ b/tripleoclient/tests/test_export.py @@ -0,0 +1,139 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +from io import StringIO +import mock +import six +from unittest import TestCase +import yaml + +from tripleoclient import export + + +class TestExport(TestCase): + def setUp(self): + self.unlink_patch = mock.patch('os.unlink') + self.addCleanup(self.unlink_patch.stop) + self.unlink_patch.start() + self.mock_log = mock.Mock('logging.getLogger') + + outputs = [ + {'output_key': 'EndpointMap', + 'output_value': dict(em_key='em_value')}, + {'output_key': 'HostsEntry', + 'output_value': 'hosts entry'}, + {'output_key': 'GlobalConfig', + 'output_value': dict(gc_key='gc_value')}, + ] + self.mock_stack = mock.Mock() + self.mock_stack.to_dict.return_value = dict(outputs=outputs) + self.mock_open = mock.mock_open(read_data='{"an_key":"an_value"}') + + @mock.patch('tripleoclient.utils.get_stack') + def test_export_stack(self, mock_get_stack): + heat = mock.Mock() + mock_get_stack.return_value = self.mock_stack + with mock.patch('six.moves.builtins.open', self.mock_open): + data = export.export_stack(heat, "overcloud") + + expected = \ + {'AllNodesExtraMapData': {u'an_key': u'an_value'}, + 'EndpointMapOverride': {'em_key': 'em_value'}, + 'ExtraHostFileEntries': 'hosts entry', + 'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}} + + self.assertEqual(expected, data) + self.mock_open.assert_called_once_with( + '/var/lib/mistral/overcloud/group_vars/overcloud.json', 'r') + + @mock.patch('tripleoclient.utils.get_stack') + def test_export_stack_should_filter(self, mock_get_stack): + heat = mock.Mock() + mock_get_stack.return_value = self.mock_stack + self.mock_open = mock.mock_open( + read_data='{"an_key":"an_value","ovn_dbs_vip":"vip"}') + with mock.patch('six.moves.builtins.open', self.mock_open): + data = export.export_stack(heat, "overcloud", should_filter=True) + + expected = \ + {'AllNodesExtraMapData': {u'ovn_dbs_vip': u'vip'}, + 'EndpointMapOverride': {'em_key': 'em_value'}, + 'ExtraHostFileEntries': 'hosts entry', + 'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}} + + self.assertEqual(expected, data) + self.mock_open.assert_called_once_with( + '/var/lib/mistral/overcloud/group_vars/overcloud.json', 'r') + + @mock.patch('tripleoclient.utils.get_stack') + def test_export_stack_cd_dir(self, mock_get_stack): + heat = mock.Mock() + mock_get_stack.return_value = self.mock_stack + with mock.patch('six.moves.builtins.open', self.mock_open): + export.export_stack(heat, "overcloud", + config_download_dir='/foo/overcloud') + self.mock_open.assert_called_once_with( + '/foo/overcloud/group_vars/overcloud.json', 'r') + + @mock.patch('tripleoclient.utils.get_stack') + def test_export_stack_stack_name(self, mock_get_stack): + heat = mock.Mock() + mock_get_stack.return_value = self.mock_stack + with mock.patch('six.moves.builtins.open', self.mock_open): + export.export_stack(heat, "control") + mock_get_stack.assert_called_once_with(heat, 'control') + + def test_export_passwords(self): + swift = mock.Mock() + mock_passwords = { + 'passwords': { + 'a': 'A', + 'b': 'B' + } + } + sio = StringIO() + sio.write(six.text_type(yaml.dump(mock_passwords))) + sio.seek(0) + swift.get_object.return_value = ("", sio) + data = export.export_passwords(swift, 'overcloud') + + swift.get_object.assert_called_once_with( + 'overcloud', 'plan-environment.yaml') + + self.assertEqual(mock_passwords['passwords'], data) + + def test_export_passwords_excludes(self): + swift = mock.Mock() + mock_passwords = { + 'passwords': { + 'a': 'A', + 'b': 'B', + 'Cephkey': 'cephkey', + 'cephkey': 'cephkey', + 'CEPH': 'cephkey' + } + } + sio = StringIO() + sio.write(six.text_type(yaml.dump(mock_passwords))) + sio.seek(0) + swift.get_object.return_value = ("", sio) + data = export.export_passwords(swift, 'overcloud') + + mock_passwords['passwords'].pop('Cephkey') + mock_passwords['passwords'].pop('cephkey') + mock_passwords['passwords'].pop('CEPH') + + self.assertEqual(mock_passwords['passwords'], data) diff --git a/tripleoclient/tests/v1/test_overcloud_export.py b/tripleoclient/tests/v1/test_overcloud_export.py new file mode 100644 index 000000000..54357971b --- /dev/null +++ b/tripleoclient/tests/v1/test_overcloud_export.py @@ -0,0 +1,132 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from osc_lib.tests import utils + +from tripleoclient.v1 import overcloud_export + + +class TestOvercloudExport(utils.TestCommand): + + def setUp(self): + super(TestOvercloudExport, self).setUp() + + self.cmd = overcloud_export.ExportOvercloud(self.app, None) + self.app.client_manager.orchestration = mock.Mock() + self.tripleoclient = mock.Mock() + self.app.client_manager.tripleoclient = self.tripleoclient + self.app.client_manager.tripleoclient.object_store = mock.Mock() + self.mock_open = mock.mock_open() + + @mock.patch('os.path.exists') + @mock.patch('yaml.safe_dump') + @mock.patch('tripleoclient.export.export_stack') + @mock.patch('tripleoclient.export.export_passwords') + def test_export(self, mock_export_passwords, + mock_export_stack, + mock_safe_dump, mock_exists): + argslist = [] + verifylist = [] + parsed_args = self.check_parser(self.cmd, argslist, verifylist) + mock_exists.return_value = False + mock_export_passwords.return_value = {'key': 'value'} + mock_export_stack.return_value = {'key0': 'value0'} + with mock.patch('six.moves.builtins.open', self.mock_open): + self.cmd.take_action(parsed_args) + mock_export_passwords.assert_called_once_with( + self.app.client_manager.tripleoclient.object_store, + 'overcloud', True) + mock_export_stack.assert_called_once_with( + self.app.client_manager.orchestration, + 'overcloud', + False, + '/var/lib/mistral/overcloud') + self.assertEqual( + {'parameter_defaults': {'key': 'value', + 'key0': 'value0'}}, + mock_safe_dump.call_args[0][0]) + + @mock.patch('os.path.exists') + @mock.patch('yaml.safe_dump') + @mock.patch('tripleoclient.export.export_stack') + @mock.patch('tripleoclient.export.export_passwords') + def test_export_stack_name(self, mock_export_passwords, + mock_export_stack, + mock_safe_dump, mock_exists): + argslist = ['--stack', 'foo'] + verifylist = [('stack', 'foo')] + parsed_args = self.check_parser(self.cmd, argslist, verifylist) + mock_exists.return_value = False + with mock.patch('six.moves.builtins.open', self.mock_open): + self.cmd.take_action(parsed_args) + mock_export_passwords.assert_called_once_with( + self.app.client_manager.tripleoclient.object_store, + 'foo', True) + mock_export_stack.assert_called_once_with( + self.app.client_manager.orchestration, + 'foo', + False, + '/var/lib/mistral/foo') + + @mock.patch('os.path.exists') + @mock.patch('yaml.safe_dump') + @mock.patch('tripleoclient.export.export_stack') + @mock.patch('tripleoclient.export.export_passwords') + def test_export_stack_name_and_dir(self, mock_export_passwords, + mock_export_stack, + mock_safe_dump, mock_exists): + argslist = ['--stack', 'foo', + '--config-download-dir', '/tmp/bar'] + verifylist = [('stack', 'foo'), + ('config_download_dir', '/tmp/bar')] + parsed_args = self.check_parser(self.cmd, argslist, verifylist) + mock_exists.return_value = False + with mock.patch('six.moves.builtins.open', self.mock_open): + self.cmd.take_action(parsed_args) + mock_export_passwords.assert_called_once_with( + self.app.client_manager.tripleoclient.object_store, + 'foo', True) + mock_export_stack.assert_called_once_with( + self.app.client_manager.orchestration, + 'foo', + False, + '/tmp/bar') + + @mock.patch('os.path.exists') + @mock.patch('yaml.safe_dump') + @mock.patch('tripleoclient.export.export_stack') + @mock.patch('tripleoclient.export.export_passwords') + def test_export_no_excludes(self, mock_export_passwords, + mock_export_stack, + mock_safe_dump, mock_exists): + argslist = ['--stack', 'foo', + '--config-download-dir', '/tmp/bar', + '--no-password-excludes'] + verifylist = [('stack', 'foo'), + ('config_download_dir', '/tmp/bar'), + ('no_password_excludes', True)] + parsed_args = self.check_parser(self.cmd, argslist, verifylist) + mock_exists.return_value = False + with mock.patch('six.moves.builtins.open', self.mock_open): + self.cmd.take_action(parsed_args) + mock_export_passwords.assert_called_once_with( + self.app.client_manager.tripleoclient.object_store, + 'foo', False) + mock_export_stack.assert_called_once_with( + self.app.client_manager.orchestration, + 'foo', + False, + '/tmp/bar') diff --git a/tripleoclient/v1/overcloud_cell.py b/tripleoclient/v1/overcloud_cell.py index 12e44f362..1d7799af1 100644 --- a/tripleoclient/v1/overcloud_cell.py +++ b/tripleoclient/v1/overcloud_cell.py @@ -13,10 +13,8 @@ from __future__ import print_function from datetime import datetime -import json import logging import os.path -import sys import yaml from osc_lib.i18n import _ @@ -24,7 +22,8 @@ from osc_lib import utils from tripleoclient import command from tripleoclient import exceptions -from tripleoclient import utils as oooutils +from tripleoclient import export + MISTRAL_VAR = os.environ.get('MISTRAL_VAR', "/var/lib/mistral/") @@ -83,95 +82,22 @@ class ExportCell(command.Command): clients = self.app.client_manager swift_client = clients.tripleoclient.object_store - # data to export - # parameter: Parameter to be exported - # file: IF file specified it is taken as source instead of heat - # output.File is relative to MISTRAL_VAR/stack_to_export. - # filter: in case only specific settings should be - # exported from parameter data. - export_data = { - "EndpointMap": { - "parameter": "EndpointMapOverride", - }, - "HostsEntry": { - "parameter": "ExtraHostFileEntries", - }, - "GlobalConfig": { - "parameter": "GlobalConfigExtraMapData", - }, - "AllNodesConfig": { - "file": "/group_vars/overcloud.json", - "parameter": "GlobalConfigExtraMapData", - "filter": ["oslo_messaging_notify_short_bootstrap_node_name", - "oslo_messaging_notify_node_names", - "oslo_messaging_rpc_node_names", - "memcached_node_ips", - "ovn_dbs_vip", - "redis_vip"]}, - } - - # export the data from swift and heat - data_real = {} - - # Export the passwords from swift - obj = 'plan-environment.yaml' - container = control_plane_stack - try: - resp_headers, content = swift_client.get_object(container, obj) - except Exception as e: - self.log.error("An error happened while exporting the password " - "file from swift: %s", str(e)) - sys.exit(1) - - data_real = {'parameter_defaults': yaml.load(content)["passwords"]} + data = export.export_passwords(swift_client, control_plane_stack) stack_to_export = control_plane_stack + should_filter = True if cell_stack: stack_to_export = cell_stack + should_filter = False - stack = oooutils.get_stack(clients.orchestration, stack_to_export) - - for export_key, export_param in export_data.items(): - data = None - if "file" in export_param: - # get stack data - file = MISTRAL_VAR + stack_to_export + export_param["file"] - with open(file, 'r') as ff: - try: - data = json.load(ff) - except Exception: - self.log.error( - _('Could not read file %s') % file) - else: - # get stack data - data = oooutils.get_stack_output_item(stack, - export_key) - - param = export_param["parameter"] - if data: - # do we just want a subset of entries? - # When we export information from a cell controller stack - # we don't want to filter. - if "filter" in export_param and not cell_stack: - for x in export_param["filter"]: - element = {x: data[x]} - if param not in data_real["parameter_defaults"]: - data_real["parameter_defaults"][param] = element - else: - data_real["parameter_defaults"][param].update( - element) - else: - if param not in data_real["parameter_defaults"]: - data_real["parameter_defaults"][param] = data - else: - data_real["parameter_defaults"][param].update(data) - else: - raise exceptions.CellExportError( - "No data returned to export %s from." % param) + config_download_dir = os.path.join(MISTRAL_VAR, stack_to_export) + data['parameter_defaults'].update(export.export_stack( + clients.orchestration, stack_to_export, should_filter, + config_download_dir)) # write the exported data with open(output_file, 'w') as f: - yaml.safe_dump(data_real, f, default_flow_style=False) + yaml.safe_dump(data, f, default_flow_style=False) print("Cell input information exported to %s." % output_file) diff --git a/tripleoclient/v1/overcloud_export.py b/tripleoclient/v1/overcloud_export.py new file mode 100644 index 000000000..e1448552f --- /dev/null +++ b/tripleoclient/v1/overcloud_export.py @@ -0,0 +1,100 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +from datetime import datetime +import logging +import os.path +import yaml + +from osc_lib.i18n import _ +from osc_lib import utils + +from tripleoclient import command +from tripleoclient import export + + +class ExportOvercloud(command.Command): + """Export stack information used as import of another stack""" + + log = logging.getLogger(__name__ + ".ExportOvercloud") + now = datetime.now().strftime('%Y%m%d%H%M%S') + + def get_parser(self, prog_name): + parser = super(ExportOvercloud, self).get_parser(prog_name) + parser.add_argument('--stack', + dest='stack', + metavar='', + help=_('Name of the environment main Heat stack ' + 'to export information from. ' + '(default=Env: OVERCLOUD_STACK_NAME)'), + default=utils.env('OVERCLOUD_STACK_NAME', + default='overcloud')) + parser.add_argument('--output-file', '-o', metavar='', + help=_('Name of the output file for the stack ' + 'data export. It will default to ' + '".yaml"')) + parser.add_argument('--force-overwrite', '-f', action='store_true', + default=False, + help=_('Overwrite output file if it exists.')) + parser.add_argument('--config-download-dir', + action='store', + help=_('Directory to search for config-download ' + 'export data. Defaults to ' + '/var/lib/mistral/')) + parser.add_argument('--no-password-excludes', + action='store_true', + dest='no_password_excludes', + help=_('Don''t exclude certain passwords from ' + 'the password export. Defaults to False ' + 'in that some passwords will be excluded ' + 'that are not typically necessary.')) + + return parser + + def take_action(self, parsed_args): + self.log.debug("take_action(%s)" % parsed_args) + + stack = parsed_args.stack + output_file = parsed_args.output_file or \ + '%s-export.yaml' % stack + + self.log.info('Running at %s with parameters %s', + self.now, + parsed_args) + + if os.path.exists(output_file) and not parsed_args.force_overwrite: + raise Exception( + "File '%s' already exists, not exporting." % output_file) + + if not parsed_args.config_download_dir: + config_download_dir = os.path.join('/var/lib/mistral', + parsed_args.stack) + else: + config_download_dir = parsed_args.config_download_dir + + # prepare clients to access the environment + clients = self.app.client_manager + swift_client = clients.tripleoclient.object_store + + data = export.export_passwords(swift_client, stack, + not parsed_args.no_password_excludes) + data.update(export.export_stack( + clients.orchestration, stack, False, config_download_dir)) + data = dict(parameter_defaults=data) + + # write the exported data + with open(output_file, 'w') as f: + yaml.safe_dump(data, f, default_flow_style=False) + + print("Stack information exported to %s." % output_file)