Add openstack overcloud export

Add a new command to export the data from a control plane stack for
multi-stack. The created export file can be used as input into a compute
stack for the multi-stack feature.

Also refactors the overcloud cell export command to use the same common
code.

Closes-Bug: #1850636
Change-Id: I6e145d7a54dcd306c4633ebd4d7471a19a967a86
This commit is contained in:
James Slagle 2019-11-04 15:59:10 -05:00
parent c6616f8d96
commit 975f771702
8 changed files with 513 additions and 84 deletions

View File

@ -0,0 +1,5 @@
---
features:
- A new command "openstack overcloud export" is added. The command is used to
export the data from a control stack for use in a compute stack for the
multi-stack feature.

View File

@ -62,6 +62,7 @@ openstack.tripleoclient.v1 =
overcloud_delete = tripleoclient.v1.overcloud_delete:DeleteOvercloud
overcloud_credentials = tripleoclient.v1.overcloud_credentials:OvercloudCredentials
overcloud_deploy = tripleoclient.v1.overcloud_deploy:DeployOvercloud
overcloud_export = tripleoclient.v1.overcloud_export:ExportOvercloud
overcloud_failures = tripleoclient.v1.overcloud_deploy:GetDeploymentFailures
overcloud_status = tripleoclient.v1.overcloud_deploy:GetDeploymentStatus
overcloud_image_build = tripleoclient.v1.overcloud_image:BuildOvercloudImage

View File

@ -135,3 +135,7 @@ CLOUDS_YAML_DIR = os.path.join('.config', 'openstack')
UNUSED_PARAMETER_EXCLUDES_RE = ['^(Docker|Container).*Image$',
'^SwiftFetchDir(Get|Put)Tempurl$',
'^PythonInterpreter$']
EXPORT_PASSWORD_EXCLUDE_PATTERNS = [
'ceph.*'
]

122
tripleoclient/export.py Normal file
View File

@ -0,0 +1,122 @@
# Copyright 2019 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import json
import logging
import os
import re
import sys
import yaml
from osc_lib.i18n import _
from tripleoclient import constants
from tripleoclient import utils as oooutils
LOG = logging.getLogger(__name__ + ".utils")
def export_passwords(swift, stack, excludes=True):
# Export the passwords from swift
obj = 'plan-environment.yaml'
container = stack
try:
resp_headers, content = swift.get_object(container, obj)
except Exception as e:
LOG.error("An error happened while exporting the password "
"file from swift: %s", str(e))
sys.exit(1)
data = yaml.load(content)["passwords"]
if excludes:
excluded_passwords = []
for k in data:
for pattern in constants.EXPORT_PASSWORD_EXCLUDE_PATTERNS:
if re.match(pattern, k, re.I):
excluded_passwords.append(k)
[data.pop(e) for e in excluded_passwords]
return data
def export_stack(heat, stack, should_filter=False,
config_download_dir='/var/lib/mistral/overcloud'):
# data to export
# parameter: Parameter to be exported
# file: IF file specified it is taken as source instead of heat
# output.File is relative to <config-download-dir>/stack.
# filter: in case only specific settings should be
# exported from parameter data.
export_data = {
"EndpointMap": {
"parameter": "EndpointMapOverride",
},
"HostsEntry": {
"parameter": "ExtraHostFileEntries",
},
"GlobalConfig": {
"parameter": "GlobalConfigExtraMapData",
},
"AllNodesConfig": {
"file": "group_vars/overcloud.json",
"parameter": "AllNodesExtraMapData",
"filter": ["oslo_messaging_notify_short_bootstrap_node_name",
"oslo_messaging_notify_node_names",
"oslo_messaging_rpc_node_names",
"memcached_node_ips",
"ovn_dbs_vip",
"redis_vip"]},
}
data = {}
heat_stack = oooutils.get_stack(heat, stack)
for export_key, export_param in export_data.items():
param = export_param["parameter"]
if "file" in export_param:
# get file data
file = os.path.join(config_download_dir,
export_param["file"])
with open(file, 'r') as ff:
try:
export_data = json.load(ff)
except Exception as e:
LOG.error(
_('Could not read file %s') % file)
LOG.error(e)
else:
# get stack data
export_data = oooutils.get_stack_output_item(
heat_stack, export_key)
if export_data:
# When we export information from a cell controller stack
# we don't want to filter.
if "filter" in export_param and should_filter:
for filter_key in export_param["filter"]:
if filter_key in export_data:
element = {filter_key: export_data[filter_key]}
data.setdefault(param, {}).update(element)
else:
data[param] = export_data
else:
raise Exception(
"No data returned to export %s from." % param)
return data

View File

@ -0,0 +1,139 @@
# Copyright 2019 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from io import StringIO
import mock
import six
from unittest import TestCase
import yaml
from tripleoclient import export
class TestExport(TestCase):
def setUp(self):
self.unlink_patch = mock.patch('os.unlink')
self.addCleanup(self.unlink_patch.stop)
self.unlink_patch.start()
self.mock_log = mock.Mock('logging.getLogger')
outputs = [
{'output_key': 'EndpointMap',
'output_value': dict(em_key='em_value')},
{'output_key': 'HostsEntry',
'output_value': 'hosts entry'},
{'output_key': 'GlobalConfig',
'output_value': dict(gc_key='gc_value')},
]
self.mock_stack = mock.Mock()
self.mock_stack.to_dict.return_value = dict(outputs=outputs)
self.mock_open = mock.mock_open(read_data='{"an_key":"an_value"}')
@mock.patch('tripleoclient.utils.get_stack')
def test_export_stack(self, mock_get_stack):
heat = mock.Mock()
mock_get_stack.return_value = self.mock_stack
with mock.patch('six.moves.builtins.open', self.mock_open):
data = export.export_stack(heat, "overcloud")
expected = \
{'AllNodesExtraMapData': {u'an_key': u'an_value'},
'EndpointMapOverride': {'em_key': 'em_value'},
'ExtraHostFileEntries': 'hosts entry',
'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}}
self.assertEqual(expected, data)
self.mock_open.assert_called_once_with(
'/var/lib/mistral/overcloud/group_vars/overcloud.json', 'r')
@mock.patch('tripleoclient.utils.get_stack')
def test_export_stack_should_filter(self, mock_get_stack):
heat = mock.Mock()
mock_get_stack.return_value = self.mock_stack
self.mock_open = mock.mock_open(
read_data='{"an_key":"an_value","ovn_dbs_vip":"vip"}')
with mock.patch('six.moves.builtins.open', self.mock_open):
data = export.export_stack(heat, "overcloud", should_filter=True)
expected = \
{'AllNodesExtraMapData': {u'ovn_dbs_vip': u'vip'},
'EndpointMapOverride': {'em_key': 'em_value'},
'ExtraHostFileEntries': 'hosts entry',
'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}}
self.assertEqual(expected, data)
self.mock_open.assert_called_once_with(
'/var/lib/mistral/overcloud/group_vars/overcloud.json', 'r')
@mock.patch('tripleoclient.utils.get_stack')
def test_export_stack_cd_dir(self, mock_get_stack):
heat = mock.Mock()
mock_get_stack.return_value = self.mock_stack
with mock.patch('six.moves.builtins.open', self.mock_open):
export.export_stack(heat, "overcloud",
config_download_dir='/foo/overcloud')
self.mock_open.assert_called_once_with(
'/foo/overcloud/group_vars/overcloud.json', 'r')
@mock.patch('tripleoclient.utils.get_stack')
def test_export_stack_stack_name(self, mock_get_stack):
heat = mock.Mock()
mock_get_stack.return_value = self.mock_stack
with mock.patch('six.moves.builtins.open', self.mock_open):
export.export_stack(heat, "control")
mock_get_stack.assert_called_once_with(heat, 'control')
def test_export_passwords(self):
swift = mock.Mock()
mock_passwords = {
'passwords': {
'a': 'A',
'b': 'B'
}
}
sio = StringIO()
sio.write(six.text_type(yaml.dump(mock_passwords)))
sio.seek(0)
swift.get_object.return_value = ("", sio)
data = export.export_passwords(swift, 'overcloud')
swift.get_object.assert_called_once_with(
'overcloud', 'plan-environment.yaml')
self.assertEqual(mock_passwords['passwords'], data)
def test_export_passwords_excludes(self):
swift = mock.Mock()
mock_passwords = {
'passwords': {
'a': 'A',
'b': 'B',
'Cephkey': 'cephkey',
'cephkey': 'cephkey',
'CEPH': 'cephkey'
}
}
sio = StringIO()
sio.write(six.text_type(yaml.dump(mock_passwords)))
sio.seek(0)
swift.get_object.return_value = ("", sio)
data = export.export_passwords(swift, 'overcloud')
mock_passwords['passwords'].pop('Cephkey')
mock_passwords['passwords'].pop('cephkey')
mock_passwords['passwords'].pop('CEPH')
self.assertEqual(mock_passwords['passwords'], data)

View File

@ -0,0 +1,132 @@
# Copyright 2019 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from osc_lib.tests import utils
from tripleoclient.v1 import overcloud_export
class TestOvercloudExport(utils.TestCommand):
def setUp(self):
super(TestOvercloudExport, self).setUp()
self.cmd = overcloud_export.ExportOvercloud(self.app, None)
self.app.client_manager.orchestration = mock.Mock()
self.tripleoclient = mock.Mock()
self.app.client_manager.tripleoclient = self.tripleoclient
self.app.client_manager.tripleoclient.object_store = mock.Mock()
self.mock_open = mock.mock_open()
@mock.patch('os.path.exists')
@mock.patch('yaml.safe_dump')
@mock.patch('tripleoclient.export.export_stack')
@mock.patch('tripleoclient.export.export_passwords')
def test_export(self, mock_export_passwords,
mock_export_stack,
mock_safe_dump, mock_exists):
argslist = []
verifylist = []
parsed_args = self.check_parser(self.cmd, argslist, verifylist)
mock_exists.return_value = False
mock_export_passwords.return_value = {'key': 'value'}
mock_export_stack.return_value = {'key0': 'value0'}
with mock.patch('six.moves.builtins.open', self.mock_open):
self.cmd.take_action(parsed_args)
mock_export_passwords.assert_called_once_with(
self.app.client_manager.tripleoclient.object_store,
'overcloud', True)
mock_export_stack.assert_called_once_with(
self.app.client_manager.orchestration,
'overcloud',
False,
'/var/lib/mistral/overcloud')
self.assertEqual(
{'parameter_defaults': {'key': 'value',
'key0': 'value0'}},
mock_safe_dump.call_args[0][0])
@mock.patch('os.path.exists')
@mock.patch('yaml.safe_dump')
@mock.patch('tripleoclient.export.export_stack')
@mock.patch('tripleoclient.export.export_passwords')
def test_export_stack_name(self, mock_export_passwords,
mock_export_stack,
mock_safe_dump, mock_exists):
argslist = ['--stack', 'foo']
verifylist = [('stack', 'foo')]
parsed_args = self.check_parser(self.cmd, argslist, verifylist)
mock_exists.return_value = False
with mock.patch('six.moves.builtins.open', self.mock_open):
self.cmd.take_action(parsed_args)
mock_export_passwords.assert_called_once_with(
self.app.client_manager.tripleoclient.object_store,
'foo', True)
mock_export_stack.assert_called_once_with(
self.app.client_manager.orchestration,
'foo',
False,
'/var/lib/mistral/foo')
@mock.patch('os.path.exists')
@mock.patch('yaml.safe_dump')
@mock.patch('tripleoclient.export.export_stack')
@mock.patch('tripleoclient.export.export_passwords')
def test_export_stack_name_and_dir(self, mock_export_passwords,
mock_export_stack,
mock_safe_dump, mock_exists):
argslist = ['--stack', 'foo',
'--config-download-dir', '/tmp/bar']
verifylist = [('stack', 'foo'),
('config_download_dir', '/tmp/bar')]
parsed_args = self.check_parser(self.cmd, argslist, verifylist)
mock_exists.return_value = False
with mock.patch('six.moves.builtins.open', self.mock_open):
self.cmd.take_action(parsed_args)
mock_export_passwords.assert_called_once_with(
self.app.client_manager.tripleoclient.object_store,
'foo', True)
mock_export_stack.assert_called_once_with(
self.app.client_manager.orchestration,
'foo',
False,
'/tmp/bar')
@mock.patch('os.path.exists')
@mock.patch('yaml.safe_dump')
@mock.patch('tripleoclient.export.export_stack')
@mock.patch('tripleoclient.export.export_passwords')
def test_export_no_excludes(self, mock_export_passwords,
mock_export_stack,
mock_safe_dump, mock_exists):
argslist = ['--stack', 'foo',
'--config-download-dir', '/tmp/bar',
'--no-password-excludes']
verifylist = [('stack', 'foo'),
('config_download_dir', '/tmp/bar'),
('no_password_excludes', True)]
parsed_args = self.check_parser(self.cmd, argslist, verifylist)
mock_exists.return_value = False
with mock.patch('six.moves.builtins.open', self.mock_open):
self.cmd.take_action(parsed_args)
mock_export_passwords.assert_called_once_with(
self.app.client_manager.tripleoclient.object_store,
'foo', False)
mock_export_stack.assert_called_once_with(
self.app.client_manager.orchestration,
'foo',
False,
'/tmp/bar')

View File

@ -13,10 +13,8 @@
from __future__ import print_function
from datetime import datetime
import json
import logging
import os.path
import sys
import yaml
from osc_lib.i18n import _
@ -24,7 +22,8 @@ from osc_lib import utils
from tripleoclient import command
from tripleoclient import exceptions
from tripleoclient import utils as oooutils
from tripleoclient import export
MISTRAL_VAR = os.environ.get('MISTRAL_VAR',
"/var/lib/mistral/")
@ -83,95 +82,22 @@ class ExportCell(command.Command):
clients = self.app.client_manager
swift_client = clients.tripleoclient.object_store
# data to export
# parameter: Parameter to be exported
# file: IF file specified it is taken as source instead of heat
# output.File is relative to MISTRAL_VAR/stack_to_export.
# filter: in case only specific settings should be
# exported from parameter data.
export_data = {
"EndpointMap": {
"parameter": "EndpointMapOverride",
},
"HostsEntry": {
"parameter": "ExtraHostFileEntries",
},
"GlobalConfig": {
"parameter": "GlobalConfigExtraMapData",
},
"AllNodesConfig": {
"file": "/group_vars/overcloud.json",
"parameter": "GlobalConfigExtraMapData",
"filter": ["oslo_messaging_notify_short_bootstrap_node_name",
"oslo_messaging_notify_node_names",
"oslo_messaging_rpc_node_names",
"memcached_node_ips",
"ovn_dbs_vip",
"redis_vip"]},
}
# export the data from swift and heat
data_real = {}
# Export the passwords from swift
obj = 'plan-environment.yaml'
container = control_plane_stack
try:
resp_headers, content = swift_client.get_object(container, obj)
except Exception as e:
self.log.error("An error happened while exporting the password "
"file from swift: %s", str(e))
sys.exit(1)
data_real = {'parameter_defaults': yaml.load(content)["passwords"]}
data = export.export_passwords(swift_client, control_plane_stack)
stack_to_export = control_plane_stack
should_filter = True
if cell_stack:
stack_to_export = cell_stack
should_filter = False
stack = oooutils.get_stack(clients.orchestration, stack_to_export)
for export_key, export_param in export_data.items():
data = None
if "file" in export_param:
# get stack data
file = MISTRAL_VAR + stack_to_export + export_param["file"]
with open(file, 'r') as ff:
try:
data = json.load(ff)
except Exception:
self.log.error(
_('Could not read file %s') % file)
else:
# get stack data
data = oooutils.get_stack_output_item(stack,
export_key)
param = export_param["parameter"]
if data:
# do we just want a subset of entries?
# When we export information from a cell controller stack
# we don't want to filter.
if "filter" in export_param and not cell_stack:
for x in export_param["filter"]:
element = {x: data[x]}
if param not in data_real["parameter_defaults"]:
data_real["parameter_defaults"][param] = element
else:
data_real["parameter_defaults"][param].update(
element)
else:
if param not in data_real["parameter_defaults"]:
data_real["parameter_defaults"][param] = data
else:
data_real["parameter_defaults"][param].update(data)
else:
raise exceptions.CellExportError(
"No data returned to export %s from." % param)
config_download_dir = os.path.join(MISTRAL_VAR, stack_to_export)
data['parameter_defaults'].update(export.export_stack(
clients.orchestration, stack_to_export, should_filter,
config_download_dir))
# write the exported data
with open(output_file, 'w') as f:
yaml.safe_dump(data_real, f, default_flow_style=False)
yaml.safe_dump(data, f, default_flow_style=False)
print("Cell input information exported to %s." % output_file)

View File

@ -0,0 +1,100 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
from datetime import datetime
import logging
import os.path
import yaml
from osc_lib.i18n import _
from osc_lib import utils
from tripleoclient import command
from tripleoclient import export
class ExportOvercloud(command.Command):
"""Export stack information used as import of another stack"""
log = logging.getLogger(__name__ + ".ExportOvercloud")
now = datetime.now().strftime('%Y%m%d%H%M%S')
def get_parser(self, prog_name):
parser = super(ExportOvercloud, self).get_parser(prog_name)
parser.add_argument('--stack',
dest='stack',
metavar='<stack>',
help=_('Name of the environment main Heat stack '
'to export information from. '
'(default=Env: OVERCLOUD_STACK_NAME)'),
default=utils.env('OVERCLOUD_STACK_NAME',
default='overcloud'))
parser.add_argument('--output-file', '-o', metavar='<output file>',
help=_('Name of the output file for the stack '
'data export. It will default to '
'"<name>.yaml"'))
parser.add_argument('--force-overwrite', '-f', action='store_true',
default=False,
help=_('Overwrite output file if it exists.'))
parser.add_argument('--config-download-dir',
action='store',
help=_('Directory to search for config-download '
'export data. Defaults to '
'/var/lib/mistral/<stack>'))
parser.add_argument('--no-password-excludes',
action='store_true',
dest='no_password_excludes',
help=_('Don''t exclude certain passwords from '
'the password export. Defaults to False '
'in that some passwords will be excluded '
'that are not typically necessary.'))
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
stack = parsed_args.stack
output_file = parsed_args.output_file or \
'%s-export.yaml' % stack
self.log.info('Running at %s with parameters %s',
self.now,
parsed_args)
if os.path.exists(output_file) and not parsed_args.force_overwrite:
raise Exception(
"File '%s' already exists, not exporting." % output_file)
if not parsed_args.config_download_dir:
config_download_dir = os.path.join('/var/lib/mistral',
parsed_args.stack)
else:
config_download_dir = parsed_args.config_download_dir
# prepare clients to access the environment
clients = self.app.client_manager
swift_client = clients.tripleoclient.object_store
data = export.export_passwords(swift_client, stack,
not parsed_args.no_password_excludes)
data.update(export.export_stack(
clients.orchestration, stack, False, config_download_dir))
data = dict(parameter_defaults=data)
# write the exported data
with open(output_file, 'w') as f:
yaml.safe_dump(data, f, default_flow_style=False)
print("Stack information exported to %s." % output_file)