From b252d45739a9e359a005dfb911dc38d516df4675 Mon Sep 17 00:00:00 2001 From: James Slagle Date: Wed, 7 Apr 2021 20:15:46 -0400 Subject: [PATCH] Add OS::TripleO::UndercloudUpgradeEphemeralHeat A new service, OS::TripleO::Services::UndercloudUpgradeEphemeralHeat is added to the Undercloud role. The service is mapped to OS::Heat::None by default, but when environments/lifecycle/undercloud-upgrade-prepare.yaml is included, the service will be enabled and will migrate any already deployed stacks in the undercloud's Heat instance to be able to be used with the ephemeral Heat deployment option from tripleoclient. Signed-off-by: James Slagle Change-Id: If11e2fc07a1ff773f6eaf209d8b48493f0b60e85 --- .../undercloud-upgrade-ephemeral-heat.yaml | 54 ++++ .../lifecycle/undercloud-upgrade-prepare.yaml | 4 +- overcloud-resource-registry-puppet.j2.yaml | 1 + ...grade-ephemeral-heat-c838a9c61fc742a3.yaml | 8 + roles/Undercloud.yaml | 1 + roles_data_undercloud.yaml | 1 + scripts/undercloud-upgrade-ephemeral-heat.py | 256 ++++++++++++++++++ 7 files changed, 323 insertions(+), 2 deletions(-) create mode 100644 deployment/undercloud/undercloud-upgrade-ephemeral-heat.yaml create mode 100644 releasenotes/notes/undercloud-upgrade-ephemeral-heat-c838a9c61fc742a3.yaml create mode 100755 scripts/undercloud-upgrade-ephemeral-heat.py diff --git a/deployment/undercloud/undercloud-upgrade-ephemeral-heat.yaml b/deployment/undercloud/undercloud-upgrade-ephemeral-heat.yaml new file mode 100644 index 0000000000..d740fc50b7 --- /dev/null +++ b/deployment/undercloud/undercloud-upgrade-ephemeral-heat.yaml @@ -0,0 +1,54 @@ +heat_template_version: wallaby + +description: > + Upgrade an undercloud to use ephemeral Heat + +parameters: + ServiceData: + default: {} + description: Dictionary packing service data + type: json + ServiceNetMap: + default: {} + description: Mapping of service_name -> network name. Typically set + via parameter_defaults in the resource registry. This + mapping overrides those in ServiceNetMapDefaults. + type: json + RoleName: + default: '' + description: Role name on which the service is applied + type: string + RoleParameters: + default: {} + description: Parameters specific to the role + type: json + EndpointMap: + default: {} + description: Mapping of service endpoint -> protocol. Typically set + via parameter_defaults in the resource registry. + type: json + +outputs: + role_data: + description: Role data for the TripleO Undercloud Upgrade Ephemeral Heat service. + value: + service_name: undercloud_upgrade_ephemeral_heat + upgrade_tasks: + - name: Create /var/lib/tripleo-config/scripts dir + file: + path: /var/lib/tripleo-config/scripts + state: directory + recurse: true + when: + - step|int == 1 + - name: Copy undercloud-upgrade-ephemeral-heat.py to /var/lib/tripleo-config/scripts + copy: + dest: /var/lib/tripleo-config/scripts/undercloud-upgrade-ephemeral-heat.py + content: {get_file: ../../scripts/undercloud-upgrade-ephemeral-heat.py} + mode: 0755 + when: + - step|int == 1 + - name: Run undercloud-upgrade-ephemeral-heat.py + shell: /var/lib/tripleo-config/scripts/undercloud-upgrade-ephemeral-heat.py + when: + - step|int == 1 diff --git a/environments/lifecycle/undercloud-upgrade-prepare.yaml b/environments/lifecycle/undercloud-upgrade-prepare.yaml index d321a1c0e8..3710d77604 100644 --- a/environments/lifecycle/undercloud-upgrade-prepare.yaml +++ b/environments/lifecycle/undercloud-upgrade-prepare.yaml @@ -1,8 +1,8 @@ -# A Heat environment file that can be used to upgrade a non-containerized undercloud -# to a containerized undercloud. +# A Heat environment file that can be used to upgrade an undercloud resource_registry: OS::TripleO::Services::UndercloudUpgrade: ../../deployment/undercloud/undercloud-upgrade.yaml + OS::TripleO::Services::UndercloudUpgradeEphemeralHeat: ../../deployment/undercloud/undercloud-upgrade-ephemeral-heat.yaml parameter_defaults: UndercloudUpgrade: true diff --git a/overcloud-resource-registry-puppet.j2.yaml b/overcloud-resource-registry-puppet.j2.yaml index 3ef34fcbff..569242718f 100644 --- a/overcloud-resource-registry-puppet.j2.yaml +++ b/overcloud-resource-registry-puppet.j2.yaml @@ -237,6 +237,7 @@ resource_registry: OS::TripleO::Services::MasqueradeNetworks: OS::Heat::None OS::TripleO::Services::TripleoValidations: OS::Heat::None OS::TripleO::Services::UndercloudUpgrade: OS::Heat::None + OS::TripleO::Services::UndercloudUpgradeEphemeralHeat: OS::Heat::None OS::TripleO::Services::Collectd: OS::Heat::None OS::TripleO::Services::ManilaApi: OS::Heat::None OS::TripleO::Services::ManilaScheduler: OS::Heat::None diff --git a/releasenotes/notes/undercloud-upgrade-ephemeral-heat-c838a9c61fc742a3.yaml b/releasenotes/notes/undercloud-upgrade-ephemeral-heat-c838a9c61fc742a3.yaml new file mode 100644 index 0000000000..15bc1753de --- /dev/null +++ b/releasenotes/notes/undercloud-upgrade-ephemeral-heat-c838a9c61fc742a3.yaml @@ -0,0 +1,8 @@ +--- +features: + - A new service, OS::TripleO::Services::UndercloudUpgradeEphemeralHeat is + added to the Undercloud role. The service is mapped to OS::Heat::None by + default, but when environments/lifecycle/undercloud-upgrade-prepare.yaml is + included, the service will be enabled and will migrate any already deployed + stacks in the undercloud's Heat instance to be able to be used with the + ephemeral Heat deployment option from tripleoclient. diff --git a/roles/Undercloud.yaml b/roles/Undercloud.yaml index 4ee43023e3..e4291e3a4a 100644 --- a/roles/Undercloud.yaml +++ b/roles/Undercloud.yaml @@ -98,5 +98,6 @@ - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::Tuned - OS::TripleO::Services::UndercloudUpgrade + - OS::TripleO::Services::UndercloudUpgradeEphemeralHeat - OS::TripleO::Services::TripleoValidations - OS::TripleO::Services::Zaqar diff --git a/roles_data_undercloud.yaml b/roles_data_undercloud.yaml index 86d9bac371..2973b26d1b 100644 --- a/roles_data_undercloud.yaml +++ b/roles_data_undercloud.yaml @@ -101,5 +101,6 @@ - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::Tuned - OS::TripleO::Services::UndercloudUpgrade + - OS::TripleO::Services::UndercloudUpgradeEphemeralHeat - OS::TripleO::Services::TripleoValidations - OS::TripleO::Services::Zaqar diff --git a/scripts/undercloud-upgrade-ephemeral-heat.py b/scripts/undercloud-upgrade-ephemeral-heat.py new file mode 100755 index 0000000000..b112e276d0 --- /dev/null +++ b/scripts/undercloud-upgrade-ephemeral-heat.py @@ -0,0 +1,256 @@ +#!/usr/libexec/platform-python +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Migrate an undercloud's stack data to use ephemeral Heat. Queries for +existing stacks and exports necessary data from the stack to the default +consistent working directory before backing up and dropping the heat database. +""" + +import argparse +import logging +import os +import subprocess +import tarfile +import time +import yaml + +from heatclient.client import Client +import keystoneauth1 +import openstack +from tripleo_common.utils import plan as plan_utils + + +LOG = logging.getLogger('undercloud') + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Upgrade an undercloud for ephemeral Heat.") + + parser.add_argument( + '--cloud', '-c', + default='undercloud', + help='The name of the cloud used for the OpenStack connection.') + parser.add_argument( + '--stack', '-s', + action='append', + help='The stack(s) to migrate to using ephemeral Heat. Can be ' + 'specified multiple times. If not specified, all stacks ' + 'will be migrated') + parser.add_argument( + '--working-dir', '-w', + help='Directory to use for saving stack state. ' + 'Defaults to ~/overcloud-deploy/') + + return parser.parse_args() + + +def database_exists(): + """Check if the heat database exists. + + :return: True if the heat database exists, otherwise False + :rtype: bool + """ + output = subprocess.check_output([ + 'sudo', 'podman', 'exec', '-u', 'root', 'mysql', + 'mysql', '-e', 'show databases like "heat"' + ]) + return 'heat' in str(output) + + +def backup_db(backup_dir): + """Backup the heat database to the specified directory + + :param backup_dir: The directory to store the backup + :type backup_dir: str + :return: Database tarfile backup path + :rtype: str + """ + heat_dir = os.path.join(backup_dir, 'heat-launcher') + if not os.path.isdir(heat_dir): + os.makedirs(heat_dir) + db_path = os.path.join(heat_dir, 'heat-db.sql') + LOG.info("Backing up heat database to {}".format(db_path)) + with open(db_path, 'w') as out: + subprocess.run([ + 'sudo', 'podman', 'exec', '-u', 'root', + 'mysql', 'mysqldump', 'heat'], stdout=out, + check=True) + os.chmod(db_path, 0o600) + + tf_name = '{}-{}.tar.bzip2'.format(db_path, time.time()) + tf = tarfile.open(tf_name, 'w:bz2') + tf.add(db_path, os.path.basename(db_path)) + tf.close() + LOG.info("Created tarfile {}".format(tf_name)) + + return tf_name + + +def _decode(encoded): + """Decode a string into utf-8 + + :param encoded: Encoded string + :type encoded: string + :return: Decoded string + :rtype: string + """ + if not encoded: + return "" + decoded = encoded.decode('utf-8') + if decoded.endswith('\n'): + decoded = decoded[:-1] + return decoded + + +def _get_ctlplane_vip(): + """Get the configured ctlplane VIP + + :return: ctlplane VIP + :rtype: string + """ + return _decode(subprocess.check_output( + ['sudo', 'hiera', 'controller_virtual_ip'])) + + +def _get_ctlplane_ip(): + """Get the configured ctlplane IP + + :return: ctlplane IP + :rtype: string + """ + return _decode(subprocess.check_output( + ['sudo', 'hiera', 'ctlplane'])) + + +def drop_db(): + """Drop the heat database and heat users + + :return: None + :rtype: None + """ + LOG.info("Dropping Heat database") + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-u', 'root', + 'mysql', 'mysql', 'heat', '-e', + 'drop database heat']) + LOG.info("Dropping Heat users") + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-u', 'root', + 'mysql', 'mysql', '-e', + 'drop user \'heat\'@\'{}\''.format(_get_ctlplane_ip())]) + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-u', 'root', + 'mysql', 'mysql', '-e', + 'drop user \'heat\'@\'{}\''.format(_get_ctlplane_vip())]) + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-u', 'root', + 'mysql', 'mysql', '-e', + 'drop user \'heat\'@\'%\'']) + + +def export_passwords(heat, stack, stack_dir): + """Export passwords from an existing stack and write them in Heat + environment file format to the specified directory. + + :param cloud: Heat client + :type cloud: heatclient.client.Client + :param stack: Stack name to query for passwords + :type stack: str + :param stack_dir: Directory to save the generated Heat environment + containing the password values. + :type stack_dir: str + :return: None + :rtype: None + """ + passwords_path = os.path.join( + stack_dir, "tripleo-{}-passwords.yaml".format(stack)) + LOG.info("Exporting passwords for stack %s to %s" + % (stack, passwords_path)) + passwords = plan_utils.generate_passwords(heat=heat, container=stack) + password_params = dict(parameter_defaults=passwords) + with open(passwords_path, 'w') as f: + f.write(yaml.safe_dump(password_params)) + os.chmod(passwords_path, 0o600) + + +def main(): + logging.basicConfig() + LOG.setLevel(logging.INFO) + args = parse_args() + + sudo_user = os.environ.get('SUDO_USER') + + if not args.working_dir: + if sudo_user: + user_home = '~{}'.format(sudo_user) + else: + user_home = '~' + + working_dir = os.path.join( + os.path.expanduser(user_home), + 'overcloud-deploy') + else: + working_dir = args.working_dir + if not os.path.isdir(working_dir): + os.makedirs(working_dir) + + try: + conn = openstack.connection.from_config(cloud=args.cloud) + heat = conn.orchestration + _heatclient = Client('1', endpoint=conn.endpoint_for('orchestration'), + token=conn.auth_token) + except keystoneauth1.exceptions.catalog.EndpointNotFound: + LOG.warning("No Heat endpoint found, won't migrate any " + "existing stack data.") + return + + try: + stacks = args.stack or [s.name for s in heat.stacks()] + except openstack.exceptions.HttpException: + LOG.warning("No connection to Heat available, won't migrate any " + "existing stack data.") + stacks = [] + + if database_exists(): + backup_dir = os.path.join( + working_dir, + 'undercloud-upgrade-ephemeral-heat') + db_tar_path = backup_db(backup_dir) + else: + LOG.warning("No database found to backup.") + db_tar_path = None + + for stack in stacks: + stack_dir = os.path.join(working_dir, stack) + if not os.path.exists(stack_dir): + os.makedirs(stack_dir) + if db_tar_path: + # Symlink to the existing db backup + os.symlink(db_tar_path, + os.path.join(stack_dir, os.path.basename(db_tar_path))) + export_passwords(_heatclient, stack, stack_dir) + + if database_exists(): + drop_db() + + # Chown all files to original user if running under sudo + if sudo_user: + subprocess.run([ + 'chown', '-R', '{}:{}'.format(sudo_user, sudo_user), + working_dir], + check=True) + + +if __name__ == '__main__': + main()