From 0f9fbfa58d10db9f78860fe22a2e2e771630da6f Mon Sep 17 00:00:00 2001 From: James Slagle Date: Mon, 8 Feb 2021 16:06:54 -0500 Subject: [PATCH] Add HeatPodLauncher Add a new subclass, HeatPodLauncher to launching an emphemeral Heat within a podman pod. The pod contains separate processes for Heat API and engine, allowing for multiple engine workers (defaults to cpu logical core count / 2). The pod makes use of the undercloud's already installed database and message bus. Change-Id: Ie8b4a5ca83284f66dceae32c6f2fc99cdc8c9ffb Signed-off-by: James Slagle --- setup.cfg | 3 + templates/ephemeral-heat/heat-pod.yaml.j2 | 105 ++++++++ templates/ephemeral-heat/heat.conf.j2 | 43 +++ tripleoclient/constants.py | 11 +- tripleoclient/heat_launcher.py | 307 ++++++++++++++++++++-- 5 files changed, 443 insertions(+), 26 deletions(-) create mode 100644 templates/ephemeral-heat/heat-pod.yaml.j2 create mode 100644 templates/ephemeral-heat/heat.conf.j2 diff --git a/setup.cfg b/setup.cfg index 5577e02e3..cc86b6862 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,9 @@ classifier = packages = tripleoclient +data_files = + share/python-tripleoclient/templates = templates/* + [entry_points] openstack.cli.extension = tripleoclient = tripleoclient.plugin diff --git a/templates/ephemeral-heat/heat-pod.yaml.j2 b/templates/ephemeral-heat/heat-pod.yaml.j2 new file mode 100644 index 000000000..a4017d707 --- /dev/null +++ b/templates/ephemeral-heat/heat-pod.yaml.j2 @@ -0,0 +1,105 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: ephemeral-heat + name: ephemeral-heat +spec: + containers: + - command: + - heat-engine + - --config-file + - /etc/heat/heat.conf + env: + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: TERM + value: xterm + - name: container + value: oci + - name: LANG + value: en_US.UTF-8 + image: {{ engine_image }} + name: engine + resources: {} + securityContext: + allowPrivilegeEscalation: true + capabilities: {} + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 0 + runAsUser: 0 + seLinuxOptions: {} + volumeMounts: + - mountPath: /var/log/heat + name: heat-log + - mountPath: /etc/heat/heat.conf + name: heat-config + readOnly: true + workingDir: / + - command: + - heat-api + - --config-file + - /etc/heat/heat.conf + ports: + - containerPort: {{ api_port }} + hostPort: {{ api_port }} + hostIP: {{ ctlplane_ip }} + env: + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: TERM + value: xterm + - name: container + value: oci + - name: LANG + value: en_US.UTF-8 + image: {{ api_image }} + name: api + resources: {} + securityContext: + allowPrivilegeEscalation: true + capabilities: {} + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 0 + runAsUser: 0 + seLinuxOptions: {} + volumeMounts: + - mountPath: /var/log/heat + name: heat-log + - mountPath: /etc/heat/heat.conf + name: heat-config + readOnly: true + - mountPath: /etc/heat/api-paste.ini + name: heat-api-paste + readOnly: true + - mountPath: /token_file.json + name: heat-token-file + readOnly: true + - mountPath: /etc/heat/noauth_policy.json + name: heat-noauth-policy + readOnly: true + workingDir: / + volumes: + - hostPath: + path: {{ heat_dir}}/log + type: Directory + name: heat-log + - hostPath: + path: {{ install_dir }}/heat.conf + type: File + name: heat-config + - hostPath: + path: {{ heat_dir }}/api-paste.ini + type: File + name: heat-api-paste + - hostPath: + path: {{ heat_dir }}/token_file.json + type: File + name: heat-token-file + - hostPath: + path: {{ policy_file }} + type: File + name: heat-noauth-policy +status: {} diff --git a/templates/ephemeral-heat/heat.conf.j2 b/templates/ephemeral-heat/heat.conf.j2 new file mode 100644 index 000000000..83152de98 --- /dev/null +++ b/templates/ephemeral-heat/heat.conf.j2 @@ -0,0 +1,43 @@ +[DEFAULT] +client_retry_limit=2 +convergence_engine = true +debug = true +default_deployment_signal_transport = HEAT_SIGNAL +deferred_auth_method = password +keystone_backend = heat.engine.clients.os.keystone.fake_keystoneclient.FakeKeystoneClient +log_dir = /var/log/heat +max_json_body_size = 8388608 +max_nested_stack_depth = 10 +max_resources_per_stack=-1 +num_engine_workers = {{ num_engine_workers }} +rpc_poll_timeout = 60 +rpc_response_timeout = 600 +transport_url={{ transport_url }} + +[oslo_messaging_notifications] +driver = noop + +[oslo_messaging_rabbit] +heatbeat_timeout_threshold=60 + +[noauth] +token_response = /token_file.json + +[heat_api] +bind_host = 0.0.0.0 +bind_port = {{ api_port }} +workers = 1 + +[database] +connection = {{ db_connection }} + +[paste_deploy] +api_paste_config = /etc/heat/api-paste.ini +flavor = noauth + +[oslo_policy] +policy_file = /etc/heat/noauth_policy.json + +[yaql] +limit_iterators=9000 +memory_quota=900000 diff --git a/tripleoclient/constants.py b/tripleoclient/constants.py index bd2219264..f51d2bc1b 100644 --- a/tripleoclient/constants.py +++ b/tripleoclient/constants.py @@ -43,6 +43,14 @@ DEFAULT_HEAT_CONTAINER = ('{}/{}/openstack-heat-all:{}'.format( DEFAULT_CONTAINER_REGISTRY, DEFAULT_CONTAINER_NAMESPACE, DEFAULT_CONTAINER_TAG)) +DEFAULT_HEAT_API_CONTAINER = ('{}/{}/openstack-heat-api:{}'.format( + DEFAULT_CONTAINER_REGISTRY, + DEFAULT_CONTAINER_NAMESPACE, + DEFAULT_CONTAINER_TAG)) +DEFAULT_HEAT_ENGINE_CONTAINER = ('{}/{}/openstack-heat-engine:{}'.format( + DEFAULT_CONTAINER_REGISTRY, + DEFAULT_CONTAINER_NAMESPACE, + DEFAULT_CONTAINER_TAG)) USER_PARAMETERS = 'user-environments/tripleoclient-parameters.yaml' @@ -98,8 +106,9 @@ VALIDATIONS_LOG_BASEDIR = '/var/log/validations' DEFAULT_WORK_DIR = os.path.join(os.environ.get('HOME', '~/'), 'config-download') -TRIPLEO_STATIC_INVENTORY = 'tripleo-ansible-inventory.yaml' +DEFAULT_TEMPLATES_DIR = "/usr/share/python-tripleoclient/templates" +TRIPLEO_STATIC_INVENTORY = 'tripleo-ansible-inventory.yaml' ANSIBLE_INVENTORY = os.path.join(DEFAULT_WORK_DIR, '{}/', TRIPLEO_STATIC_INVENTORY) ANSIBLE_VALIDATION_DIR = ( diff --git a/tripleoclient/heat_launcher.py b/tripleoclient/heat_launcher.py index da9630148..d3c98af01 100644 --- a/tripleoclient/heat_launcher.py +++ b/tripleoclient/heat_launcher.py @@ -17,14 +17,18 @@ import datetime import grp import json import logging +import multiprocessing import os import pwd import signal import subprocess import tempfile +import jinja2 from oslo_utils import timeutils +from tripleoclient import constants + log = logging.getLogger(__name__) NEXT_DAY = (timeutils.utcnow() + datetime.timedelta(days=2)).isoformat() @@ -111,11 +115,21 @@ class HeatBaseLauncher(object): # The init function will need permission to touch these files # and chown them accordingly for the heat user - def __init__(self, api_port, container_image, user='heat', - heat_dir='/var/log/heat-launcher', - use_tmp_dir=True): + def __init__( + self, api_port=8006, + all_container_image=constants.DEFAULT_HEAT_CONTAINER, + api_container_image=constants.DEFAULT_HEAT_API_CONTAINER, + engine_container_image=constants.DEFAULT_HEAT_ENGINE_CONTAINER, + user='heat', heat_dir='/var/log/heat-launcher', use_tmp_dir=True): + self.api_port = api_port - self.heat_dir = heat_dir + self.all_container_image = all_container_image + self.api_container_image = api_container_image + self.engine_container_image = engine_container_image + self.heat_dir = os.path.abspath(heat_dir) + self.host = "127.0.0.1" + self.db_dump_path = os.path.join( + self.heat_dir, 'heat-db-dump.sql') if os.path.isdir(self.heat_dir): # This one may fail but it's just cleanup. @@ -153,7 +167,8 @@ class HeatBaseLauncher(object): if retval != 0: # It's ok if this fails, it will still work. It just won't # be on tmpfs. - log.warning('Unable to mount tmpfs for logs and database %s: %s' % + log.warning('Unable to mount tmpfs for logs and ' + 'database %s: %s' % (self.heat_dir, cmd_stderr)) self.policy_file = os.path.join(os.path.dirname(__file__), @@ -163,7 +178,6 @@ class HeatBaseLauncher(object): prefix='%s/undercloud_deploy-' % self.heat_dir) else: self.install_dir = self.heat_dir - self.container_image = container_image self.user = user self.sql_db = os.path.join(self.install_dir, 'heat.sqlite') self.log_file = os.path.join(self.install_dir, 'heat.log') @@ -172,6 +186,7 @@ class HeatBaseLauncher(object): self.token_file = os.path.join(self.install_dir, 'token_file.json') self._write_fake_keystone_token(self.api_port, self.token_file) self._write_heat_config() + self._write_api_paste_config() uid = int(self.get_heat_uid()) gid = int(self.get_heat_gid()) os.chown(self.install_dir, uid, gid) @@ -223,9 +238,12 @@ limit_iterators=9000 ''' % {'sqlite_db': self.sql_db, 'log_file': self.log_file, 'api_port': self.api_port, 'policy_file': self.policy_file, 'token_file': self.token_file} + with open(self.config_file, 'w') as temp_file: temp_file.write(heat_config) + def _write_api_paste_config(self): + heat_api_paste_config = ''' [pipeline:heat-api-noauth] pipeline = faultwrap noauth context versionnegotiation apiv1app @@ -258,27 +276,31 @@ heat.filter_factory = heat.api.openstack:faultwrap_filter def get_heat_gid(self): return grp.getgrnam(self.user).gr_gid + def check_database(self): + return True + + def check_message_bus(self): + return True + class HeatContainerLauncher(HeatBaseLauncher): - def __init__(self, api_port, container_image, user='heat', - heat_dir='/var/log/heat-launcher', - use_tmp_dir=True): - self.container_image = container_image + heat_type = 'container' + + def __init__(self, *args, **kwargs): + super(HeatContainerLauncher, self).__init__(*args, **kwargs) self._fetch_container_image() - super(HeatContainerLauncher, self).__init__(api_port, container_image, - user, heat_dir, - use_tmp_dir) + self.host = "127.0.0.1" def _fetch_container_image(self): # force pull of latest container image - cmd = ['podman', 'pull', self.container_image] + cmd = ['podman', 'pull', self.all_container_image] log.debug(' '.join(cmd)) try: subprocess.check_output(cmd) except subprocess.CalledProcessError as e: raise Exception('Unable to fetch container image {}.' - 'Error: {}'.format(self.container_image, e)) + 'Error: {}'.format(self.all_container_image, e)) def launch_heat(self): # run the heat-all process @@ -295,7 +317,7 @@ class HeatContainerLauncher(HeatBaseLauncher): self.install_dir}, '--volume', '%(pfile)s:%(pfile)s:ro' % {'pfile': self.policy_file}, - self.container_image, 'heat-all' + self.all_container_image, 'heat-all' ] log.debug(' '.join(cmd)) os.execvp('podman', cmd) @@ -309,7 +331,7 @@ class HeatContainerLauncher(HeatBaseLauncher): self.config_file}, '--volume', '%(inst_tmp)s:%(inst_tmp)s:Z' % {'inst_tmp': self.install_dir}, - self.container_image, + self.all_container_image, 'heat-manage', 'db_sync'] log.debug(' '.join(cmd)) subprocess.check_call(cmd) @@ -317,7 +339,7 @@ class HeatContainerLauncher(HeatBaseLauncher): def get_heat_uid(self): cmd = [ 'podman', 'run', '--rm', - self.container_image, + self.all_container_image, 'getent', 'passwd', self.user ] log.debug(' '.join(cmd)) @@ -331,7 +353,7 @@ class HeatContainerLauncher(HeatBaseLauncher): def get_heat_gid(self): cmd = [ 'podman', 'run', '--rm', - self.container_image, + self.all_container_image, 'getent', 'group', self.user ] log.debug(' '.join(cmd)) @@ -342,7 +364,7 @@ class HeatContainerLauncher(HeatBaseLauncher): return result.split(':')[2] raise Exception('Could not find heat gid') - def kill_heat(self, pid): + def kill_heat(self, pid, backup_db=False): cmd = ['podman', 'stop', 'heat_all'] log.debug(' '.join(cmd)) # We don't want to hear from this command.. @@ -351,10 +373,11 @@ class HeatContainerLauncher(HeatBaseLauncher): class HeatNativeLauncher(HeatBaseLauncher): - def __init__(self, api_port, container_image, user='heat', - heat_dir='/var/log/heat-launcher'): - super(HeatNativeLauncher, self).__init__(api_port, container_image, - user, heat_dir) + heat_type = 'native' + + def __init__(self, *args, **kwargs): + super(HeatNativeLauncher, self).__init__(*args, **kwargs) + self.host = "127.0.0.1" def launch_heat(self): os.execvp('heat-all', ['heat-all', '--config-file', self.config_file]) @@ -363,5 +386,239 @@ class HeatNativeLauncher(HeatBaseLauncher): subprocess.check_call(['heat-manage', '--config-file', self.config_file, 'db_sync']) - def kill_heat(self, pid): + def kill_heat(self, pid, backup_db=False): os.kill(pid, signal.SIGKILL) + + +class HeatPodLauncher(HeatContainerLauncher): + + heat_type = 'pod' + + def __init__(self, *args, **kwargs): + super(HeatPodLauncher, self).__init__(*args, **kwargs) + log_dir = os.path.join(self.heat_dir, 'log') + if not os.path.isdir(log_dir): + os.makedirs(log_dir) + self.host = self._get_ctlplane_ip() + self._chcon() + + def _chcon(self): + subprocess.check_call( + ['chcon', '-R', '-t', 'container_file_t', + '-l', 's0', self.heat_dir]) + + def _fetch_container_image(self): + # force pull of latest container image + for image in self.api_container_image, self.engine_container_image: + log.info("Pulling conatiner image {}.".format(image)) + cmd = ['sudo', 'podman', 'pull', image] + log.debug(' '.join(cmd)) + try: + subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + raise Exception('Unable to fetch container image {}.' + 'Error: {}'.format(image, e)) + + def launch_heat(self): + inspect = subprocess.run([ + 'sudo', 'podman', 'pod', 'inspect', '--format', + '"{{.State}}"', 'ephemeral-heat'], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if "Running" in self._decode(inspect.stdout): + log.info("ephemeral-heat pod already running, skipping launch") + return + self._write_heat_pod() + subprocess.check_call([ + 'sudo', 'podman', 'play', 'kube', + os.path.join(self.heat_dir, 'heat-pod.yaml') + ]) + + def heat_db_sync(self, restore_db=False): + if not self.database_exists(): + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-it', '-u', 'root', + 'mysql', 'mysql', '-e', 'create database heat' + ]) + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-it', '-u', 'root', + 'mysql', 'mysql', '-e', + 'create user if not exists ' + '\'heat\'@\'%\' identified by \'heat\'' + ]) + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-it', '-u', 'root', + 'mysql', 'mysql', 'heat', '-e', + 'grant all privileges on heat.* to \'heat\'@\'%\'' + ]) + cmd = [ + 'sudo', 'podman', 'run', '--rm', + '--user', 'heat', + '--volume', '%(conf)s:/etc/heat/heat.conf:z' % {'conf': + self.config_file}, + '--volume', '%(inst_tmp)s:%(inst_tmp)s:z' % {'inst_tmp': + self.install_dir}, + self.api_container_image, + 'heat-manage', 'db_sync'] + log.debug(' '.join(cmd)) + subprocess.check_call(cmd) + if restore_db: + self.do_restore_db() + + def do_restore_db(self, db_dump_path=None): + if not db_dump_path: + db_dump_path = self.db_dump_path + subprocess.run([ + 'sudo', 'podman', 'exec', '-i', '-u', 'root', + 'mysql', 'mysql', 'heat'], stdin=open(db_dump_path), + check=True) + + def rm_heat(self, backup_db=False): + if self.database_exists(): + if backup_db: + try: + with open(self.db_dump_path, 'w') as out: + subprocess.run([ + 'sudo', 'podman', 'exec', '-it', '-u', 'root', + 'mysql', 'mysqldump', 'heat'], stdout=out, + check=True) + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-it', '-u', 'root', + 'mysql', 'mysql', 'heat', '-e', + 'drop database heat']) + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-it', '-u', 'root', + 'mysql', 'mysql', 'heat', '-e', + 'drop user \'heat\'@\'%\'']) + except subprocess.CalledProcessError: + pass + subprocess.call([ + 'sudo', 'podman', 'pod', 'rm', '-f', 'ephemeral-heat' + ]) + + def stop_heat(self): + subprocess.check_call([ + 'sudo', 'podman', 'pod', 'stop', 'ephemeral-heat' + ]) + + def check_message_bus(self): + log.info("Checking that message bus (rabbitmq) is up") + try: + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-u', 'root', 'rabbitmq', + 'rabbitmqctl', 'list_queues'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + return True + except subprocess.CalledProcessError as cpe: + log.error("The message bus (rabbitmq) does not seem " + "to be available") + log.error(cpe) + raise + + def check_database(self): + log.info("Checking that database is up") + try: + subprocess.check_call([ + 'sudo', 'podman', 'exec', '-u', 'root', 'mysql', + 'mysql', '-h', self._get_ctlplane_ip(), '-e', + 'show databases;'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + return True + except subprocess.CalledProcessError as cpe: + log.error("The database does not seem to be available") + log.error(cpe) + raise + + def database_exists(self): + output = subprocess.check_output([ + 'sudo', 'podman', 'exec', '-it', '-u', 'root', 'mysql', + 'mysql', '-e', 'show databases like "heat"' + ]) + return 'heat' in str(output) + + def kill_heat(self, pid, backup_db=False): + subprocess.call([ + 'sudo', 'podman', 'pod', 'kill', 'ephemeral-heat' + ]) + + def _decode(self, encoded): + if not encoded: + return "" + decoded = encoded.decode('utf-8') + if decoded.endswith('\n'): + decoded = decoded[:-1] + return decoded + + def _get_transport_url(self): + user = self._decode(subprocess.check_output( + ['sudo', 'hiera', 'rabbitmq::default_user'])) + password = self._decode(subprocess.check_output( + ['sudo', 'hiera', 'rabbitmq::default_pass'])) + fqdn_ctlplane = self._decode(subprocess.check_output( + ['sudo', 'hiera', 'fqdn_ctlplane'])) + port = self._decode(subprocess.check_output( + ['sudo', 'hiera', 'rabbitmq::port'])) + + transport_url = "rabbit://%s:%s@%s:%s/?ssl=0" % \ + (user, password, fqdn_ctlplane, port) + return transport_url + + def _get_db_connection(self): + return ('mysql+pymysql://' + 'heat:heat@{}/heat?read_default_file=' + '/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo'.format( + self._get_ctlplane_vip())) + + def _get_ctlplane_vip(self): + return self._decode(subprocess.check_output( + ['sudo', 'hiera', 'controller_virtual_ip'])) + + def _get_ctlplane_ip(self): + return self._decode(subprocess.check_output( + ['sudo', 'hiera', 'ctlplane'])) + + def _get_num_engine_workers(self): + return int(multiprocessing.cpu_count() / 2) + + def _write_heat_config(self): + heat_config_tmpl_path = os.path.join(constants.DEFAULT_TEMPLATES_DIR, + "ephemeral-heat", + "heat.conf.j2") + with open(heat_config_tmpl_path) as tmpl: + heat_config_tmpl = jinja2.Template(tmpl.read()) + + config_vars = { + "transport_url": self._get_transport_url(), + "db_connection": self._get_db_connection(), + "api_port": self.api_port, + "num_engine_workers": self._get_num_engine_workers(), + } + heat_config = heat_config_tmpl.render(**config_vars) + + with open(self.config_file, 'w') as conf: + conf.write(heat_config) + + def _write_heat_pod(self): + heat_pod_tmpl_path = os.path.join(constants.DEFAULT_TEMPLATES_DIR, + "ephemeral-heat", + "heat-pod.yaml.j2") + with open(heat_pod_tmpl_path) as tmpl: + heat_pod_tmpl = jinja2.Template(tmpl.read()) + + pod_vars = { + "install_dir": self.install_dir, + "heat_dir": self.heat_dir, + "policy_file": self.policy_file, + "ctlplane_ip": self.host, + "api_port": self.api_port, + "api_image": self.api_container_image, + "engine_image": self.engine_container_image, + } + heat_pod = heat_pod_tmpl.render(**pod_vars) + + heat_pod_path = os.path.join(self.heat_dir, "heat-pod.yaml") + with open(heat_pod_path, 'w') as conf: + conf.write(heat_pod)