openstack
/
python-tripleoclient
Code Issues Proposed changes

Add HeatPodLauncher 

Add a new subclass, HeatPodLauncher to launching an emphemeral Heat
within a podman pod. The pod contains separate processes for Heat API
and engine, allowing for multiple engine workers (defaults to cpu
logical core count / 2).

The pod makes use of the undercloud's already installed database and
message bus.

Change-Id: Ie8b4a5ca83284f66dceae32c6f2fc99cdc8c9ffb
Signed-off-by: James Slagle <jslagle@redhat.com>
This commit is contained in:
James Slagle 2021-02-08 16:06:54 -05:00
parent 03e587dbca
commit 0f9fbfa58d
5 changed files with 443 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
setup.cfg
View File

@ -27,6 +27,9 @@ classifier =
packages =
tripleoclient
data_files =
share/python-tripleoclient/templates = templates/*
[entry_points]
openstack.cli.extension =
tripleoclient = tripleoclient.plugin

105
templates/ephemeral-heat/heat-pod.yaml.j2 Normal file
View File

@ -0,0 +1,105 @@
apiVersion: v1
kind: Pod
metadata:
labels:
app: ephemeral-heat
name: ephemeral-heat
spec:
containers:
- command:
- heat-engine
- --config-file
- /etc/heat/heat.conf
env:
- name: PATH
value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
- name: TERM
value: xterm
- name: container
value: oci
- name: LANG
value: en_US.UTF-8
image: {{ engine_image }}
name: engine
resources: {}
securityContext:
allowPrivilegeEscalation: true
capabilities: {}
privileged: false
readOnlyRootFilesystem: false
runAsGroup: 0
runAsUser: 0
seLinuxOptions: {}
volumeMounts:
- mountPath: /var/log/heat
name: heat-log
- mountPath: /etc/heat/heat.conf
name: heat-config
readOnly: true
workingDir: /
- command:
- heat-api
- --config-file
- /etc/heat/heat.conf
ports:
- containerPort: {{ api_port }}
hostPort: {{ api_port }}
hostIP: {{ ctlplane_ip }}
env:
- name: PATH
value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
- name: TERM
value: xterm
- name: container
value: oci
- name: LANG
value: en_US.UTF-8
image: {{ api_image }}
name: api
resources: {}
securityContext:
allowPrivilegeEscalation: true
capabilities: {}
privileged: false
readOnlyRootFilesystem: false
runAsGroup: 0
runAsUser: 0
seLinuxOptions: {}
volumeMounts:
- mountPath: /var/log/heat
name: heat-log
- mountPath: /etc/heat/heat.conf
name: heat-config
readOnly: true
- mountPath: /etc/heat/api-paste.ini
name: heat-api-paste
readOnly: true
- mountPath: /token_file.json
name: heat-token-file
readOnly: true
- mountPath: /etc/heat/noauth_policy.json
name: heat-noauth-policy
readOnly: true
workingDir: /
volumes:
- hostPath:
path: {{ heat_dir}}/log
type: Directory
name: heat-log
- hostPath:
path: {{ install_dir }}/heat.conf
type: File
name: heat-config
- hostPath:
path: {{ heat_dir }}/api-paste.ini
type: File
name: heat-api-paste
- hostPath:
path: {{ heat_dir }}/token_file.json
type: File
name: heat-token-file
- hostPath:
path: {{ policy_file }}
type: File
name: heat-noauth-policy
status: {}

43
templates/ephemeral-heat/heat.conf.j2 Normal file
View File

@ -0,0 +1,43 @@
[DEFAULT]
client_retry_limit=2
convergence_engine = true
debug = true
default_deployment_signal_transport = HEAT_SIGNAL
deferred_auth_method = password
keystone_backend = heat.engine.clients.os.keystone.fake_keystoneclient.FakeKeystoneClient
log_dir = /var/log/heat
max_json_body_size = 8388608
max_nested_stack_depth = 10
max_resources_per_stack=-1
num_engine_workers = {{ num_engine_workers }}
rpc_poll_timeout = 60
rpc_response_timeout = 600
transport_url={{ transport_url }}
[oslo_messaging_notifications]
driver = noop
[oslo_messaging_rabbit]
heatbeat_timeout_threshold=60
[noauth]
token_response = /token_file.json
[heat_api]
bind_host = 0.0.0.0
bind_port = {{ api_port }}
workers = 1
[database]
connection = {{ db_connection }}
[paste_deploy]
api_paste_config = /etc/heat/api-paste.ini
flavor = noauth
[oslo_policy]
policy_file = /etc/heat/noauth_policy.json
[yaql]
limit_iterators=9000
memory_quota=900000

11
tripleoclient/constants.py
View File

@ -43,6 +43,14 @@ DEFAULT_HEAT_CONTAINER = ('{}/{}/openstack-heat-all:{}'.format(
DEFAULT_CONTAINER_REGISTRY,
DEFAULT_CONTAINER_NAMESPACE,
DEFAULT_CONTAINER_TAG))
DEFAULT_HEAT_API_CONTAINER = ('{}/{}/openstack-heat-api:{}'.format(
DEFAULT_CONTAINER_REGISTRY,
DEFAULT_CONTAINER_NAMESPACE,
DEFAULT_CONTAINER_TAG))
DEFAULT_HEAT_ENGINE_CONTAINER = ('{}/{}/openstack-heat-engine:{}'.format(
DEFAULT_CONTAINER_REGISTRY,
DEFAULT_CONTAINER_NAMESPACE,
DEFAULT_CONTAINER_TAG))
USER_PARAMETERS = 'user-environments/tripleoclient-parameters.yaml'
@ -98,8 +106,9 @@ VALIDATIONS_LOG_BASEDIR = '/var/log/validations'
DEFAULT_WORK_DIR = os.path.join(os.environ.get('HOME', '~/'),
'config-download')
TRIPLEO_STATIC_INVENTORY = 'tripleo-ansible-inventory.yaml'
DEFAULT_TEMPLATES_DIR = "/usr/share/python-tripleoclient/templates"
TRIPLEO_STATIC_INVENTORY = 'tripleo-ansible-inventory.yaml'
ANSIBLE_INVENTORY = os.path.join(DEFAULT_WORK_DIR,
'{}/', TRIPLEO_STATIC_INVENTORY)
ANSIBLE_VALIDATION_DIR = (

307
tripleoclient/heat_launcher.py
View File

@ -17,14 +17,18 @@ import datetime
import grp
import json
import logging
import multiprocessing
import os
import pwd
import signal
import subprocess
import tempfile
import jinja2
from oslo_utils import timeutils
from tripleoclient import constants
log = logging.getLogger(__name__)
NEXT_DAY = (timeutils.utcnow() + datetime.timedelta(days=2)).isoformat()
@ -111,11 +115,21 @@ class HeatBaseLauncher(object):
# The init function will need permission to touch these files
# and chown them accordingly for the heat user
def __init__(self, api_port, container_image, user='heat',
heat_dir='/var/log/heat-launcher',
use_tmp_dir=True):
def __init__(
self, api_port=8006,
all_container_image=constants.DEFAULT_HEAT_CONTAINER,
api_container_image=constants.DEFAULT_HEAT_API_CONTAINER,
engine_container_image=constants.DEFAULT_HEAT_ENGINE_CONTAINER,
user='heat', heat_dir='/var/log/heat-launcher', use_tmp_dir=True):
self.api_port = api_port
self.heat_dir = heat_dir
self.all_container_image = all_container_image
self.api_container_image = api_container_image
self.engine_container_image = engine_container_image
self.heat_dir = os.path.abspath(heat_dir)
self.host = "127.0.0.1"
self.db_dump_path = os.path.join(
self.heat_dir, 'heat-db-dump.sql')
if os.path.isdir(self.heat_dir):
# This one may fail but it's just cleanup.
@ -153,7 +167,8 @@ class HeatBaseLauncher(object):
if retval != 0:
# It's ok if this fails, it will still work. It just won't
# be on tmpfs.
log.warning('Unable to mount tmpfs for logs and database %s: %s' %
log.warning('Unable to mount tmpfs for logs and '
'database %s: %s' %
(self.heat_dir, cmd_stderr))
self.policy_file = os.path.join(os.path.dirname(__file__),
@ -163,7 +178,6 @@ class HeatBaseLauncher(object):
prefix='%s/undercloud_deploy-' % self.heat_dir)
else:
self.install_dir = self.heat_dir
self.container_image = container_image
self.user = user
self.sql_db = os.path.join(self.install_dir, 'heat.sqlite')
self.log_file = os.path.join(self.install_dir, 'heat.log')
@ -172,6 +186,7 @@ class HeatBaseLauncher(object):
self.token_file = os.path.join(self.install_dir, 'token_file.json')
self._write_fake_keystone_token(self.api_port, self.token_file)
self._write_heat_config()
self._write_api_paste_config()
uid = int(self.get_heat_uid())
gid = int(self.get_heat_gid())
os.chown(self.install_dir, uid, gid)
@ -223,9 +238,12 @@ limit_iterators=9000
''' % {'sqlite_db': self.sql_db, 'log_file': self.log_file,
'api_port': self.api_port, 'policy_file': self.policy_file,
'token_file': self.token_file}
with open(self.config_file, 'w') as temp_file:
temp_file.write(heat_config)
def _write_api_paste_config(self):
heat_api_paste_config = '''
[pipeline:heat-api-noauth]
pipeline = faultwrap noauth context versionnegotiation apiv1app
@ -258,27 +276,31 @@ heat.filter_factory = heat.api.openstack:faultwrap_filter
def get_heat_gid(self):
return grp.getgrnam(self.user).gr_gid
def check_database(self):
return True
def check_message_bus(self):
return True
class HeatContainerLauncher(HeatBaseLauncher):
def __init__(self, api_port, container_image, user='heat',
heat_dir='/var/log/heat-launcher',
use_tmp_dir=True):
self.container_image = container_image
heat_type = 'container'
def __init__(self, *args, **kwargs):
super(HeatContainerLauncher, self).__init__(*args, **kwargs)
self._fetch_container_image()
super(HeatContainerLauncher, self).__init__(api_port, container_image,
user, heat_dir,
use_tmp_dir)
self.host = "127.0.0.1"
def _fetch_container_image(self):
# force pull of latest container image
cmd = ['podman', 'pull', self.container_image]
cmd = ['podman', 'pull', self.all_container_image]
log.debug(' '.join(cmd))
try:
subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise Exception('Unable to fetch container image {}.'
'Error: {}'.format(self.container_image, e))
'Error: {}'.format(self.all_container_image, e))
def launch_heat(self):
# run the heat-all process
@ -295,7 +317,7 @@ class HeatContainerLauncher(HeatBaseLauncher):
self.install_dir},
'--volume', '%(pfile)s:%(pfile)s:ro' % {'pfile':
self.policy_file},
self.container_image, 'heat-all'
self.all_container_image, 'heat-all'
]
log.debug(' '.join(cmd))
os.execvp('podman', cmd)
@ -309,7 +331,7 @@ class HeatContainerLauncher(HeatBaseLauncher):
self.config_file},
'--volume', '%(inst_tmp)s:%(inst_tmp)s:Z' % {'inst_tmp':
self.install_dir},
self.container_image,
self.all_container_image,
'heat-manage', 'db_sync']
log.debug(' '.join(cmd))
subprocess.check_call(cmd)
@ -317,7 +339,7 @@ class HeatContainerLauncher(HeatBaseLauncher):
def get_heat_uid(self):
cmd = [
'podman', 'run', '--rm',
self.container_image,
self.all_container_image,
'getent', 'passwd', self.user
]
log.debug(' '.join(cmd))
@ -331,7 +353,7 @@ class HeatContainerLauncher(HeatBaseLauncher):
def get_heat_gid(self):
cmd = [
'podman', 'run', '--rm',
self.container_image,
self.all_container_image,
'getent', 'group', self.user
]
log.debug(' '.join(cmd))
@ -342,7 +364,7 @@ class HeatContainerLauncher(HeatBaseLauncher):
return result.split(':')[2]
raise Exception('Could not find heat gid')
def kill_heat(self, pid):
def kill_heat(self, pid, backup_db=False):
cmd = ['podman', 'stop', 'heat_all']
log.debug(' '.join(cmd))
# We don't want to hear from this command..
@ -351,10 +373,11 @@ class HeatContainerLauncher(HeatBaseLauncher):
class HeatNativeLauncher(HeatBaseLauncher):
def __init__(self, api_port, container_image, user='heat',
heat_dir='/var/log/heat-launcher'):
super(HeatNativeLauncher, self).__init__(api_port, container_image,
user, heat_dir)
heat_type = 'native'
def __init__(self, *args, **kwargs):
super(HeatNativeLauncher, self).__init__(*args, **kwargs)
self.host = "127.0.0.1"
def launch_heat(self):
os.execvp('heat-all', ['heat-all', '--config-file', self.config_file])
@ -363,5 +386,239 @@ class HeatNativeLauncher(HeatBaseLauncher):
subprocess.check_call(['heat-manage', '--config-file',
self.config_file, 'db_sync'])
def kill_heat(self, pid):
def kill_heat(self, pid, backup_db=False):
os.kill(pid, signal.SIGKILL)
class HeatPodLauncher(HeatContainerLauncher):
heat_type = 'pod'
def __init__(self, *args, **kwargs):
super(HeatPodLauncher, self).__init__(*args, **kwargs)
log_dir = os.path.join(self.heat_dir, 'log')
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
self.host = self._get_ctlplane_ip()
self._chcon()
def _chcon(self):
subprocess.check_call(
['chcon', '-R', '-t', 'container_file_t',
'-l', 's0', self.heat_dir])
def _fetch_container_image(self):
# force pull of latest container image
for image in self.api_container_image, self.engine_container_image:
log.info("Pulling conatiner image {}.".format(image))
cmd = ['sudo', 'podman', 'pull', image]
log.debug(' '.join(cmd))
try:
subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise Exception('Unable to fetch container image {}.'
'Error: {}'.format(image, e))
def launch_heat(self):
inspect = subprocess.run([
'sudo', 'podman', 'pod', 'inspect', '--format',
'"{{.State}}"', 'ephemeral-heat'],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if "Running" in self._decode(inspect.stdout):
log.info("ephemeral-heat pod already running, skipping launch")
return
self._write_heat_pod()
subprocess.check_call([
'sudo', 'podman', 'play', 'kube',
os.path.join(self.heat_dir, 'heat-pod.yaml')
])
def heat_db_sync(self, restore_db=False):
if not self.database_exists():
subprocess.check_call([
'sudo', 'podman', 'exec', '-it', '-u', 'root',
'mysql', 'mysql', '-e', 'create database heat'
])
subprocess.check_call([
'sudo', 'podman', 'exec', '-it', '-u', 'root',
'mysql', 'mysql', '-e',
'create user if not exists '
'\'heat\'@\'%\' identified by \'heat\''
])
subprocess.check_call([
'sudo', 'podman', 'exec', '-it', '-u', 'root',
'mysql', 'mysql', 'heat', '-e',
'grant all privileges on heat.* to \'heat\'@\'%\''
])
cmd = [
'sudo', 'podman', 'run', '--rm',
'--user', 'heat',
'--volume', '%(conf)s:/etc/heat/heat.conf:z' % {'conf':
self.config_file},
'--volume', '%(inst_tmp)s:%(inst_tmp)s:z' % {'inst_tmp':
self.install_dir},
self.api_container_image,
'heat-manage', 'db_sync']
log.debug(' '.join(cmd))
subprocess.check_call(cmd)
if restore_db:
self.do_restore_db()
def do_restore_db(self, db_dump_path=None):
if not db_dump_path:
db_dump_path = self.db_dump_path
subprocess.run([
'sudo', 'podman', 'exec', '-i', '-u', 'root',
'mysql', 'mysql', 'heat'], stdin=open(db_dump_path),
check=True)
def rm_heat(self, backup_db=False):
if self.database_exists():
if backup_db:
try:
with open(self.db_dump_path, 'w') as out:
subprocess.run([
'sudo', 'podman', 'exec', '-it', '-u', 'root',
'mysql', 'mysqldump', 'heat'], stdout=out,
check=True)
subprocess.check_call([
'sudo', 'podman', 'exec', '-it', '-u', 'root',
'mysql', 'mysql', 'heat', '-e',
'drop database heat'])
subprocess.check_call([
'sudo', 'podman', 'exec', '-it', '-u', 'root',
'mysql', 'mysql', 'heat', '-e',
'drop user \'heat\'@\'%\''])
except subprocess.CalledProcessError:
pass
subprocess.call([
'sudo', 'podman', 'pod', 'rm', '-f', 'ephemeral-heat'
])
def stop_heat(self):
subprocess.check_call([
'sudo', 'podman', 'pod', 'stop', 'ephemeral-heat'
])
def check_message_bus(self):
log.info("Checking that message bus (rabbitmq) is up")
try:
subprocess.check_call([
'sudo', 'podman', 'exec', '-u', 'root', 'rabbitmq',
'rabbitmqctl', 'list_queues'],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
return True
except subprocess.CalledProcessError as cpe:
log.error("The message bus (rabbitmq) does not seem "
"to be available")
log.error(cpe)
raise
def check_database(self):
log.info("Checking that database is up")
try:
subprocess.check_call([
'sudo', 'podman', 'exec', '-u', 'root', 'mysql',
'mysql', '-h', self._get_ctlplane_ip(), '-e',
'show databases;'],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
return True
except subprocess.CalledProcessError as cpe:
log.error("The database does not seem to be available")
log.error(cpe)
raise
def database_exists(self):
output = subprocess.check_output([
'sudo', 'podman', 'exec', '-it', '-u', 'root', 'mysql',
'mysql', '-e', 'show databases like "heat"'
])
return 'heat' in str(output)
def kill_heat(self, pid, backup_db=False):
subprocess.call([
'sudo', 'podman', 'pod', 'kill', 'ephemeral-heat'
])
def _decode(self, encoded):
if not encoded:
return ""
decoded = encoded.decode('utf-8')
if decoded.endswith('\n'):
decoded = decoded[:-1]
return decoded
def _get_transport_url(self):
user = self._decode(subprocess.check_output(
['sudo', 'hiera', 'rabbitmq::default_user']))
password = self._decode(subprocess.check_output(
['sudo', 'hiera', 'rabbitmq::default_pass']))
fqdn_ctlplane = self._decode(subprocess.check_output(
['sudo', 'hiera', 'fqdn_ctlplane']))
port = self._decode(subprocess.check_output(
['sudo', 'hiera', 'rabbitmq::port']))
transport_url = "rabbit://%s:%s@%s:%s/?ssl=0" % \
(user, password, fqdn_ctlplane, port)
return transport_url
def _get_db_connection(self):
return ('mysql+pymysql://'
'heat:heat@{}/heat?read_default_file='
'/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo'.format(
self._get_ctlplane_vip()))
def _get_ctlplane_vip(self):
return self._decode(subprocess.check_output(
['sudo', 'hiera', 'controller_virtual_ip']))
def _get_ctlplane_ip(self):
return self._decode(subprocess.check_output(
['sudo', 'hiera', 'ctlplane']))
def _get_num_engine_workers(self):
return int(multiprocessing.cpu_count() / 2)
def _write_heat_config(self):
heat_config_tmpl_path = os.path.join(constants.DEFAULT_TEMPLATES_DIR,
"ephemeral-heat",
"heat.conf.j2")
with open(heat_config_tmpl_path) as tmpl:
heat_config_tmpl = jinja2.Template(tmpl.read())
config_vars = {
"transport_url": self._get_transport_url(),
"db_connection": self._get_db_connection(),
"api_port": self.api_port,
"num_engine_workers": self._get_num_engine_workers(),
}
heat_config = heat_config_tmpl.render(**config_vars)
with open(self.config_file, 'w') as conf:
conf.write(heat_config)
def _write_heat_pod(self):
heat_pod_tmpl_path = os.path.join(constants.DEFAULT_TEMPLATES_DIR,
"ephemeral-heat",
"heat-pod.yaml.j2")
with open(heat_pod_tmpl_path) as tmpl:
heat_pod_tmpl = jinja2.Template(tmpl.read())
pod_vars = {
"install_dir": self.install_dir,
"heat_dir": self.heat_dir,
"policy_file": self.policy_file,
"ctlplane_ip": self.host,
"api_port": self.api_port,
"api_image": self.api_container_image,
"engine_image": self.engine_container_image,
}
heat_pod = heat_pod_tmpl.render(**pod_vars)
heat_pod_path = os.path.join(self.heat_dir, "heat-pod.yaml")
with open(heat_pod_path, 'w') as conf:
conf.write(heat_pod)