openstack/python-tripleoclient
Code Issues Proposed changes

Merge "Unmount heat-all tmpfs dir when done"

Browse Source
This commit is contained in:
Zuul
2021-08-26 12:11:22 +00:00
committed by Gerrit Code Review
parent 8251ddbb3d 6a0adcbbf6
commit 16319d2efc
5 changed files with 70 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
tripleoclient/heat_launcher.py
View File

@@ -22,6 +22,7 @@ import logging
import multiprocessing import multiprocessing
import os import os
import pwd import pwd
import shutil
import signal import signal
import subprocess import subprocess
import tarfile import tarfile
@@ -148,38 +149,33 @@ class HeatBaseLauncher(object):
self.skip_heat_pull = skip_heat_pull self.skip_heat_pull = skip_heat_pull
self.zipped_db_suffix = '.tar.bzip2' self.zipped_db_suffix = '.tar.bzip2'
self.log_dir = os.path.join(self.heat_dir, 'log') self.log_dir = os.path.join(self.heat_dir, 'log')
self.use_tmp_dir = use_tmp_dir
if os.path.isdir(self.heat_dir): if not os.path.isdir(self.heat_dir):
if use_root:
# This one may fail but it's just cleanup.
p = subprocess.Popen(['umount', self.heat_dir],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
cmd_stdout, cmd_stderr = p.communicate()
retval = p.returncode
if retval != 0:
log.info('Cleanup unmount of %s failed (probably because '
'it was not mounted): %s' %
(self.heat_dir, cmd_stderr))
else:
log.info('umount of %s success' % (self.heat_dir))
else:
# Create the directory if it doesn't exist. # Create the directory if it doesn't exist.
try: try:
os.makedirs(self.heat_dir, mode=0o700) os.makedirs(self.heat_dir, mode=0o755)
except Exception as e: except Exception as e:
log.error('Creating temp directory "%s" failed: %s' % log.error('Creating temp directory "%s" failed: %s' %
(self.heat_dir, e)) (self.heat_dir, e))
raise Exception('Could not create temp directory %s: %s' % raise Exception('Could not create temp directory %s: %s' %
(self.heat_dir, e)) (self.heat_dir, e))
if self.use_tmp_dir:
self.install_dir = tempfile.mkdtemp(
prefix='%s/tripleo_deploy-' % self.heat_dir)
else:
self.install_dir = self.heat_dir
if use_root: if use_root:
self.umount_install_dir()
if use_root and use_tmp_dir:
# As an optimization we mount the tmp directory in a tmpfs (in # As an optimization we mount the tmp directory in a tmpfs (in
# memory) filesystem. Depending on your system this can cut the # memory) filesystem. Depending on your system this can cut the
# heat deployment times by half. # heat deployment times by half.
p = subprocess.Popen(['mount', '-t', 'tmpfs', '-o', 'size=500M', p = subprocess.Popen(['mount', '-t', 'tmpfs', '-o', 'size=500M',
'tmpfs', self.heat_dir], 'tmpfs', self.install_dir],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True) universal_newlines=True)
@@ -192,12 +188,6 @@ class HeatBaseLauncher(object):
'database %s: %s' % 'database %s: %s' %
(self.heat_dir, cmd_stderr)) (self.heat_dir, cmd_stderr))
if use_tmp_dir:
self.install_dir = tempfile.mkdtemp(
prefix='%s/undercloud_deploy-' % self.heat_dir)
else:
self.install_dir = self.heat_dir
self.log_file = self._get_log_file_path() self.log_file = self._get_log_file_path()
self.sql_db = os.path.join(self.install_dir, 'heat.sqlite') self.sql_db = os.path.join(self.install_dir, 'heat.sqlite')
self.config_file = os.path.join(self.install_dir, 'heat.conf') self.config_file = os.path.join(self.install_dir, 'heat.conf')
@@ -219,6 +209,21 @@ class HeatBaseLauncher(object):
self.kill_heat(None) self.kill_heat(None)
self.rm_heat() self.rm_heat()
def umount_install_dir(self):
# This one may fail but it's just cleanup.
p = subprocess.Popen(['umount', self.install_dir],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
cmd_stdout, cmd_stderr = p.communicate()
retval = p.returncode
if retval != 0:
log.info('Cleanup unmount of %s failed (probably because '
'it was not mounted): %s' %
(self.heat_dir, cmd_stderr))
else:
log.info('umount of %s success' % (self.heat_dir))
def _get_log_file_path(self): def _get_log_file_path(self):
return os.path.join(self.install_dir, 'heat.log') return os.path.join(self.install_dir, 'heat.log')
@@ -430,12 +435,19 @@ class HeatNativeLauncher(HeatBaseLauncher):
def launch_heat(self): def launch_heat(self):
os.execvp('heat-all', ['heat-all', '--config-file', self.config_file]) os.execvp('heat-all', ['heat-all', '--config-file', self.config_file])
def heat_db_sync(self): def heat_db_sync(self, restore_db=False):
subprocess.check_call(['heat-manage', '--config-file', subprocess.check_call(['heat-manage', '--config-file',
self.config_file, 'db_sync']) self.config_file, 'db_sync'])
def kill_heat(self, pid): def kill_heat(self, pid):
os.kill(pid, signal.SIGKILL) os.kill(pid, signal.SIGKILL)
if self.use_tmp_dir:
shutil.copytree(
self.install_dir,
os.path.join(self.heat_dir,
'tripleo_deploy-%s' % self.timestamp))
self.umount_install_dir()
shutil.rmtree(self.install_dir)
class HeatPodLauncher(HeatContainerLauncher): class HeatPodLauncher(HeatContainerLauncher):

9
tripleoclient/tests/test_heat_launcher.py
View File

@@ -539,6 +539,7 @@ class TestHeatPodLauncherUtils(base.TestCase):
utils._local_orchestration_client = None utils._local_orchestration_client = None
mock_launcher = mock.Mock() mock_launcher = mock.Mock()
mock_launcher.api_port = 1234 mock_launcher.api_port = 1234
mock_launcher.heat_type = 'pod'
mock_get_heat_launcher.return_value = mock_launcher mock_get_heat_launcher.return_value = mock_launcher
mock_socket.return_value = 'socket' mock_socket.return_value = 'socket'
utils.launch_heat() utils.launch_heat()
@@ -594,10 +595,12 @@ class TestHeatNativeLauncher(base.TestCase):
mock_mkdtemp.return_value = self.tmp_dir mock_mkdtemp.return_value = self.tmp_dir
def test_install_dir(): def test_install_dir():
mock_mkdtemp.assert_not_called() mock_mkdtemp.assert_called()
return ("", "") return ("", "")
# Test that tempfile.mkdtemp is *not* called before the tmpfs is setup, # Test that tempfile.mkdtemp is called before the tmpfs is setup,
# otherwise the tmpfs will cause the temp dir to be lost # so that the tmpfs mount is created at the temp dir.
self.mock_popen.communicate.side_effect = test_install_dir self.mock_popen.communicate.side_effect = test_install_dir
self.get_launcher() self.get_launcher()
self.assertEqual(['mount', '-t', 'tmpfs'],
self.popen.call_args_list[1][0][0][0:3])

6
tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py
View File

@@ -769,7 +769,7 @@ class TestDeployUndercloud(TestPluginV1):
self.cmd.take_action(parsed_args) self.cmd.take_action(parsed_args)
mock_createdirs.assert_called_once() mock_createdirs.assert_called_once()
mock_puppet.assert_called_once() mock_puppet.assert_called_once()
mock_launchheat.assert_called_with(parsed_args) mock_launchheat.assert_called_with(parsed_args, self.cmd.output_dir)
mock_tht.assert_called_once_with(self.cmd, fake_orchestration, mock_tht.assert_called_once_with(self.cmd, fake_orchestration,
parsed_args) parsed_args)
mock_download.assert_called_with(self.cmd, fake_orchestration, mock_download.assert_called_with(self.cmd, fake_orchestration,
@@ -865,7 +865,7 @@ class TestDeployUndercloud(TestPluginV1):
self.cmd.take_action, parsed_args) self.cmd.take_action, parsed_args)
mock_createdirs.assert_called_once() mock_createdirs.assert_called_once()
mock_puppet.assert_called_once() mock_puppet.assert_called_once()
mock_launchheat.assert_called_with(parsed_args) mock_launchheat.assert_called_with(parsed_args, self.cmd.output_dir)
mock_tht.assert_called_once_with(self.cmd, fake_orchestration, mock_tht.assert_called_once_with(self.cmd, fake_orchestration,
parsed_args) parsed_args)
mock_download.assert_called_with(self.cmd, fake_orchestration, mock_download.assert_called_with(self.cmd, fake_orchestration,
@@ -945,7 +945,7 @@ class TestDeployUndercloud(TestPluginV1):
self.cmd.take_action, parsed_args) self.cmd.take_action, parsed_args)
mock_createdirs.assert_called_once() mock_createdirs.assert_called_once()
mock_puppet.assert_called_once() mock_puppet.assert_called_once()
mock_launchheat.assert_called_with(parsed_args) mock_launchheat.assert_called_with(parsed_args, self.cmd.output_dir)
mock_tht.assert_not_called() mock_tht.assert_not_called()
mock_download.assert_not_called() mock_download.assert_not_called()
mock_tarball.assert_called_once() mock_tarball.assert_called_once()

5
tripleoclient/utils.py
View File

@@ -2645,7 +2645,7 @@ def write_user_environment(env_map, abs_env_path, tht_root,
return user_env_path return user_env_path
def launch_heat(launcher=None, restore_db=False): def launch_heat(launcher=None, restore_db=False, heat_type='pod'):
global _local_orchestration_client global _local_orchestration_client
global _heat_pid global _heat_pid
@@ -2655,7 +2655,7 @@ def launch_heat(launcher=None, restore_db=False):
return _local_orchestration_client return _local_orchestration_client
if not launcher: if not launcher:
launcher = get_heat_launcher() launcher = get_heat_launcher(heat_type)
_heat_pid = 0 _heat_pid = 0
if launcher.heat_type == 'native': if launcher.heat_type == 'native':
@@ -2669,6 +2669,7 @@ def launch_heat(launcher=None, restore_db=False):
# Wait for the API to be listening # Wait for the API to be listening
heat_api_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) heat_api_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
test_heat_api_port(heat_api_socket, launcher.host, int(launcher.api_port)) test_heat_api_port(heat_api_socket, launcher.host, int(launcher.api_port))
if launcher.heat_type == 'pod':
launcher.wait_for_message_queue() launcher.wait_for_message_queue()
_local_orchestration_client = tc_heat_utils.local_orchestration_client( _local_orchestration_client = tc_heat_utils.local_orchestration_client(

33
tripleoclient/v1/tripleo_deploy.py
View File

@@ -448,19 +448,27 @@ class Deploy(command.Command):
pid, ret = os.waitpid(self.heat_pid, 0) pid, ret = os.waitpid(self.heat_pid, 0)
self.heat_pid = None self.heat_pid = None
def _launch_heat(self, parsed_args): def _launch_heat(self, parsed_args, output_dir):
# we do this as root to chown config files properly for docker, etc. # we do this as root to chown config files properly for docker, etc.
heat_launcher_path = os.path.join(output_dir, 'heat_launcher')
if parsed_args.heat_user:
heat_user = parsed_args.heat_user
else:
heat_user = parsed_args.deployment_user
if parsed_args.heat_native is not None and \ if parsed_args.heat_native is not None and \
parsed_args.heat_native.lower() == "false": parsed_args.heat_native.lower() == "false":
self.heat_launch = heat_launcher.HeatContainerLauncher( self.heat_launch = heat_launcher.HeatContainerLauncher(
parsed_args.heat_api_port, api_port=parsed_args.heat_api_port,
parsed_args.heat_container_image, all_container_image=parsed_args.heat_container_image,
parsed_args.heat_user) user=heat_user,
heat_dir=heat_launcher_path)
else: else:
self.heat_launch = heat_launcher.HeatNativeLauncher( self.heat_launch = heat_launcher.HeatNativeLauncher(
parsed_args.heat_api_port, api_port=parsed_args.heat_api_port,
parsed_args.heat_container_image, user=heat_user,
parsed_args.heat_user, heat_dir=heat_launcher_path,
use_root=True) use_root=True)
# NOTE(dprince): we launch heat with fork exec because # NOTE(dprince): we launch heat with fork exec because
@@ -473,12 +481,12 @@ class Deploy(command.Command):
if parsed_args.heat_native is not None and \ if parsed_args.heat_native is not None and \
parsed_args.heat_native.lower() == "true": parsed_args.heat_native.lower() == "true":
try: try:
uid = pwd.getpwnam(parsed_args.heat_user).pw_uid uid = pwd.getpwnam(heat_user).pw_uid
gid = pwd.getpwnam(parsed_args.heat_user).pw_gid gid = pwd.getpwnam(heat_user).pw_gid
except KeyError: except KeyError:
msg = _( msg = _(
"Please create a %s user account before " "Please create a %s user account before "
"proceeding.") % parsed_args.heat_user "proceeding.") % heat_user
self.log.error(msg) self.log.error(msg)
raise exceptions.DeploymentError(msg) raise exceptions.DeploymentError(msg)
os.setgid(gid) os.setgid(gid)
@@ -947,9 +955,8 @@ class Deploy(command.Command):
parser.add_argument( parser.add_argument(
'--heat-user', metavar='<HEAT_USER>', '--heat-user', metavar='<HEAT_USER>',
dest='heat_user', dest='heat_user',
default='heat',
help=_('User to execute the non-privileged heat-all process. ' help=_('User to execute the non-privileged heat-all process. '
'Defaults to heat.') 'Defaults to the value of --deployment-user.')
) )
# TODO(cjeanner) drop that once using oslo.privsep # TODO(cjeanner) drop that once using oslo.privsep
parser.add_argument( parser.add_argument(
@@ -1240,7 +1247,7 @@ class Deploy(command.Command):
self._set_stack_action(parsed_args) self._set_stack_action(parsed_args)
# Launch heat. # Launch heat.
orchestration_client = self._launch_heat(parsed_args) orchestration_client = self._launch_heat(parsed_args, output_dir)
# Wait for heat to be ready. # Wait for heat to be ready.
utils.wait_api_port_ready(parsed_args.heat_api_port) utils.wait_api_port_ready(parsed_args.heat_api_port)
# Deploy TripleO Heat templates. # Deploy TripleO Heat templates.