openstack
/
python-tripleoclient
Code Issues Proposed changes

Merge "Unmount heat-all tmpfs dir when done"

This commit is contained in:
Zuul 2021-08-26 12:11:22 +00:00 committed by Gerrit Code Review
parent 8251ddbb3d 6a0adcbbf6
commit 16319d2efc
5 changed files with 70 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
tripleoclient/heat_launcher.py
View File

@ -22,6 +22,7 @@ import logging
import multiprocessing
import os
import pwd
import shutil
import signal
import subprocess
import tarfile
@ -148,38 +149,33 @@ class HeatBaseLauncher(object):
self.skip_heat_pull = skip_heat_pull
self.zipped_db_suffix = '.tar.bzip2'
self.log_dir = os.path.join(self.heat_dir, 'log')
self.use_tmp_dir = use_tmp_dir
if os.path.isdir(self.heat_dir):
if use_root:
# This one may fail but it's just cleanup.
p = subprocess.Popen(['umount', self.heat_dir],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
cmd_stdout, cmd_stderr = p.communicate()
retval = p.returncode
if retval != 0:
log.info('Cleanup unmount of %s failed (probably because '
'it was not mounted): %s' %
(self.heat_dir, cmd_stderr))
else:
log.info('umount of %s success' % (self.heat_dir))
else:
if not os.path.isdir(self.heat_dir):
# Create the directory if it doesn't exist.
try:
os.makedirs(self.heat_dir, mode=0o700)
os.makedirs(self.heat_dir, mode=0o755)
except Exception as e:
log.error('Creating temp directory "%s" failed: %s' %
(self.heat_dir, e))
raise Exception('Could not create temp directory %s: %s' %
(self.heat_dir, e))
if self.use_tmp_dir:
self.install_dir = tempfile.mkdtemp(
prefix='%s/tripleo_deploy-' % self.heat_dir)
else:
self.install_dir = self.heat_dir
if use_root:
self.umount_install_dir()
if use_root and use_tmp_dir:
# As an optimization we mount the tmp directory in a tmpfs (in
# memory) filesystem. Depending on your system this can cut the
# heat deployment times by half.
p = subprocess.Popen(['mount', '-t', 'tmpfs', '-o', 'size=500M',
'tmpfs', self.heat_dir],
'tmpfs', self.install_dir],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
@ -192,12 +188,6 @@ class HeatBaseLauncher(object):
'database %s: %s' %
(self.heat_dir, cmd_stderr))
if use_tmp_dir:
self.install_dir = tempfile.mkdtemp(
prefix='%s/undercloud_deploy-' % self.heat_dir)
else:
self.install_dir = self.heat_dir
self.log_file = self._get_log_file_path()
self.sql_db = os.path.join(self.install_dir, 'heat.sqlite')
self.config_file = os.path.join(self.install_dir, 'heat.conf')
@ -219,6 +209,21 @@ class HeatBaseLauncher(object):
self.kill_heat(None)
self.rm_heat()
def umount_install_dir(self):
# This one may fail but it's just cleanup.
p = subprocess.Popen(['umount', self.install_dir],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
cmd_stdout, cmd_stderr = p.communicate()
retval = p.returncode
if retval != 0:
log.info('Cleanup unmount of %s failed (probably because '
'it was not mounted): %s' %
(self.heat_dir, cmd_stderr))
else:
log.info('umount of %s success' % (self.heat_dir))
def _get_log_file_path(self):
return os.path.join(self.install_dir, 'heat.log')
@ -430,12 +435,19 @@ class HeatNativeLauncher(HeatBaseLauncher):
def launch_heat(self):
os.execvp('heat-all', ['heat-all', '--config-file', self.config_file])
def heat_db_sync(self):
def heat_db_sync(self, restore_db=False):
subprocess.check_call(['heat-manage', '--config-file',
self.config_file, 'db_sync'])
def kill_heat(self, pid):
os.kill(pid, signal.SIGKILL)
if self.use_tmp_dir:
shutil.copytree(
self.install_dir,
os.path.join(self.heat_dir,
'tripleo_deploy-%s' % self.timestamp))
self.umount_install_dir()
shutil.rmtree(self.install_dir)
class HeatPodLauncher(HeatContainerLauncher):

9
tripleoclient/tests/test_heat_launcher.py
View File

@ -539,6 +539,7 @@ class TestHeatPodLauncherUtils(base.TestCase):
utils._local_orchestration_client = None
mock_launcher = mock.Mock()
mock_launcher.api_port = 1234
mock_launcher.heat_type = 'pod'
mock_get_heat_launcher.return_value = mock_launcher
mock_socket.return_value = 'socket'
utils.launch_heat()
@ -594,10 +595,12 @@ class TestHeatNativeLauncher(base.TestCase):
mock_mkdtemp.return_value = self.tmp_dir
def test_install_dir():
mock_mkdtemp.assert_not_called()
mock_mkdtemp.assert_called()
return ("", "")
# Test that tempfile.mkdtemp is *not* called before the tmpfs is setup,
# otherwise the tmpfs will cause the temp dir to be lost
# Test that tempfile.mkdtemp is called before the tmpfs is setup,
# so that the tmpfs mount is created at the temp dir.
self.mock_popen.communicate.side_effect = test_install_dir
self.get_launcher()
self.assertEqual(['mount', '-t', 'tmpfs'],
self.popen.call_args_list[1][0][0][0:3])

6
tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py
View File

@ -769,7 +769,7 @@ class TestDeployUndercloud(TestPluginV1):
self.cmd.take_action(parsed_args)
mock_createdirs.assert_called_once()
mock_puppet.assert_called_once()
mock_launchheat.assert_called_with(parsed_args)
mock_launchheat.assert_called_with(parsed_args, self.cmd.output_dir)
mock_tht.assert_called_once_with(self.cmd, fake_orchestration,
parsed_args)
mock_download.assert_called_with(self.cmd, fake_orchestration,
@ -865,7 +865,7 @@ class TestDeployUndercloud(TestPluginV1):
self.cmd.take_action, parsed_args)
mock_createdirs.assert_called_once()
mock_puppet.assert_called_once()
mock_launchheat.assert_called_with(parsed_args)
mock_launchheat.assert_called_with(parsed_args, self.cmd.output_dir)
mock_tht.assert_called_once_with(self.cmd, fake_orchestration,
parsed_args)
mock_download.assert_called_with(self.cmd, fake_orchestration,
@ -945,7 +945,7 @@ class TestDeployUndercloud(TestPluginV1):
self.cmd.take_action, parsed_args)
mock_createdirs.assert_called_once()
mock_puppet.assert_called_once()
mock_launchheat.assert_called_with(parsed_args)
mock_launchheat.assert_called_with(parsed_args, self.cmd.output_dir)
mock_tht.assert_not_called()
mock_download.assert_not_called()
mock_tarball.assert_called_once()

7
tripleoclient/utils.py
View File

@ -2645,7 +2645,7 @@ def write_user_environment(env_map, abs_env_path, tht_root,
return user_env_path
def launch_heat(launcher=None, restore_db=False):
def launch_heat(launcher=None, restore_db=False, heat_type='pod'):
global _local_orchestration_client
global _heat_pid
@ -2655,7 +2655,7 @@ def launch_heat(launcher=None, restore_db=False):
return _local_orchestration_client
if not launcher:
launcher = get_heat_launcher()
launcher = get_heat_launcher(heat_type)
_heat_pid = 0
if launcher.heat_type == 'native':
@ -2669,7 +2669,8 @@ def launch_heat(launcher=None, restore_db=False):
# Wait for the API to be listening
heat_api_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
test_heat_api_port(heat_api_socket, launcher.host, int(launcher.api_port))
launcher.wait_for_message_queue()
if launcher.heat_type == 'pod':
launcher.wait_for_message_queue()
_local_orchestration_client = tc_heat_utils.local_orchestration_client(
launcher.host, launcher.api_port)

33
tripleoclient/v1/tripleo_deploy.py
View File

@ -448,19 +448,27 @@ class Deploy(command.Command):
pid, ret = os.waitpid(self.heat_pid, 0)
self.heat_pid = None
def _launch_heat(self, parsed_args):
def _launch_heat(self, parsed_args, output_dir):
# we do this as root to chown config files properly for docker, etc.
heat_launcher_path = os.path.join(output_dir, 'heat_launcher')
if parsed_args.heat_user:
heat_user = parsed_args.heat_user
else:
heat_user = parsed_args.deployment_user
if parsed_args.heat_native is not None and \
parsed_args.heat_native.lower() == "false":
self.heat_launch = heat_launcher.HeatContainerLauncher(
parsed_args.heat_api_port,
parsed_args.heat_container_image,
parsed_args.heat_user)
api_port=parsed_args.heat_api_port,
all_container_image=parsed_args.heat_container_image,
user=heat_user,
heat_dir=heat_launcher_path)
else:
self.heat_launch = heat_launcher.HeatNativeLauncher(
parsed_args.heat_api_port,
parsed_args.heat_container_image,
parsed_args.heat_user,
api_port=parsed_args.heat_api_port,
user=heat_user,
heat_dir=heat_launcher_path,
use_root=True)
# NOTE(dprince): we launch heat with fork exec because
@ -473,12 +481,12 @@ class Deploy(command.Command):
if parsed_args.heat_native is not None and \
parsed_args.heat_native.lower() == "true":
try:
uid = pwd.getpwnam(parsed_args.heat_user).pw_uid
gid = pwd.getpwnam(parsed_args.heat_user).pw_gid
uid = pwd.getpwnam(heat_user).pw_uid
gid = pwd.getpwnam(heat_user).pw_gid
except KeyError:
msg = _(
"Please create a %s user account before "
"proceeding.") % parsed_args.heat_user
"proceeding.") % heat_user
self.log.error(msg)
raise exceptions.DeploymentError(msg)
os.setgid(gid)
@ -947,9 +955,8 @@ class Deploy(command.Command):
parser.add_argument(
'--heat-user', metavar='<HEAT_USER>',
dest='heat_user',
default='heat',
help=_('User to execute the non-privileged heat-all process. '
'Defaults to heat.')
'Defaults to the value of --deployment-user.')
)
# TODO(cjeanner) drop that once using oslo.privsep
parser.add_argument(
@ -1240,7 +1247,7 @@ class Deploy(command.Command):
self._set_stack_action(parsed_args)
# Launch heat.
orchestration_client = self._launch_heat(parsed_args)
orchestration_client = self._launch_heat(parsed_args, output_dir)
# Wait for heat to be ready.
utils.wait_api_port_ready(parsed_args.heat_api_port)
# Deploy TripleO Heat templates.