From 6a0adcbbf6daa941a264fc0f1b575cb5dbb68678 Mon Sep 17 00:00:00 2001 From: James Slagle Date: Wed, 7 Jul 2021 07:34:37 -0400 Subject: [PATCH] Unmount heat-all tmpfs dir when done Adds a step to unmount the tmpfs dir used by the HeatNativeLauncher (heat-all aio process) when the process is stopped. Previously, the mount was kept and would hang around until the next reboot. As part of this change, the following is also included: - The tmpfs mount dir is also changed to be at the generated temporary directory within the heat launcher dir (heat_dir), instead of one level higher up. This provides a location to backup the files before unmounting. - The default value for the heat_dir directory is also changed from /var/log/heat_launcher to ~/tripleo-deploy//heat_launcher, so that the artifacts are saved in the generated tarball. - heat-all is started as the deployment user ($SUDO_USER, e.g. stack) instead of heat. There is no reason to run heat-all as heat. This enables keeping the generated files within the home directory of the deployment user - The heat_dir directory, owned by root, has it's mode changed from 700 to 755. This is safe as the directory is now in the deployment user home directory. This is necessary so that the deployment user can read the directory. - A step to backup the heat-all files is added such that the files are not lost when the unmount is done, and they are saved as part of the deployment artifacts. Signed-off-by: James Slagle Change-Id: I751fc605cb09ad9fcf169b3e2eab0532feb379a5 --- tripleoclient/heat_launcher.py | 62 +++++++++++-------- tripleoclient/tests/test_heat_launcher.py | 9 ++- .../tests/v1/tripleo/test_tripleo_deploy.py | 6 +- tripleoclient/utils.py | 7 ++- tripleoclient/v1/tripleo_deploy.py | 33 ++++++---- 5 files changed, 70 insertions(+), 47 deletions(-) diff --git a/tripleoclient/heat_launcher.py b/tripleoclient/heat_launcher.py index d16544e54..fe79aa625 100644 --- a/tripleoclient/heat_launcher.py +++ b/tripleoclient/heat_launcher.py @@ -22,6 +22,7 @@ import logging import multiprocessing import os import pwd +import shutil import signal import subprocess import tarfile @@ -148,38 +149,33 @@ class HeatBaseLauncher(object): self.skip_heat_pull = skip_heat_pull self.zipped_db_suffix = '.tar.bzip2' self.log_dir = os.path.join(self.heat_dir, 'log') + self.use_tmp_dir = use_tmp_dir - if os.path.isdir(self.heat_dir): - if use_root: - # This one may fail but it's just cleanup. - p = subprocess.Popen(['umount', self.heat_dir], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - cmd_stdout, cmd_stderr = p.communicate() - retval = p.returncode - if retval != 0: - log.info('Cleanup unmount of %s failed (probably because ' - 'it was not mounted): %s' % - (self.heat_dir, cmd_stderr)) - else: - log.info('umount of %s success' % (self.heat_dir)) - else: + if not os.path.isdir(self.heat_dir): # Create the directory if it doesn't exist. try: - os.makedirs(self.heat_dir, mode=0o700) + os.makedirs(self.heat_dir, mode=0o755) except Exception as e: log.error('Creating temp directory "%s" failed: %s' % (self.heat_dir, e)) raise Exception('Could not create temp directory %s: %s' % (self.heat_dir, e)) + if self.use_tmp_dir: + self.install_dir = tempfile.mkdtemp( + prefix='%s/tripleo_deploy-' % self.heat_dir) + else: + self.install_dir = self.heat_dir + if use_root: + self.umount_install_dir() + + if use_root and use_tmp_dir: # As an optimization we mount the tmp directory in a tmpfs (in # memory) filesystem. Depending on your system this can cut the # heat deployment times by half. p = subprocess.Popen(['mount', '-t', 'tmpfs', '-o', 'size=500M', - 'tmpfs', self.heat_dir], + 'tmpfs', self.install_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) @@ -192,12 +188,6 @@ class HeatBaseLauncher(object): 'database %s: %s' % (self.heat_dir, cmd_stderr)) - if use_tmp_dir: - self.install_dir = tempfile.mkdtemp( - prefix='%s/undercloud_deploy-' % self.heat_dir) - else: - self.install_dir = self.heat_dir - self.log_file = self._get_log_file_path() self.sql_db = os.path.join(self.install_dir, 'heat.sqlite') self.config_file = os.path.join(self.install_dir, 'heat.conf') @@ -221,6 +211,21 @@ class HeatBaseLauncher(object): self.kill_heat(None) self.rm_heat() + def umount_install_dir(self): + # This one may fail but it's just cleanup. + p = subprocess.Popen(['umount', self.install_dir], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + cmd_stdout, cmd_stderr = p.communicate() + retval = p.returncode + if retval != 0: + log.info('Cleanup unmount of %s failed (probably because ' + 'it was not mounted): %s' % + (self.heat_dir, cmd_stderr)) + else: + log.info('umount of %s success' % (self.heat_dir)) + def _get_log_file_path(self): return os.path.join(self.install_dir, 'heat.log') @@ -437,12 +442,19 @@ class HeatNativeLauncher(HeatBaseLauncher): def launch_heat(self): os.execvp('heat-all', ['heat-all', '--config-file', self.config_file]) - def heat_db_sync(self): + def heat_db_sync(self, restore_db=False): subprocess.check_call(['heat-manage', '--config-file', self.config_file, 'db_sync']) def kill_heat(self, pid): os.kill(pid, signal.SIGKILL) + if self.use_tmp_dir: + shutil.copytree( + self.install_dir, + os.path.join(self.heat_dir, + 'tripleo_deploy-%s' % self.timestamp)) + self.umount_install_dir() + shutil.rmtree(self.install_dir) class HeatPodLauncher(HeatContainerLauncher): diff --git a/tripleoclient/tests/test_heat_launcher.py b/tripleoclient/tests/test_heat_launcher.py index 7c117ec45..170811a4f 100644 --- a/tripleoclient/tests/test_heat_launcher.py +++ b/tripleoclient/tests/test_heat_launcher.py @@ -539,6 +539,7 @@ class TestHeatPodLauncherUtils(base.TestCase): utils._local_orchestration_client = None mock_launcher = mock.Mock() mock_launcher.api_port = 1234 + mock_launcher.heat_type = 'pod' mock_get_heat_launcher.return_value = mock_launcher mock_socket.return_value = 'socket' utils.launch_heat() @@ -594,10 +595,12 @@ class TestHeatNativeLauncher(base.TestCase): mock_mkdtemp.return_value = self.tmp_dir def test_install_dir(): - mock_mkdtemp.assert_not_called() + mock_mkdtemp.assert_called() return ("", "") - # Test that tempfile.mkdtemp is *not* called before the tmpfs is setup, - # otherwise the tmpfs will cause the temp dir to be lost + # Test that tempfile.mkdtemp is called before the tmpfs is setup, + # so that the tmpfs mount is created at the temp dir. self.mock_popen.communicate.side_effect = test_install_dir self.get_launcher() + self.assertEqual(['mount', '-t', 'tmpfs'], + self.popen.call_args_list[1][0][0][0:3]) diff --git a/tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py b/tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py index c837d7ee1..23ea2077f 100644 --- a/tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py +++ b/tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py @@ -768,7 +768,7 @@ class TestDeployUndercloud(TestPluginV1): self.cmd.take_action(parsed_args) mock_createdirs.assert_called_once() mock_puppet.assert_called_once() - mock_launchheat.assert_called_with(parsed_args) + mock_launchheat.assert_called_with(parsed_args, self.cmd.output_dir) mock_tht.assert_called_once_with(self.cmd, fake_orchestration, parsed_args) mock_download.assert_called_with(self.cmd, fake_orchestration, @@ -864,7 +864,7 @@ class TestDeployUndercloud(TestPluginV1): self.cmd.take_action, parsed_args) mock_createdirs.assert_called_once() mock_puppet.assert_called_once() - mock_launchheat.assert_called_with(parsed_args) + mock_launchheat.assert_called_with(parsed_args, self.cmd.output_dir) mock_tht.assert_called_once_with(self.cmd, fake_orchestration, parsed_args) mock_download.assert_called_with(self.cmd, fake_orchestration, @@ -944,7 +944,7 @@ class TestDeployUndercloud(TestPluginV1): self.cmd.take_action, parsed_args) mock_createdirs.assert_called_once() mock_puppet.assert_called_once() - mock_launchheat.assert_called_with(parsed_args) + mock_launchheat.assert_called_with(parsed_args, self.cmd.output_dir) mock_tht.assert_not_called() mock_download.assert_not_called() mock_tarball.assert_called_once() diff --git a/tripleoclient/utils.py b/tripleoclient/utils.py index 1ca0b4e43..37bd8587a 100644 --- a/tripleoclient/utils.py +++ b/tripleoclient/utils.py @@ -2612,7 +2612,7 @@ def write_user_environment(env_map, abs_env_path, tht_root, return user_env_path -def launch_heat(launcher=None, restore_db=False): +def launch_heat(launcher=None, restore_db=False, heat_type='pod'): global _local_orchestration_client global _heat_pid @@ -2622,7 +2622,7 @@ def launch_heat(launcher=None, restore_db=False): return _local_orchestration_client if not launcher: - launcher = get_heat_launcher() + launcher = get_heat_launcher(heat_type) _heat_pid = 0 if launcher.heat_type == 'native': @@ -2636,7 +2636,8 @@ def launch_heat(launcher=None, restore_db=False): # Wait for the API to be listening heat_api_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) test_heat_api_port(heat_api_socket, launcher.host, int(launcher.api_port)) - launcher.wait_for_message_queue() + if launcher.heat_type == 'pod': + launcher.wait_for_message_queue() _local_orchestration_client = tc_heat_utils.local_orchestration_client( launcher.host, launcher.api_port) diff --git a/tripleoclient/v1/tripleo_deploy.py b/tripleoclient/v1/tripleo_deploy.py index 85aec2b21..2c4da349a 100644 --- a/tripleoclient/v1/tripleo_deploy.py +++ b/tripleoclient/v1/tripleo_deploy.py @@ -449,19 +449,27 @@ class Deploy(command.Command): pid, ret = os.waitpid(self.heat_pid, 0) self.heat_pid = None - def _launch_heat(self, parsed_args): + def _launch_heat(self, parsed_args, output_dir): # we do this as root to chown config files properly for docker, etc. + heat_launcher_path = os.path.join(output_dir, 'heat_launcher') + + if parsed_args.heat_user: + heat_user = parsed_args.heat_user + else: + heat_user = parsed_args.deployment_user + if parsed_args.heat_native is not None and \ parsed_args.heat_native.lower() == "false": self.heat_launch = heat_launcher.HeatContainerLauncher( - parsed_args.heat_api_port, - parsed_args.heat_container_image, - parsed_args.heat_user) + api_port=parsed_args.heat_api_port, + all_container_image=parsed_args.heat_container_image, + user=heat_user, + heat_dir=heat_launcher_path) else: self.heat_launch = heat_launcher.HeatNativeLauncher( - parsed_args.heat_api_port, - parsed_args.heat_container_image, - parsed_args.heat_user, + api_port=parsed_args.heat_api_port, + user=heat_user, + heat_dir=heat_launcher_path, use_root=True) # NOTE(dprince): we launch heat with fork exec because @@ -474,12 +482,12 @@ class Deploy(command.Command): if parsed_args.heat_native is not None and \ parsed_args.heat_native.lower() == "true": try: - uid = pwd.getpwnam(parsed_args.heat_user).pw_uid - gid = pwd.getpwnam(parsed_args.heat_user).pw_gid + uid = pwd.getpwnam(heat_user).pw_uid + gid = pwd.getpwnam(heat_user).pw_gid except KeyError: msg = _( "Please create a %s user account before " - "proceeding.") % parsed_args.heat_user + "proceeding.") % heat_user self.log.error(msg) raise exceptions.DeploymentError(msg) os.setgid(gid) @@ -948,9 +956,8 @@ class Deploy(command.Command): parser.add_argument( '--heat-user', metavar='', dest='heat_user', - default='heat', help=_('User to execute the non-privileged heat-all process. ' - 'Defaults to heat.') + 'Defaults to the value of --deployment-user.') ) # TODO(cjeanner) drop that once using oslo.privsep parser.add_argument( @@ -1234,7 +1241,7 @@ class Deploy(command.Command): self._set_stack_action(parsed_args) # Launch heat. - orchestration_client = self._launch_heat(parsed_args) + orchestration_client = self._launch_heat(parsed_args, output_dir) # Wait for heat to be ready. utils.wait_api_port_ready(parsed_args.heat_api_port) # Deploy TripleO Heat templates.