diff --git a/templates/ephemeral-heat/heat.conf.j2 b/templates/ephemeral-heat/heat.conf.j2 index 83152de98..9f20d69d6 100644 --- a/templates/ephemeral-heat/heat.conf.j2 +++ b/templates/ephemeral-heat/heat.conf.j2 @@ -6,6 +6,7 @@ default_deployment_signal_transport = HEAT_SIGNAL deferred_auth_method = password keystone_backend = heat.engine.clients.os.keystone.fake_keystoneclient.FakeKeystoneClient log_dir = /var/log/heat +log_file = {{ log_file }} max_json_body_size = 8388608 max_nested_stack_depth = 10 max_resources_per_stack=-1 diff --git a/tripleoclient/heat_launcher.py b/tripleoclient/heat_launcher.py index bf3e655a3..a15cfcec2 100644 --- a/tripleoclient/heat_launcher.py +++ b/tripleoclient/heat_launcher.py @@ -13,6 +13,7 @@ # under the License. # +import configparser import datetime import glob import grp @@ -23,7 +24,9 @@ import os import pwd import signal import subprocess +import tarfile import tempfile +import time import jinja2 from oslo_utils import timeutils @@ -139,29 +142,27 @@ class HeatBaseLauncher(object): self.engine_container_image = engine_container_image self.heat_dir = os.path.abspath(heat_dir) self.host = "127.0.0.1" - self.db_dump_path = os.path.join( - self.heat_dir, 'heat-db-dump-{}.sql'.format( - datetime.datetime.utcnow().isoformat())) + self.timestamp = time.time() + self.db_dump_path = os.path.join(self.heat_dir, 'heat-db.sql') self.skip_heat_pull = skip_heat_pull - - if rm_heat: - self.kill_heat(None) - self.rm_heat() + self.zipped_db_suffix = '.tar.bzip2' + self.log_dir = os.path.join(self.heat_dir, 'log') if os.path.isdir(self.heat_dir): - # This one may fail but it's just cleanup. - p = subprocess.Popen(['umount', self.heat_dir], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - cmd_stdout, cmd_stderr = p.communicate() - retval = p.returncode - if retval != 0: - log.info('Cleanup unmount of %s failed (probably because ' - 'it was not mounted): %s' % - (self.heat_dir, cmd_stderr)) - else: - log.info('umount of %s success' % (self.heat_dir)) + if use_root: + # This one may fail but it's just cleanup. + p = subprocess.Popen(['umount', self.heat_dir], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + cmd_stdout, cmd_stderr = p.communicate() + retval = p.returncode + if retval != 0: + log.info('Cleanup unmount of %s failed (probably because ' + 'it was not mounted): %s' % + (self.heat_dir, cmd_stderr)) + else: + log.info('umount of %s success' % (self.heat_dir)) else: # Create the directory if it doesn't exist. try: @@ -171,36 +172,40 @@ class HeatBaseLauncher(object): (self.heat_dir, e)) raise Exception('Could not create temp directory %s: %s' % (self.heat_dir, e)) - # As an optimization we mount the tmp directory in a tmpfs (in memory) - # filesystem. Depending on your system this can cut the heat - # deployment times by half. - p = subprocess.Popen(['mount', '-t', 'tmpfs', '-o', 'size=500M', - 'tmpfs', self.heat_dir], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - cmd_stdout, cmd_stderr = p.communicate() - retval = p.returncode - if retval != 0: - # It's ok if this fails, it will still work. It just won't - # be on tmpfs. - log.warning('Unable to mount tmpfs for logs and ' - 'database %s: %s' % - (self.heat_dir, cmd_stderr)) - self.policy_file = os.path.join(os.path.dirname(__file__), - 'noauth_policy.json') + if use_root: + # As an optimization we mount the tmp directory in a tmpfs (in + # memory) filesystem. Depending on your system this can cut the + # heat deployment times by half. + p = subprocess.Popen(['mount', '-t', 'tmpfs', '-o', 'size=500M', + 'tmpfs', self.heat_dir], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + cmd_stdout, cmd_stderr = p.communicate() + retval = p.returncode + if retval != 0: + # It's ok if this fails, it will still work. It just won't + # be on tmpfs. + log.warning('Unable to mount tmpfs for logs and ' + 'database %s: %s' % + (self.heat_dir, cmd_stderr)) + if use_tmp_dir: self.install_dir = tempfile.mkdtemp( prefix='%s/undercloud_deploy-' % self.heat_dir) else: self.install_dir = self.heat_dir - self.user = user + + self.log_file = self._get_log_file_path() self.sql_db = os.path.join(self.install_dir, 'heat.sqlite') - self.log_file = os.path.join(self.install_dir, 'heat.log') self.config_file = os.path.join(self.install_dir, 'heat.conf') self.paste_file = os.path.join(self.install_dir, 'api-paste.ini') self.token_file = os.path.join(self.install_dir, 'token_file.json') + + self.policy_file = os.path.join(os.path.dirname(__file__), + 'noauth_policy.json') + self.user = user self._write_fake_keystone_token(self.api_port, self.token_file) self._write_heat_config() self._write_api_paste_config() @@ -211,6 +216,13 @@ class HeatBaseLauncher(object): os.chown(self.config_file, uid, gid) os.chown(self.paste_file, uid, gid) + if rm_heat: + self.kill_heat(None) + self.rm_heat() + + def _get_log_file_path(self): + return os.path.join(self.install_dir, 'heat.log') + def _write_heat_config(self): # TODO(ksambor) It will be nice to have possibilities to configure heat heat_config = ''' @@ -300,6 +312,20 @@ heat.filter_factory = heat.api.openstack:faultwrap_filter def check_message_bus(self): return True + def tar_file(self, file_path, cleanup=True): + tf_name = '{}-{}.tar.bzip2'.format(file_path, self.timestamp) + tf = tarfile.open(tf_name, 'w:bz2') + tf.add(file_path, os.path.basename(file_path)) + tf.close() + log.info("Created tarfile {}".format(tf_name)) + if cleanup: + log.info("Deleting {}".format(file_path)) + os.unlink(file_path) + + def untar_file(self, tar_path, extract_dir): + tf = tarfile.open(tar_path, 'r:bz2') + tf.extractall(extract_dir) + class HeatContainerLauncher(HeatBaseLauncher): @@ -385,7 +411,7 @@ class HeatContainerLauncher(HeatBaseLauncher): return result.split(':')[2] raise Exception('Could not find heat gid') - def kill_heat(self, pid, backup_db=False): + def kill_heat(self, pid): cmd = ['podman', 'stop', 'heat_all'] log.debug(' '.join(cmd)) # We don't want to hear from this command.. @@ -413,7 +439,7 @@ class HeatNativeLauncher(HeatBaseLauncher): subprocess.check_call(['heat-manage', '--config-file', self.config_file, 'db_sync']) - def kill_heat(self, pid, backup_db=False): + def kill_heat(self, pid): os.kill(pid, signal.SIGKILL) @@ -423,9 +449,8 @@ class HeatPodLauncher(HeatContainerLauncher): def __init__(self, *args, **kwargs): super(HeatPodLauncher, self).__init__(*args, **kwargs) - log_dir = os.path.join(self.heat_dir, 'log') - if not os.path.isdir(log_dir): - os.makedirs(log_dir) + if not os.path.isdir(self.log_dir): + os.makedirs(self.log_dir) self.host = self._get_ctlplane_ip() self._chcon() @@ -449,14 +474,17 @@ class HeatPodLauncher(HeatContainerLauncher): raise Exception('Unable to fetch container image {}.' 'Error: {}'.format(image, e)) - def launch_heat(self): + def get_pod_state(self): inspect = subprocess.run([ 'sudo', 'podman', 'pod', 'inspect', '--format', '"{{.State}}"', 'ephemeral-heat'], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if "Running" in self._decode(inspect.stdout): + return self._decode(inspect.stdout) + + def launch_heat(self): + if "Running" in self.get_pod_state(): log.info("ephemeral-heat pod already running, skipping launch") return self._write_heat_pod() @@ -503,16 +531,25 @@ class HeatPodLauncher(HeatContainerLauncher): def do_restore_db(self, db_dump_path=None): if not db_dump_path: # Find the latest dump from self.heat_dir - db_dumps = glob.glob('{}/heat-db-dump*'.format(self.heat_dir)) + db_dumps = glob.glob( + '{}/heat-db-dump*{}'.format + (self.heat_dir, + self.zipped_db_suffix)) if not db_dumps: raise Exception('No db backups found to restore in %s' % self.heat_dir) db_dump_path = max(db_dumps, key=os.path.getmtime) + self.untar_file(db_dump_path, self.heat_dir) + db_dump_path = db_dump_path.rstrip(self.zipped_db_suffix) log.info("Restoring db from {}".format(db_dump_path)) - subprocess.run([ - 'sudo', 'podman', 'exec', '-u', 'root', - 'mysql', 'mysql', 'heat'], stdin=open(db_dump_path), - check=True) + try: + with open(db_dump_path) as f: + subprocess.run([ + 'sudo', 'podman', 'exec', '-u', 'root', + 'mysql', 'mysql', 'heat'], stdin=f, + check=True) + finally: + os.unlink(db_dump_path) def do_backup_db(self, db_dump_path=None): if not db_dump_path: @@ -520,13 +557,26 @@ class HeatPodLauncher(HeatContainerLauncher): if os.path.exists(db_dump_path): raise Exception("Won't overwrite existing db dump at %s. " "Remove it first." % db_dump_path) + log.info("Starting back up of heat db") with open(db_dump_path, 'w') as out: subprocess.run([ 'sudo', 'podman', 'exec', '-u', 'root', 'mysql', 'mysqldump', 'heat'], stdout=out, check=True) - def rm_heat(self, backup_db=False): + self.tar_file(db_dump_path) + + def pod_exists(self): + try: + subprocess.check_call( + ['sudo', 'podman', 'pod', 'inspect', 'ephemeral-heat'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + return True + except subprocess.CalledProcessError: + return False + + def rm_heat(self, backup_db=True): if self.database_exists(): if backup_db: self.do_backup_db() @@ -541,14 +591,24 @@ class HeatPodLauncher(HeatContainerLauncher): 'drop user \'heat\'@\'%\'']) except subprocess.CalledProcessError: pass - subprocess.call([ - 'sudo', 'podman', 'pod', 'rm', '-f', 'ephemeral-heat' - ]) + if self.pod_exists(): + log.info("Removing pod: ephemeral-heat") + subprocess.call([ + 'sudo', 'podman', 'pod', 'rm', '-f', 'ephemeral-heat' + ]) + config = self._read_heat_config() + log_file_path = os.path.join(self.log_dir, + config['DEFAULT']['log_file']) + if os.path.exists(log_file_path): + self.tar_file(log_file_path) def stop_heat(self): - subprocess.check_call([ - 'sudo', 'podman', 'pod', 'stop', 'ephemeral-heat' - ]) + if self.pod_exists() and self.get_pod_state() != 'Exited': + log.info("Stopping pod: ephemeral-heat") + subprocess.check_call([ + 'sudo', 'podman', 'pod', 'stop', 'ephemeral-heat' + ]) + log.info("Stopped pod: ephemeral-heat") def check_message_bus(self): log.info("Checking that message bus (rabbitmq) is up") @@ -587,10 +647,15 @@ class HeatPodLauncher(HeatContainerLauncher): ]) return 'heat' in str(output) - def kill_heat(self, pid, backup_db=False): - subprocess.call([ - 'sudo', 'podman', 'pod', 'kill', 'ephemeral-heat' - ]) + def kill_heat(self, pid): + if self.pod_exists(): + log.info("Killing pod: ephemeral-heat") + subprocess.call([ + 'sudo', 'podman', 'pod', 'kill', 'ephemeral-heat' + ]) + log.info("Killed pod: ephemeral-heat") + else: + log.info("Pod does not exist: ephemeral-heat") def _decode(self, encoded): if not encoded: @@ -644,6 +709,14 @@ class HeatPodLauncher(HeatContainerLauncher): msg = "Message queue for ephemeral heat not created in time." raise HeatPodMessageQueueException(msg) + def _get_log_file_path(self): + return 'heat-{}.log'.format(self.timestamp) + + def _read_heat_config(self): + config = configparser.ConfigParser() + config.read(self.config_file) + return config + def _write_heat_config(self): heat_config_tmpl_path = os.path.join(DEFAULT_TEMPLATES_DIR, "ephemeral-heat", @@ -656,6 +729,7 @@ class HeatPodLauncher(HeatContainerLauncher): "db_connection": self._get_db_connection(), "api_port": self.api_port, "num_engine_workers": self._get_num_engine_workers(), + "log_file": self.log_file, } heat_config = heat_config_tmpl.render(**config_vars) diff --git a/tripleoclient/tests/test_heat_launcher.py b/tripleoclient/tests/test_heat_launcher.py new file mode 100644 index 000000000..19b7addfc --- /dev/null +++ b/tripleoclient/tests/test_heat_launcher.py @@ -0,0 +1,603 @@ +# Copyright 2021 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures +import mock +import os +from pathlib import Path +import shutil +import subprocess +import time + +from tripleoclient import heat_launcher +from tripleoclient.exceptions import HeatPodMessageQueueException +from tripleoclient.tests import base +from tripleoclient import utils + + +class TestHeatPodLauncher(base.TestCase): + def setUp(self): + super(TestHeatPodLauncher, self).setUp() + self.run = mock.patch('subprocess.run').start() + self.call = mock.patch('subprocess.call').start() + self.check_call = mock.patch('subprocess.check_call').start() + self.check_output = mock.patch('subprocess.check_output').start() + self.templates_dir = mock.patch( + 'tripleoclient.heat_launcher.DEFAULT_TEMPLATES_DIR', + os.path.join(os.path.dirname(__file__), + '..', '..', 'templates')).start() + self.heat_dir = self.useFixture(fixtures.TempDir()).path + + self.addCleanup(mock.patch.stopall) + + def get_launcher(self, **kwargs): + return heat_launcher.HeatPodLauncher( + heat_dir=self.heat_dir, + use_tmp_dir=False, + **kwargs) + + def check_calls(self, check_call, mock_obj): + for call in mock_obj.call_args_list: + call_str = ' '.join(call.args[0]) + if check_call in call_str: + return True + return False + + def test_rm_heat_launcher(self): + self.assertIsInstance(self.get_launcher(rm_heat=True), + heat_launcher.HeatPodLauncher) + + def test_chcon(self): + launcher = self.get_launcher() + launcher._chcon() + self.check_calls('chcon', self.check_call) + self.check_calls(launcher.heat_dir, self.check_call) + + def test_fetch_container_image(self): + launcher = self.get_launcher(skip_heat_pull=True) + launcher._fetch_container_image() + self.assertFalse(self.check_calls('podman pull', self.check_output)) + + launcher = self.get_launcher(skip_heat_pull=False) + launcher._fetch_container_image() + self.assertTrue(self.check_calls('podman pull', self.check_output)) + + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher._decode') + def test_get_pod_state(self, mock_decode): + launcher = self.get_launcher() + launcher.get_pod_state() + self.check_calls('podman pod inspect', self.run) + self.assertTrue(mock_decode.called) + + @mock.patch( + 'tripleoclient.heat_launcher.HeatPodLauncher._write_heat_config') + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher._write_heat_pod') + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.get_pod_state') + def test_lauch_heat( + self, mock_get_pod_state, mock_write_heat_pod, + mock_write_heat_config): + + launcher = self.get_launcher() + + mock_get_pod_state.return_value = 'Running' + launcher.launch_heat() + self.assertFalse(mock_write_heat_pod.called) + self.assertFalse(self.check_calls('podman play kube', self.check_call)) + + mock_get_pod_state.return_value = 'Exited' + launcher.launch_heat() + self.assertTrue(mock_write_heat_pod.called) + self.assertTrue(self.check_calls('podman play kube', self.check_call)) + + mock_get_pod_state.return_value = '' + launcher.launch_heat() + self.assertTrue(mock_write_heat_pod.called) + self.assertTrue(self.check_calls('podman play kube', self.check_call)) + + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.do_restore_db') + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.database_exists') + def test_heat_db_sync( + self, mock_db_exists, mock_do_restore_db): + + launcher = self.get_launcher() + mock_db_exists.return_value = True + launcher.heat_db_sync(restore_db=False) + self.assertFalse(self.check_calls('create database', self.check_call)) + self.assertFalse(self.check_calls('create user', self.check_call)) + self.assertFalse(self.check_calls('grant all', self.check_call)) + self.assertFalse(self.check_calls('flush priv', self.check_call)) + self.assertTrue(self.check_calls('heat-manage', self.check_call)) + self.assertFalse(mock_do_restore_db.called) + + self.check_call.reset_mock() + + mock_db_exists.return_value = True + launcher.heat_db_sync(restore_db=True) + self.assertFalse(self.check_calls('create database', self.check_call)) + self.assertFalse(self.check_calls('create user', self.check_call)) + self.assertFalse(self.check_calls('grant all', self.check_call)) + self.assertFalse(self.check_calls('flush priv', self.check_call)) + self.assertTrue(self.check_calls('heat-manage', self.check_call)) + self.assertTrue(mock_do_restore_db.called) + + self.check_call.reset_mock() + mock_db_exists.return_value = False + launcher.heat_db_sync(restore_db=True) + self.assertTrue(self.check_calls('create database', self.check_call)) + self.assertTrue(self.check_calls('create user', self.check_call)) + self.assertTrue(self.check_calls('grant all', self.check_call)) + self.assertTrue(self.check_calls('flush priv', self.check_call)) + self.assertTrue(self.check_calls('heat-manage', self.check_call)) + self.assertTrue(mock_do_restore_db.called) + + self.check_call.reset_mock() + mock_do_restore_db.reset_mock() + mock_db_exists.return_value = False + launcher.heat_db_sync(restore_db=False) + self.assertTrue(self.check_calls('create database', self.check_call)) + self.assertTrue(self.check_calls('create user', self.check_call)) + self.assertTrue(self.check_calls('grant all', self.check_call)) + self.assertTrue(self.check_calls('flush priv', self.check_call)) + self.assertTrue(self.check_calls('heat-manage', self.check_call)) + self.assertFalse(mock_do_restore_db.called) + + @mock.patch('os.unlink') + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.untar_file') + @mock.patch('glob.glob') + def test_do_restore_db( + self, mock_glob, mock_untar, mock_unlink): + + launcher = self.get_launcher() + + one = Path(os.path.join(launcher.heat_dir, + 'heat-db-dump-one.tar.bz2')) + two = Path(os.path.join(launcher.heat_dir, + 'heat-db-dump-two.tar.bz2')) + three = Path(os.path.join(launcher.heat_dir, + 'heat-db-dump-three.tar.bz2')) + + now = time.time() + one.touch() + two.touch() + three.touch() + os.utime(str(one), (now, 1000)) + os.utime(str(two), (now, 2000)) + os.utime(str(three), (now, 3000)) + mock_glob.return_value = [str(one), str(two), str(three)] + + def untar(path, dir): + p = Path(path.rstrip('.tar.bz2')) + p.touch() + + mock_untar.side_effect = untar + + mock_open = mock.mock_open() + with mock.patch('six.moves.builtins.open', mock_open): + # pylint: disable=bad-str-strip-call + launcher.do_restore_db() + self.assertEqual(mock.call(str(three), launcher.heat_dir), + mock_untar.call_args) + self.assertEqual(mock.call(str(three).rstrip('.tar.bz2')), + mock_unlink.call_args) + mock_open.assert_called_with(str(three).rstrip('.tar.bz2')) # noqa + self.assertTrue(self.check_call('mysql heat', self.run)) + + mock_unlink.reset_mock() + self.run.reset_mock() + two.touch() + mock_open = mock.mock_open() + with mock.patch('six.moves.builtins.open', mock_open): + # pylint: disable=bad-str-strip-call + launcher.do_restore_db() + self.assertEqual(mock.call(str(two), launcher.heat_dir), + mock_untar.call_args) + self.assertEqual(mock.call(str(two).rstrip('.tar.bz2')), # noqa + mock_unlink.call_args) + mock_open.assert_called_with(str(two).rstrip('.tar.bz2')) # noqa + self.assertTrue(self.check_call('mysql heat', self.run)) + + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.tar_file') + def test_do_backup_db(self, mock_tar): + launcher = self.get_launcher() + p = Path(os.path.join(launcher.heat_dir, 'heat-db.sql')) + p.touch() + self.assertRaises(Exception, launcher.do_backup_db, str(p)) + + p.unlink() + launcher.do_backup_db() + mock_tar.assert_called_with(str(p)) + self.assertTrue(self.check_calls('mysqldump heat', self.run)) + + def test_pod_exists(self): + launcher = self.get_launcher() + self.assertTrue(launcher.pod_exists()) + self.check_calls('pod inspect', self.check_call) + + self.check_call.reset_mock() + self.check_call.side_effect = subprocess.CalledProcessError(1, 'test') + self.assertFalse(launcher.pod_exists()) + self.check_calls('pod inspect', self.check_call) + + @mock.patch('os.path.exists') + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.tar_file') + @mock.patch( + 'tripleoclient.heat_launcher.HeatPodLauncher._read_heat_config') + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.pod_exists') + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.do_backup_db') + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.database_exists') + def test_rm_heat(self, mock_db_exists, mock_backup_db, mock_pod_exists, + mock_read_heat_config, mock_tar, mock_exists): + + launcher = self.get_launcher() + launcher.log_dir = '/log' + + mock_db_exists.return_value = True + mock_pod_exists.return_value = True + mock_exists.return_value = True + mock_read_heat_config.return_value = { + 'DEFAULT': { + 'log_file': 'heat-log'}} + launcher.rm_heat() + mock_backup_db.assert_called() + self.check_calls('drop database heat', self.check_call) + self.check_calls('drop user', self.check_call) + mock_pod_exists.assert_called() + self.check_calls('podman pod rm -f', self.call) + mock_read_heat_config.assert_called() + mock_tar.assert_called_with('/log/heat-log') + + mock_backup_db.reset_mock() + self.call.reset_mock() + mock_tar.reset_mock() + mock_db_exists.return_value = False + mock_pod_exists.return_value = False + mock_exists.return_value = False + launcher.rm_heat() + mock_backup_db.assert_not_called() + self.call.assert_not_called() + mock_tar.assert_not_called() + + mock_backup_db.reset_mock() + self.call.reset_mock() + mock_tar.reset_mock() + mock_exists.reset_mock() + mock_db_exists.return_value = False + mock_pod_exists.return_value = True + mock_exists.return_value = True + launcher.rm_heat(backup_db=False) + mock_backup_db.assert_not_called() + self.check_calls('podman pod rm -f', self.call) + + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.get_pod_state') + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.pod_exists') + def test_stop_heat(self, mock_pod_exists, mock_pod_state): + launcher = self.get_launcher() + mock_pod_exists.return_value = True + mock_pod_state.return_value = 'Running' + launcher.stop_heat() + mock_pod_exists.assert_called() + mock_pod_state.assert_called() + self.check_calls('podman pod stop', self.check_call) + + self.check_call.reset_mock() + mock_pod_exists.reset_mock() + mock_pod_state.reset_mock() + mock_pod_state.return_value = 'Exited' + mock_pod_exists.return_value = True + launcher.stop_heat() + mock_pod_exists.assert_called() + mock_pod_state.assert_called() + self.check_call.assert_not_called() + + self.check_call.reset_mock() + mock_pod_exists.reset_mock() + mock_pod_state.reset_mock() + mock_pod_state.return_value = 'Exited' + mock_pod_exists.return_value = False + launcher.stop_heat() + mock_pod_exists.assert_called() + mock_pod_state.assert_not_called() + self.check_call.assert_not_called() + + def test_check_message_bus(self): + launcher = self.get_launcher() + launcher.check_message_bus() + self.check_calls('rabbitmqctl list_queues', self.check_call) + + self.check_call.reset_mock() + self.check_call.side_effect = subprocess.CalledProcessError(1, 'test') + self.assertRaises(subprocess.CalledProcessError, + launcher.check_message_bus) + + @mock.patch( + 'tripleoclient.heat_launcher.HeatPodLauncher._get_ctlplane_ip') + def test_check_database(self, mock_ctlplane_ip): + launcher = self.get_launcher() + + mock_ctlplane_ip.return_value = '1.1.1.1' + self.assertTrue(launcher.check_database()) + mock_ctlplane_ip.assert_called() + self.check_calls('show databases', self.check_call) + + self.check_call.reset_mock() + mock_ctlplane_ip.reset_mock() + self.check_call.side_effect = subprocess.CalledProcessError(1, '/test') + self.assertRaises(subprocess.CalledProcessError, + launcher.check_database) + + def test_database_exists(self): + launcher = self.get_launcher() + self.check_output.return_value = 'heat' + self.assertTrue(launcher.database_exists()) + self.check_calls('show databases like "heat"', self.check_output) + + self.check_output.reset_mock() + self.check_output.return_value = 'nova' + self.assertFalse(launcher.database_exists()) + self.check_calls('show databases like "heat"', self.check_output) + + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.pod_exists') + def test_kill_heat(self, mock_pod_exists): + launcher = self.get_launcher() + mock_pod_exists.return_value = True + launcher.kill_heat(0) + self.check_calls('podman pod kill', self.call) + mock_pod_exists.assert_called() + + mock_pod_exists.reset_mock() + self.call.reset_mock() + mock_pod_exists.return_value = False + launcher.kill_heat(0) + mock_pod_exists.assert_called() + self.call.assert_not_called() + + def test_decode(self): + launcher = self.get_launcher() + mock_encoded = mock.Mock() + mock_decoded = mock.Mock() + mock_encoded.decode.return_value = mock_decoded + mock_decoded.endswith.return_value = False + launcher._decode(mock_encoded) + mock_encoded.decode.assert_called_with('utf-8') + + self.assertEqual('test', launcher._decode(b'test\n')) + + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher._decode') + def test_get_transport_url(self, mock_decode): + launcher = self.get_launcher() + mock_decode.side_effect = ['user', 'password', 'fqdn_ctlplane', 'port'] + self.assertEqual("rabbit://user:password@fqdn_ctlplane:port/?ssl=0", + launcher._get_transport_url()) + + @mock.patch( + 'tripleoclient.heat_launcher.HeatPodLauncher._get_ctlplane_vip') + def test_get_db_connection(self, mock_ctlplane_vip): + launcher = self.get_launcher() + mock_ctlplane_vip.return_value = '1.1.1.1' + self.assertEqual( + 'mysql+pymysql://' + 'heat:heat@1.1.1.1/heat?read_default_file=' + '/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo', + launcher._get_db_connection()) + + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher._decode') + def test_get_ctlplane_vip(self, mock_decode): + launcher = self.get_launcher() + self.check_output.return_value = '1.1.1.1' + launcher._get_ctlplane_vip() + self.check_calls('sudo hiera controller_virtual_ip', self.check_output) + mock_decode.assert_called_with('1.1.1.1') + + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher._decode') + def test_get_ctlplane_ip(self, mock_decode): + launcher = self.get_launcher() + self.check_output.return_value = '1.1.1.1' + launcher._get_ctlplane_ip() + self.check_calls('sudo hiera ctlplane', self.check_output) + mock_decode.assert_called_with('1.1.1.1') + + @mock.patch('multiprocessing.cpu_count') + def test_get_num_engine_workers(self, mock_cpu_count): + launcher = self.get_launcher() + mock_cpu_count.return_value = 4 + self.assertEqual(2, launcher._get_num_engine_workers()) + + def test_wait_for_message_queue(self): + launcher = self.get_launcher() + wait_mq = launcher.wait_for_message_queue.__wrapped__ + self.check_output.return_value = 'heat' + wait_mq(launcher) + + self.check_output.reset_mock() + self.check_output.return_value = 'test' + self.assertRaises(HeatPodMessageQueueException, wait_mq, launcher) + + def test_get_log_file_path(self): + launcher = self.get_launcher() + launcher.timestamp = '1111' + self.assertEqual('heat-1111.log', launcher._get_log_file_path()) + + @mock.patch('configparser.ConfigParser') + def test_read_heat_config(self, mock_config_parser): + launcher = self.get_launcher() + mock_cp = mock.Mock() + mock_cp.read.return_value = 'test' + mock_config_parser.return_value = mock_cp + self.assertEqual(mock_cp, launcher._read_heat_config()) + mock_config_parser.assert_called() + mock_cp.read.assert_called_with(launcher.config_file) + + @mock.patch('tripleoclient.heat_launcher.' + 'HeatPodLauncher._get_num_engine_workers') + @mock.patch( + 'tripleoclient.heat_launcher.HeatPodLauncher._get_db_connection') + @mock.patch( + 'tripleoclient.heat_launcher.HeatPodLauncher._get_transport_url') + def test_write_heat_config(self, mock_get_transport_url, mock_get_db_conn, + mock_num_engine_workers): + launcher = self.get_launcher() + launcher.api_port = '1234' + launcher.log_file = '/log/heat' + mock_get_transport_url.return_value = 'transport-url' + mock_get_db_conn.return_value = 'db-connection' + mock_num_engine_workers.return_value = 'num-engine-workers' + launcher._write_heat_config() + with open(launcher.config_file) as f: + config = f.read() + self.assertIn('num_engine_workers = num-engine-workers\n', config) + self.assertIn('connection = db-connection\n', config) + self.assertIn('transport_url=transport-url\n', config) + self.assertIn('bind_port = 1234\n', config) + self.assertIn('log_file = /log/heat\n', config) + + def test_write_heat_pod(self): + launcher = self.get_launcher() + launcher.install_dir = 'install-dir' + launcher.host = '1.1.1.1' + launcher.api_port = '1234' + launcher.api_container_image = 'api-image' + launcher.engine_container_image = 'engine-image' + launcher._write_heat_pod() + with open(os.path.join(launcher.heat_dir, 'heat-pod.yaml')) as f: + pod = f.read() + self.assertIn('hostPort: 1234', pod) + self.assertIn('hostIP: 1.1.1.1', pod) + self.assertIn('image: api-image', pod) + self.assertIn('image: engine-image', pod) + + +class TestHeatPodLauncherUtils(base.TestCase): + def setUp(self): + super(TestHeatPodLauncherUtils, self).setUp() + + def test_rm_heat(self): + launcher = mock.Mock() + utils.rm_heat(launcher) + launcher.rm_heat.assert_called_once_with(False) + launcher.reset_mock() + utils.rm_heat(launcher, True) + launcher.rm_heat.assert_called_once_with(True) + launcher.reset_mock() + utils.rm_heat(launcher, False) + launcher.rm_heat.assert_called_once_with(False) + + def test_kill_heat(self): + launcher = mock.Mock() + utils.kill_heat(launcher) + launcher.kill_heat.assert_called_once_with(None) + launcher.reset_mock() + utils._heat_pid = 111 + utils.kill_heat(launcher) + launcher.kill_heat.assert_called_once_with(111) + launcher.reset_mock() + utils.kill_heat(launcher) + launcher.kill_heat.assert_called_once_with(111) + launcher.reset_mock() + utils.kill_heat(launcher) + launcher.kill_heat.assert_called_once_with(111) + + @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher') + @mock.patch('tripleoclient.heat_launcher.HeatNativeLauncher') + @mock.patch('tripleoclient.heat_launcher.HeatContainerLauncher') + def test_get_heat_launcher(self, mock_container, mock_native, mock_pod): + utils.get_heat_launcher('pod', 1, 2, 3, a='a', b='b', c='c') + mock_pod.assert_called_once_with(1, 2, 3, a='a', b='b', c='c') + utils.get_heat_launcher('native', 1, 2, 3, a='a', b='b', c='c') + mock_native.assert_called_once_with(1, 2, 3, a='a', b='b', c='c') + utils.get_heat_launcher('container', 1, 2, 3, a='a', b='b', c='c') + mock_container.assert_called_once_with(1, 2, 3, a='a', b='b', c='c') + + def test_heat_api_port(self): + test_port = utils.test_heat_api_port.__wrapped__ + mock_socket = mock.Mock() + host = '1.1.1.1' + port = 1234 + test_port(mock_socket, host, port) + mock_socket.connect.assert_called_once_with((host, port)) + + @mock.patch('tripleoclient.utils.test_heat_api_port') + @mock.patch('tripleo_common.utils.heat.local_orchestration_client') + @mock.patch('socket.socket') + @mock.patch('tripleoclient.utils.get_heat_launcher') + def test_launch_heat(self, mock_get_heat_launcher, mock_socket, + mock_local_client, mock_test_port): + utils._local_orchestration_client = 'client' + self.assertEqual('client', utils.launch_heat()) + mock_get_heat_launcher.assert_not_called() + + utils._local_orchestration_client = None + mock_launcher = mock.Mock() + mock_launcher.api_port = 1234 + mock_get_heat_launcher.return_value = mock_launcher + mock_socket.return_value = 'socket' + utils.launch_heat() + mock_get_heat_launcher.assert_called_once() + mock_launcher.check_database.assert_called_once_with() + mock_launcher.check_message_bus.assert_called_once_with() + mock_launcher.heat_db_sync.assert_called_once_with(False) + mock_launcher.launch_heat.assert_called_once_with() + mock_test_port.assert_called_once_with( + 'socket', mock_launcher.host, + int(mock_launcher.api_port)) + mock_launcher.wait_for_message_queue.assert_called_once_with() + mock_local_client.assert_called_once_with( + mock_launcher.host, + mock_launcher.api_port) + + +class TestHeatNativeLauncher(base.TestCase): + def setUp(self): + super(TestHeatNativeLauncher, self).setUp() + self.run = mock.patch('subprocess.run').start() + self.popen = mock.patch('subprocess.Popen').start() + self.mock_popen = mock.Mock() + self.mock_popen.communicate.return_value = ("", "") + self.popen.return_value = self.mock_popen + self.getpwnam = mock.patch('pwd.getpwnam').start() + self.getgrnam = mock.patch('grp.getgrnam').start() + self.chown = mock.patch('os.chown').start() + + self.templates_dir = mock.patch( + 'tripleoclient.heat_launcher.DEFAULT_TEMPLATES_DIR', + os.path.join(os.path.dirname(__file__), + '..', '..', 'templates')).start() + self.heat_dir = self.useFixture(fixtures.TempDir()).path + self.tmp_dir = self.useFixture(fixtures.TempDir()).path + + self.addCleanup(mock.patch.stopall) + + def get_launcher(self, **kwargs): + return heat_launcher.HeatNativeLauncher( + heat_dir=self.heat_dir, + use_tmp_dir=True, + use_root=True, + **kwargs) + + def test_heat_dir_no_exist(self): + shutil.rmtree(self.heat_dir) + launcher = self.get_launcher() + self.assertNotEqual(self.heat_dir, launcher.install_dir) + + @mock.patch('tempfile.mkdtemp') + def test_get_launcher(self, mock_mkdtemp): + mock_mkdtemp.return_value = self.tmp_dir + + def test_install_dir(): + mock_mkdtemp.assert_not_called() + return ("", "") + + # Test that tempfile.mkdtemp is *not* called before the tmpfs is setup, + # otherwise the tmpfs will cause the temp dir to be lost + self.mock_popen.communicate.side_effect = test_install_dir + self.get_launcher() diff --git a/tripleoclient/utils.py b/tripleoclient/utils.py index 931e3d661..f29c11017 100644 --- a/tripleoclient/utils.py +++ b/tripleoclient/utils.py @@ -2659,11 +2659,11 @@ def get_heat_launcher(heat_type, *args, **kwargs): return heat_launcher.HeatPodLauncher(*args, **kwargs) -def kill_heat(launcher, backup_db=True): +def kill_heat(launcher): global _heat_pid if _heat_pid: LOG.debug("Attempting to kill heat pid %s" % _heat_pid) - launcher.kill_heat(_heat_pid, backup_db) + launcher.kill_heat(_heat_pid) def rm_heat(launcher, backup_db=False): diff --git a/tripleoclient/v1/tripleo_launch_heat.py b/tripleoclient/v1/tripleo_launch_heat.py index c627103dc..65ee9867d 100644 --- a/tripleoclient/v1/tripleo_launch_heat.py +++ b/tripleoclient/v1/tripleo_launch_heat.py @@ -31,7 +31,7 @@ from tripleoclient import utils class LaunchHeat(command.Command): """Launch ephemeral Heat process.""" - log = logging.getLogger(__name__ + ".Deploy") + log = logging.getLogger("tripleoclient") auth_required = False heat_pid = None @@ -43,8 +43,8 @@ class LaunchHeat(command.Command): when cleanup is requested. """ - self.log.info("Attempting to kill ephemeral heat") if parsed_args.heat_type == "native": + self.log.info("Attempting to kill ephemeral heat") if self.heat_pid: self.log.info("Using heat pid: %s" % self.heat_pid) self.heat_launcher.kill_heat(self.heat_pid) @@ -52,8 +52,6 @@ class LaunchHeat(command.Command): self.heat_pid = None else: self.log.info("No heat pid set, can't kill.") - else: - self.heat_launcher.kill_heat(None, backup_db=True) return 0 @@ -137,7 +135,7 @@ class LaunchHeat(command.Command): help=_('If specified and --heat-type is container or pod ' 'any existing container or pod of a previous ' 'ephemeral Heat process will be deleted first. ' - 'Ignored if --heat-type is native.') + 'Ignored if --heat-type is native or --kill.') ) parser.add_argument( '--skip-heat-pull', @@ -190,6 +188,11 @@ class LaunchHeat(command.Command): else: heat_type = parsed_args.heat_type + if parsed_args.kill: + rm_heat = True + else: + rm_heat = parsed_args.rm_heat + self.heat_launcher = utils.get_heat_launcher( heat_type, parsed_args.heat_api_port, parsed_args.heat_container_image, @@ -199,7 +202,7 @@ class LaunchHeat(command.Command): parsed_args.heat_dir, False, False, - parsed_args.rm_heat, + rm_heat, parsed_args.skip_heat_pull) if parsed_args.kill: