Ephemeral Heat unit tests

This patch adds unit tests for the ephemeral Heat feature.

Additionally, the following changes are combined into this commit:
- Adds compression for ephemeral Heat db and log file backups
- Configures a specific log_file in the generated heat.conf so we can
  have a unique log file per execution so as not to overwrite previous
  log files. Provides better debuggability as each exeuction is in it's
  own log file.
- Guards the mount/umount of tmpfs for the heat dir with use_root, so as
  to supress the warnings about the commands failing when not using
  root.
- Configures the logger in tripleo_launch_heat.py as just 'tripleoclient'
  so that the configuration propagates to other loggers that inherit
  from 'tripleoclient'
- Fixes tripleo_launch_heat.py to only attempt to kill heat once when
  --kill is passed.
- Minor fixes uncovered during the unit test addition.

Signed-off-by: James Slagle <jslagle@redhat.com>
Change-Id: I44f8d8a208f1b38ccfdd2b4b225c877e539e57cd
(cherry picked from commit 04323f12e728439af44593276b4d6766ecbc679f)

templates/ephemeral-heat/heat.conf.j2

@@ -6,6 +6,7 @@ default_deployment_signal_transport = HEAT_SIGNAL
 deferred_auth_method = password
 keystone_backend = heat.engine.clients.os.keystone.fake_keystoneclient.FakeKeystoneClient
 log_dir = /var/log/heat
+log_file = {{ log_file }}
 max_json_body_size = 8388608
 max_nested_stack_depth = 10
 max_resources_per_stack=-1

tripleoclient/heat_launcher.py

@@ -13,6 +13,7 @@
 #   under the License.
 #
 
+import configparser
 import datetime
 import glob
 import grp
@@ -23,7 +24,9 @@ import os
 import pwd
 import signal
 import subprocess
+import tarfile
 import tempfile
+import time
 
 import jinja2
 from oslo_utils import timeutils
@@ -139,29 +142,27 @@ class HeatBaseLauncher(object):
         self.engine_container_image = engine_container_image
         self.heat_dir = os.path.abspath(heat_dir)
         self.host = "127.0.0.1"
-        self.db_dump_path = os.path.join(
-            self.heat_dir, 'heat-db-dump-{}.sql'.format(
-                datetime.datetime.utcnow().isoformat()))
+        self.timestamp = time.time()
+        self.db_dump_path = os.path.join(self.heat_dir, 'heat-db.sql')
         self.skip_heat_pull = skip_heat_pull
-
-        if rm_heat:
-            self.kill_heat(None)
-            self.rm_heat()
+        self.zipped_db_suffix = '.tar.bzip2'
+        self.log_dir = os.path.join(self.heat_dir, 'log')
 
         if os.path.isdir(self.heat_dir):
-            # This one may fail but it's just cleanup.
-            p = subprocess.Popen(['umount', self.heat_dir],
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE,
-                                 universal_newlines=True)
-            cmd_stdout, cmd_stderr = p.communicate()
-            retval = p.returncode
-            if retval != 0:
-                log.info('Cleanup unmount of %s failed (probably because '
-                         'it was not mounted): %s' %
-                         (self.heat_dir, cmd_stderr))
-            else:
-                log.info('umount of %s success' % (self.heat_dir))
+            if use_root:
+                # This one may fail but it's just cleanup.
+                p = subprocess.Popen(['umount', self.heat_dir],
+                                     stdout=subprocess.PIPE,
+                                     stderr=subprocess.PIPE,
+                                     universal_newlines=True)
+                cmd_stdout, cmd_stderr = p.communicate()
+                retval = p.returncode
+                if retval != 0:
+                    log.info('Cleanup unmount of %s failed (probably because '
+                             'it was not mounted): %s' %
+                             (self.heat_dir, cmd_stderr))
+                else:
+                    log.info('umount of %s success' % (self.heat_dir))
         else:
             # Create the directory if it doesn't exist.
             try:
@@ -171,36 +172,40 @@ class HeatBaseLauncher(object):
                           (self.heat_dir, e))
                 raise Exception('Could not create temp directory %s: %s' %
                                 (self.heat_dir, e))
-        # As an optimization we mount the tmp directory in a tmpfs (in memory)
-        # filesystem.  Depending on your system this can cut the heat
-        # deployment times by half.
-        p = subprocess.Popen(['mount', '-t', 'tmpfs', '-o', 'size=500M',
-                              'tmpfs', self.heat_dir],
-                             stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE,
-                             universal_newlines=True)
-        cmd_stdout, cmd_stderr = p.communicate()
-        retval = p.returncode
-        if retval != 0:
-            # It's ok if this fails, it will still work.  It just won't
-            # be on tmpfs.
-            log.warning('Unable to mount tmpfs for logs and '
-                        'database %s: %s' %
-                        (self.heat_dir, cmd_stderr))
 
-        self.policy_file = os.path.join(os.path.dirname(__file__),
-                                        'noauth_policy.json')
+        if use_root:
+            # As an optimization we mount the tmp directory in a tmpfs (in
+            # memory) filesystem.  Depending on your system this can cut the
+            # heat deployment times by half.
+            p = subprocess.Popen(['mount', '-t', 'tmpfs', '-o', 'size=500M',
+                                  'tmpfs', self.heat_dir],
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE,
+                                 universal_newlines=True)
+            cmd_stdout, cmd_stderr = p.communicate()
+            retval = p.returncode
+            if retval != 0:
+                # It's ok if this fails, it will still work.  It just won't
+                # be on tmpfs.
+                log.warning('Unable to mount tmpfs for logs and '
+                            'database %s: %s' %
+                            (self.heat_dir, cmd_stderr))
+
         if use_tmp_dir:
             self.install_dir = tempfile.mkdtemp(
                 prefix='%s/undercloud_deploy-' % self.heat_dir)
         else:
             self.install_dir = self.heat_dir
-        self.user = user
+
+        self.log_file = self._get_log_file_path()
         self.sql_db = os.path.join(self.install_dir, 'heat.sqlite')
-        self.log_file = os.path.join(self.install_dir, 'heat.log')
         self.config_file = os.path.join(self.install_dir, 'heat.conf')
         self.paste_file = os.path.join(self.install_dir, 'api-paste.ini')
         self.token_file = os.path.join(self.install_dir, 'token_file.json')
+
+        self.policy_file = os.path.join(os.path.dirname(__file__),
+                                        'noauth_policy.json')
+        self.user = user
         self._write_fake_keystone_token(self.api_port, self.token_file)
         self._write_heat_config()
         self._write_api_paste_config()
@@ -211,6 +216,13 @@ class HeatBaseLauncher(object):
             os.chown(self.config_file, uid, gid)
             os.chown(self.paste_file, uid, gid)
 
+        if rm_heat:
+            self.kill_heat(None)
+            self.rm_heat()
+
+    def _get_log_file_path(self):
+        return os.path.join(self.install_dir, 'heat.log')
+
     def _write_heat_config(self):
         # TODO(ksambor) It will be nice to have possibilities to configure heat
         heat_config = '''
@@ -300,6 +312,20 @@ heat.filter_factory = heat.api.openstack:faultwrap_filter
     def check_message_bus(self):
         return True
 
+    def tar_file(self, file_path, cleanup=True):
+        tf_name = '{}-{}.tar.bzip2'.format(file_path, self.timestamp)
+        tf = tarfile.open(tf_name, 'w:bz2')
+        tf.add(file_path, os.path.basename(file_path))
+        tf.close()
+        log.info("Created tarfile {}".format(tf_name))
+        if cleanup:
+            log.info("Deleting {}".format(file_path))
+            os.unlink(file_path)
+
+    def untar_file(self, tar_path, extract_dir):
+        tf = tarfile.open(tar_path, 'r:bz2')
+        tf.extractall(extract_dir)
+
 
 class HeatContainerLauncher(HeatBaseLauncher):
 
@@ -385,7 +411,7 @@ class HeatContainerLauncher(HeatBaseLauncher):
             return result.split(':')[2]
         raise Exception('Could not find heat gid')
 
-    def kill_heat(self, pid, backup_db=False):
+    def kill_heat(self, pid):
         cmd = ['podman', 'stop', 'heat_all']
         log.debug(' '.join(cmd))
         # We don't want to hear from this command..
@@ -413,7 +439,7 @@ class HeatNativeLauncher(HeatBaseLauncher):
         subprocess.check_call(['heat-manage', '--config-file',
                                self.config_file, 'db_sync'])
 
-    def kill_heat(self, pid, backup_db=False):
+    def kill_heat(self, pid):
         os.kill(pid, signal.SIGKILL)
 
 
@@ -423,9 +449,8 @@ class HeatPodLauncher(HeatContainerLauncher):
 
     def __init__(self, *args, **kwargs):
         super(HeatPodLauncher, self).__init__(*args, **kwargs)
-        log_dir = os.path.join(self.heat_dir, 'log')
-        if not os.path.isdir(log_dir):
-            os.makedirs(log_dir)
+        if not os.path.isdir(self.log_dir):
+            os.makedirs(self.log_dir)
         self.host = self._get_ctlplane_ip()
         self._chcon()
 
@@ -449,14 +474,17 @@ class HeatPodLauncher(HeatContainerLauncher):
                 raise Exception('Unable to fetch container image {}.'
                                 'Error: {}'.format(image, e))
 
-    def launch_heat(self):
+    def get_pod_state(self):
         inspect = subprocess.run([
             'sudo', 'podman', 'pod', 'inspect', '--format',
             '"{{.State}}"', 'ephemeral-heat'],
             check=False,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT)
-        if "Running" in self._decode(inspect.stdout):
+        return self._decode(inspect.stdout)
+
+    def launch_heat(self):
+        if "Running" in self.get_pod_state():
             log.info("ephemeral-heat pod already running, skipping launch")
             return
         self._write_heat_pod()
@@ -503,16 +531,25 @@ class HeatPodLauncher(HeatContainerLauncher):
     def do_restore_db(self, db_dump_path=None):
         if not db_dump_path:
             # Find the latest dump from self.heat_dir
-            db_dumps = glob.glob('{}/heat-db-dump*'.format(self.heat_dir))
+            db_dumps = glob.glob(
+                '{}/heat-db-dump*{}'.format
+                (self.heat_dir,
+                 self.zipped_db_suffix))
             if not db_dumps:
                 raise Exception('No db backups found to restore in %s' %
                                 self.heat_dir)
             db_dump_path = max(db_dumps, key=os.path.getmtime)
+            self.untar_file(db_dump_path, self.heat_dir)
+            db_dump_path = db_dump_path.rstrip(self.zipped_db_suffix)
             log.info("Restoring db from {}".format(db_dump_path))
-        subprocess.run([
-            'sudo', 'podman', 'exec', '-u', 'root',
-            'mysql', 'mysql', 'heat'], stdin=open(db_dump_path),
-            check=True)
+        try:
+            with open(db_dump_path) as f:
+                subprocess.run([
+                    'sudo', 'podman', 'exec', '-u', 'root',
+                    'mysql', 'mysql', 'heat'], stdin=f,
+                    check=True)
+        finally:
+            os.unlink(db_dump_path)
 
     def do_backup_db(self, db_dump_path=None):
         if not db_dump_path:
@@ -520,13 +557,26 @@ class HeatPodLauncher(HeatContainerLauncher):
         if os.path.exists(db_dump_path):
             raise Exception("Won't overwrite existing db dump at %s. "
                             "Remove it first." % db_dump_path)
+        log.info("Starting back up of heat db")
         with open(db_dump_path, 'w') as out:
             subprocess.run([
                 'sudo', 'podman', 'exec', '-u', 'root',
                 'mysql', 'mysqldump', 'heat'], stdout=out,
                 check=True)
 
-    def rm_heat(self, backup_db=False):
+        self.tar_file(db_dump_path)
+
+    def pod_exists(self):
+        try:
+            subprocess.check_call(
+                ['sudo', 'podman', 'pod', 'inspect', 'ephemeral-heat'],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL)
+            return True
+        except subprocess.CalledProcessError:
+            return False
+
+    def rm_heat(self, backup_db=True):
         if self.database_exists():
             if backup_db:
                 self.do_backup_db()
@@ -541,14 +591,24 @@ class HeatPodLauncher(HeatContainerLauncher):
                     'drop user \'heat\'@\'%\''])
             except subprocess.CalledProcessError:
                 pass
-        subprocess.call([
-            'sudo', 'podman', 'pod', 'rm', '-f', 'ephemeral-heat'
-        ])
+        if self.pod_exists():
+            log.info("Removing pod: ephemeral-heat")
+            subprocess.call([
+                'sudo', 'podman', 'pod', 'rm', '-f', 'ephemeral-heat'
+            ])
+        config = self._read_heat_config()
+        log_file_path = os.path.join(self.log_dir,
+                                     config['DEFAULT']['log_file'])
+        if os.path.exists(log_file_path):
+            self.tar_file(log_file_path)
 
     def stop_heat(self):
-        subprocess.check_call([
-            'sudo', 'podman', 'pod', 'stop', 'ephemeral-heat'
-        ])
+        if self.pod_exists() and self.get_pod_state() != 'Exited':
+            log.info("Stopping pod: ephemeral-heat")
+            subprocess.check_call([
+                'sudo', 'podman', 'pod', 'stop', 'ephemeral-heat'
+            ])
+            log.info("Stopped pod: ephemeral-heat")
 
     def check_message_bus(self):
         log.info("Checking that message bus (rabbitmq) is up")
@@ -587,10 +647,15 @@ class HeatPodLauncher(HeatContainerLauncher):
         ])
         return 'heat' in str(output)
 
-    def kill_heat(self, pid, backup_db=False):
-        subprocess.call([
-            'sudo', 'podman', 'pod', 'kill', 'ephemeral-heat'
-        ])
+    def kill_heat(self, pid):
+        if self.pod_exists():
+            log.info("Killing pod: ephemeral-heat")
+            subprocess.call([
+                'sudo', 'podman', 'pod', 'kill', 'ephemeral-heat'
+            ])
+            log.info("Killed pod: ephemeral-heat")
+        else:
+            log.info("Pod does not exist: ephemeral-heat")
 
     def _decode(self, encoded):
         if not encoded:
@@ -644,6 +709,14 @@ class HeatPodLauncher(HeatContainerLauncher):
             msg = "Message queue for ephemeral heat not created in time."
             raise HeatPodMessageQueueException(msg)
 
+    def _get_log_file_path(self):
+        return 'heat-{}.log'.format(self.timestamp)
+
+    def _read_heat_config(self):
+        config = configparser.ConfigParser()
+        config.read(self.config_file)
+        return config
+
     def _write_heat_config(self):
         heat_config_tmpl_path = os.path.join(DEFAULT_TEMPLATES_DIR,
                                              "ephemeral-heat",
@@ -656,6 +729,7 @@ class HeatPodLauncher(HeatContainerLauncher):
             "db_connection": self._get_db_connection(),
             "api_port": self.api_port,
             "num_engine_workers": self._get_num_engine_workers(),
+            "log_file": self.log_file,
         }
         heat_config = heat_config_tmpl.render(**config_vars)
 

tripleoclient/tests/test_heat_launcher.py

@@ -0,0 +1,603 @@
+#   Copyright 2021 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+import fixtures
+import mock
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import time
+
+from tripleoclient import heat_launcher
+from tripleoclient.exceptions import HeatPodMessageQueueException
+from tripleoclient.tests import base
+from tripleoclient import utils
+
+
+class TestHeatPodLauncher(base.TestCase):
+    def setUp(self):
+        super(TestHeatPodLauncher, self).setUp()
+        self.run = mock.patch('subprocess.run').start()
+        self.call = mock.patch('subprocess.call').start()
+        self.check_call = mock.patch('subprocess.check_call').start()
+        self.check_output = mock.patch('subprocess.check_output').start()
+        self.templates_dir = mock.patch(
+            'tripleoclient.heat_launcher.DEFAULT_TEMPLATES_DIR',
+            os.path.join(os.path.dirname(__file__),
+                         '..', '..', 'templates')).start()
+        self.heat_dir = self.useFixture(fixtures.TempDir()).path
+
+        self.addCleanup(mock.patch.stopall)
+
+    def get_launcher(self, **kwargs):
+        return heat_launcher.HeatPodLauncher(
+                heat_dir=self.heat_dir,
+                use_tmp_dir=False,
+                **kwargs)
+
+    def check_calls(self, check_call, mock_obj):
+        for call in mock_obj.call_args_list:
+            call_str = ' '.join(call.args[0])
+            if check_call in call_str:
+                return True
+        return False
+
+    def test_rm_heat_launcher(self):
+        self.assertIsInstance(self.get_launcher(rm_heat=True),
+                              heat_launcher.HeatPodLauncher)
+
+    def test_chcon(self):
+        launcher = self.get_launcher()
+        launcher._chcon()
+        self.check_calls('chcon', self.check_call)
+        self.check_calls(launcher.heat_dir, self.check_call)
+
+    def test_fetch_container_image(self):
+        launcher = self.get_launcher(skip_heat_pull=True)
+        launcher._fetch_container_image()
+        self.assertFalse(self.check_calls('podman pull', self.check_output))
+
+        launcher = self.get_launcher(skip_heat_pull=False)
+        launcher._fetch_container_image()
+        self.assertTrue(self.check_calls('podman pull', self.check_output))
+
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher._decode')
+    def test_get_pod_state(self, mock_decode):
+        launcher = self.get_launcher()
+        launcher.get_pod_state()
+        self.check_calls('podman pod inspect', self.run)
+        self.assertTrue(mock_decode.called)
+
+    @mock.patch(
+        'tripleoclient.heat_launcher.HeatPodLauncher._write_heat_config')
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher._write_heat_pod')
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.get_pod_state')
+    def test_lauch_heat(
+            self, mock_get_pod_state, mock_write_heat_pod,
+            mock_write_heat_config):
+
+        launcher = self.get_launcher()
+
+        mock_get_pod_state.return_value = 'Running'
+        launcher.launch_heat()
+        self.assertFalse(mock_write_heat_pod.called)
+        self.assertFalse(self.check_calls('podman play kube', self.check_call))
+
+        mock_get_pod_state.return_value = 'Exited'
+        launcher.launch_heat()
+        self.assertTrue(mock_write_heat_pod.called)
+        self.assertTrue(self.check_calls('podman play kube', self.check_call))
+
+        mock_get_pod_state.return_value = ''
+        launcher.launch_heat()
+        self.assertTrue(mock_write_heat_pod.called)
+        self.assertTrue(self.check_calls('podman play kube', self.check_call))
+
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.do_restore_db')
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.database_exists')
+    def test_heat_db_sync(
+            self, mock_db_exists, mock_do_restore_db):
+
+        launcher = self.get_launcher()
+        mock_db_exists.return_value = True
+        launcher.heat_db_sync(restore_db=False)
+        self.assertFalse(self.check_calls('create database', self.check_call))
+        self.assertFalse(self.check_calls('create user', self.check_call))
+        self.assertFalse(self.check_calls('grant all', self.check_call))
+        self.assertFalse(self.check_calls('flush priv', self.check_call))
+        self.assertTrue(self.check_calls('heat-manage', self.check_call))
+        self.assertFalse(mock_do_restore_db.called)
+
+        self.check_call.reset_mock()
+
+        mock_db_exists.return_value = True
+        launcher.heat_db_sync(restore_db=True)
+        self.assertFalse(self.check_calls('create database', self.check_call))
+        self.assertFalse(self.check_calls('create user', self.check_call))
+        self.assertFalse(self.check_calls('grant all', self.check_call))
+        self.assertFalse(self.check_calls('flush priv', self.check_call))
+        self.assertTrue(self.check_calls('heat-manage', self.check_call))
+        self.assertTrue(mock_do_restore_db.called)
+
+        self.check_call.reset_mock()
+        mock_db_exists.return_value = False
+        launcher.heat_db_sync(restore_db=True)
+        self.assertTrue(self.check_calls('create database', self.check_call))
+        self.assertTrue(self.check_calls('create user', self.check_call))
+        self.assertTrue(self.check_calls('grant all', self.check_call))
+        self.assertTrue(self.check_calls('flush priv', self.check_call))
+        self.assertTrue(self.check_calls('heat-manage', self.check_call))
+        self.assertTrue(mock_do_restore_db.called)
+
+        self.check_call.reset_mock()
+        mock_do_restore_db.reset_mock()
+        mock_db_exists.return_value = False
+        launcher.heat_db_sync(restore_db=False)
+        self.assertTrue(self.check_calls('create database', self.check_call))
+        self.assertTrue(self.check_calls('create user', self.check_call))
+        self.assertTrue(self.check_calls('grant all', self.check_call))
+        self.assertTrue(self.check_calls('flush priv', self.check_call))
+        self.assertTrue(self.check_calls('heat-manage', self.check_call))
+        self.assertFalse(mock_do_restore_db.called)
+
+    @mock.patch('os.unlink')
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.untar_file')
+    @mock.patch('glob.glob')
+    def test_do_restore_db(
+            self, mock_glob, mock_untar, mock_unlink):
+
+        launcher = self.get_launcher()
+
+        one = Path(os.path.join(launcher.heat_dir,
+                                'heat-db-dump-one.tar.bz2'))
+        two = Path(os.path.join(launcher.heat_dir,
+                                'heat-db-dump-two.tar.bz2'))
+        three = Path(os.path.join(launcher.heat_dir,
+                                  'heat-db-dump-three.tar.bz2'))
+
+        now = time.time()
+        one.touch()
+        two.touch()
+        three.touch()
+        os.utime(str(one), (now, 1000))
+        os.utime(str(two), (now, 2000))
+        os.utime(str(three), (now, 3000))
+        mock_glob.return_value = [str(one), str(two), str(three)]
+
+        def untar(path, dir):
+            p = Path(path.rstrip('.tar.bz2'))
+            p.touch()
+
+        mock_untar.side_effect = untar
+
+        mock_open = mock.mock_open()
+        with mock.patch('six.moves.builtins.open', mock_open):
+            # pylint: disable=bad-str-strip-call
+            launcher.do_restore_db()
+            self.assertEqual(mock.call(str(three), launcher.heat_dir),
+                             mock_untar.call_args)
+            self.assertEqual(mock.call(str(three).rstrip('.tar.bz2')),
+                             mock_unlink.call_args)
+            mock_open.assert_called_with(str(three).rstrip('.tar.bz2')) # noqa
+            self.assertTrue(self.check_call('mysql heat', self.run))
+
+        mock_unlink.reset_mock()
+        self.run.reset_mock()
+        two.touch()
+        mock_open = mock.mock_open()
+        with mock.patch('six.moves.builtins.open', mock_open):
+            # pylint: disable=bad-str-strip-call
+            launcher.do_restore_db()
+            self.assertEqual(mock.call(str(two), launcher.heat_dir),
+                             mock_untar.call_args)
+            self.assertEqual(mock.call(str(two).rstrip('.tar.bz2')), # noqa
+                             mock_unlink.call_args)
+            mock_open.assert_called_with(str(two).rstrip('.tar.bz2')) # noqa
+            self.assertTrue(self.check_call('mysql heat', self.run))
+
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.tar_file')
+    def test_do_backup_db(self, mock_tar):
+        launcher = self.get_launcher()
+        p = Path(os.path.join(launcher.heat_dir, 'heat-db.sql'))
+        p.touch()
+        self.assertRaises(Exception, launcher.do_backup_db, str(p))
+
+        p.unlink()
+        launcher.do_backup_db()
+        mock_tar.assert_called_with(str(p))
+        self.assertTrue(self.check_calls('mysqldump heat', self.run))
+
+    def test_pod_exists(self):
+        launcher = self.get_launcher()
+        self.assertTrue(launcher.pod_exists())
+        self.check_calls('pod inspect', self.check_call)
+
+        self.check_call.reset_mock()
+        self.check_call.side_effect = subprocess.CalledProcessError(1, 'test')
+        self.assertFalse(launcher.pod_exists())
+        self.check_calls('pod inspect', self.check_call)
+
+    @mock.patch('os.path.exists')
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.tar_file')
+    @mock.patch(
+        'tripleoclient.heat_launcher.HeatPodLauncher._read_heat_config')
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.pod_exists')
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.do_backup_db')
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.database_exists')
+    def test_rm_heat(self, mock_db_exists, mock_backup_db, mock_pod_exists,
+                     mock_read_heat_config, mock_tar, mock_exists):
+
+        launcher = self.get_launcher()
+        launcher.log_dir = '/log'
+
+        mock_db_exists.return_value = True
+        mock_pod_exists.return_value = True
+        mock_exists.return_value = True
+        mock_read_heat_config.return_value = {
+            'DEFAULT': {
+                'log_file': 'heat-log'}}
+        launcher.rm_heat()
+        mock_backup_db.assert_called()
+        self.check_calls('drop database heat', self.check_call)
+        self.check_calls('drop user', self.check_call)
+        mock_pod_exists.assert_called()
+        self.check_calls('podman pod rm -f', self.call)
+        mock_read_heat_config.assert_called()
+        mock_tar.assert_called_with('/log/heat-log')
+
+        mock_backup_db.reset_mock()
+        self.call.reset_mock()
+        mock_tar.reset_mock()
+        mock_db_exists.return_value = False
+        mock_pod_exists.return_value = False
+        mock_exists.return_value = False
+        launcher.rm_heat()
+        mock_backup_db.assert_not_called()
+        self.call.assert_not_called()
+        mock_tar.assert_not_called()
+
+        mock_backup_db.reset_mock()
+        self.call.reset_mock()
+        mock_tar.reset_mock()
+        mock_exists.reset_mock()
+        mock_db_exists.return_value = False
+        mock_pod_exists.return_value = True
+        mock_exists.return_value = True
+        launcher.rm_heat(backup_db=False)
+        mock_backup_db.assert_not_called()
+        self.check_calls('podman pod rm -f', self.call)
+
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.get_pod_state')
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.pod_exists')
+    def test_stop_heat(self, mock_pod_exists, mock_pod_state):
+        launcher = self.get_launcher()
+        mock_pod_exists.return_value = True
+        mock_pod_state.return_value = 'Running'
+        launcher.stop_heat()
+        mock_pod_exists.assert_called()
+        mock_pod_state.assert_called()
+        self.check_calls('podman pod stop', self.check_call)
+
+        self.check_call.reset_mock()
+        mock_pod_exists.reset_mock()
+        mock_pod_state.reset_mock()
+        mock_pod_state.return_value = 'Exited'
+        mock_pod_exists.return_value = True
+        launcher.stop_heat()
+        mock_pod_exists.assert_called()
+        mock_pod_state.assert_called()
+        self.check_call.assert_not_called()
+
+        self.check_call.reset_mock()
+        mock_pod_exists.reset_mock()
+        mock_pod_state.reset_mock()
+        mock_pod_state.return_value = 'Exited'
+        mock_pod_exists.return_value = False
+        launcher.stop_heat()
+        mock_pod_exists.assert_called()
+        mock_pod_state.assert_not_called()
+        self.check_call.assert_not_called()
+
+    def test_check_message_bus(self):
+        launcher = self.get_launcher()
+        launcher.check_message_bus()
+        self.check_calls('rabbitmqctl list_queues', self.check_call)
+
+        self.check_call.reset_mock()
+        self.check_call.side_effect = subprocess.CalledProcessError(1, 'test')
+        self.assertRaises(subprocess.CalledProcessError,
+                          launcher.check_message_bus)
+
+    @mock.patch(
+        'tripleoclient.heat_launcher.HeatPodLauncher._get_ctlplane_ip')
+    def test_check_database(self, mock_ctlplane_ip):
+        launcher = self.get_launcher()
+
+        mock_ctlplane_ip.return_value = '1.1.1.1'
+        self.assertTrue(launcher.check_database())
+        mock_ctlplane_ip.assert_called()
+        self.check_calls('show databases', self.check_call)
+
+        self.check_call.reset_mock()
+        mock_ctlplane_ip.reset_mock()
+        self.check_call.side_effect = subprocess.CalledProcessError(1, '/test')
+        self.assertRaises(subprocess.CalledProcessError,
+                          launcher.check_database)
+
+    def test_database_exists(self):
+        launcher = self.get_launcher()
+        self.check_output.return_value = 'heat'
+        self.assertTrue(launcher.database_exists())
+        self.check_calls('show databases like "heat"', self.check_output)
+
+        self.check_output.reset_mock()
+        self.check_output.return_value = 'nova'
+        self.assertFalse(launcher.database_exists())
+        self.check_calls('show databases like "heat"', self.check_output)
+
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher.pod_exists')
+    def test_kill_heat(self, mock_pod_exists):
+        launcher = self.get_launcher()
+        mock_pod_exists.return_value = True
+        launcher.kill_heat(0)
+        self.check_calls('podman pod kill', self.call)
+        mock_pod_exists.assert_called()
+
+        mock_pod_exists.reset_mock()
+        self.call.reset_mock()
+        mock_pod_exists.return_value = False
+        launcher.kill_heat(0)
+        mock_pod_exists.assert_called()
+        self.call.assert_not_called()
+
+    def test_decode(self):
+        launcher = self.get_launcher()
+        mock_encoded = mock.Mock()
+        mock_decoded = mock.Mock()
+        mock_encoded.decode.return_value = mock_decoded
+        mock_decoded.endswith.return_value = False
+        launcher._decode(mock_encoded)
+        mock_encoded.decode.assert_called_with('utf-8')
+
+        self.assertEqual('test', launcher._decode(b'test\n'))
+
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher._decode')
+    def test_get_transport_url(self, mock_decode):
+        launcher = self.get_launcher()
+        mock_decode.side_effect = ['user', 'password', 'fqdn_ctlplane', 'port']
+        self.assertEqual("rabbit://user:password@fqdn_ctlplane:port/?ssl=0",
+                         launcher._get_transport_url())
+
+    @mock.patch(
+        'tripleoclient.heat_launcher.HeatPodLauncher._get_ctlplane_vip')
+    def test_get_db_connection(self, mock_ctlplane_vip):
+        launcher = self.get_launcher()
+        mock_ctlplane_vip.return_value = '1.1.1.1'
+        self.assertEqual(
+            'mysql+pymysql://'
+            'heat:heat@1.1.1.1/heat?read_default_file='
+            '/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo',
+            launcher._get_db_connection())
+
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher._decode')
+    def test_get_ctlplane_vip(self, mock_decode):
+        launcher = self.get_launcher()
+        self.check_output.return_value = '1.1.1.1'
+        launcher._get_ctlplane_vip()
+        self.check_calls('sudo hiera controller_virtual_ip', self.check_output)
+        mock_decode.assert_called_with('1.1.1.1')
+
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher._decode')
+    def test_get_ctlplane_ip(self, mock_decode):
+        launcher = self.get_launcher()
+        self.check_output.return_value = '1.1.1.1'
+        launcher._get_ctlplane_ip()
+        self.check_calls('sudo hiera ctlplane', self.check_output)
+        mock_decode.assert_called_with('1.1.1.1')
+
+    @mock.patch('multiprocessing.cpu_count')
+    def test_get_num_engine_workers(self, mock_cpu_count):
+        launcher = self.get_launcher()
+        mock_cpu_count.return_value = 4
+        self.assertEqual(2, launcher._get_num_engine_workers())
+
+    def test_wait_for_message_queue(self):
+        launcher = self.get_launcher()
+        wait_mq = launcher.wait_for_message_queue.__wrapped__
+        self.check_output.return_value = 'heat'
+        wait_mq(launcher)
+
+        self.check_output.reset_mock()
+        self.check_output.return_value = 'test'
+        self.assertRaises(HeatPodMessageQueueException, wait_mq, launcher)
+
+    def test_get_log_file_path(self):
+        launcher = self.get_launcher()
+        launcher.timestamp = '1111'
+        self.assertEqual('heat-1111.log', launcher._get_log_file_path())
+
+    @mock.patch('configparser.ConfigParser')
+    def test_read_heat_config(self, mock_config_parser):
+        launcher = self.get_launcher()
+        mock_cp = mock.Mock()
+        mock_cp.read.return_value = 'test'
+        mock_config_parser.return_value = mock_cp
+        self.assertEqual(mock_cp, launcher._read_heat_config())
+        mock_config_parser.assert_called()
+        mock_cp.read.assert_called_with(launcher.config_file)
+
+    @mock.patch('tripleoclient.heat_launcher.'
+                'HeatPodLauncher._get_num_engine_workers')
+    @mock.patch(
+        'tripleoclient.heat_launcher.HeatPodLauncher._get_db_connection')
+    @mock.patch(
+        'tripleoclient.heat_launcher.HeatPodLauncher._get_transport_url')
+    def test_write_heat_config(self, mock_get_transport_url, mock_get_db_conn,
+                               mock_num_engine_workers):
+        launcher = self.get_launcher()
+        launcher.api_port = '1234'
+        launcher.log_file = '/log/heat'
+        mock_get_transport_url.return_value = 'transport-url'
+        mock_get_db_conn.return_value = 'db-connection'
+        mock_num_engine_workers.return_value = 'num-engine-workers'
+        launcher._write_heat_config()
+        with open(launcher.config_file) as f:
+            config = f.read()
+            self.assertIn('num_engine_workers = num-engine-workers\n', config)
+            self.assertIn('connection = db-connection\n', config)
+            self.assertIn('transport_url=transport-url\n', config)
+            self.assertIn('bind_port = 1234\n', config)
+            self.assertIn('log_file = /log/heat\n', config)
+
+    def test_write_heat_pod(self):
+        launcher = self.get_launcher()
+        launcher.install_dir = 'install-dir'
+        launcher.host = '1.1.1.1'
+        launcher.api_port = '1234'
+        launcher.api_container_image = 'api-image'
+        launcher.engine_container_image = 'engine-image'
+        launcher._write_heat_pod()
+        with open(os.path.join(launcher.heat_dir, 'heat-pod.yaml')) as f:
+            pod = f.read()
+            self.assertIn('hostPort: 1234', pod)
+            self.assertIn('hostIP: 1.1.1.1', pod)
+            self.assertIn('image: api-image', pod)
+            self.assertIn('image: engine-image', pod)
+
+
+class TestHeatPodLauncherUtils(base.TestCase):
+    def setUp(self):
+        super(TestHeatPodLauncherUtils, self).setUp()
+
+    def test_rm_heat(self):
+        launcher = mock.Mock()
+        utils.rm_heat(launcher)
+        launcher.rm_heat.assert_called_once_with(False)
+        launcher.reset_mock()
+        utils.rm_heat(launcher, True)
+        launcher.rm_heat.assert_called_once_with(True)
+        launcher.reset_mock()
+        utils.rm_heat(launcher, False)
+        launcher.rm_heat.assert_called_once_with(False)
+
+    def test_kill_heat(self):
+        launcher = mock.Mock()
+        utils.kill_heat(launcher)
+        launcher.kill_heat.assert_called_once_with(None)
+        launcher.reset_mock()
+        utils._heat_pid = 111
+        utils.kill_heat(launcher)
+        launcher.kill_heat.assert_called_once_with(111)
+        launcher.reset_mock()
+        utils.kill_heat(launcher)
+        launcher.kill_heat.assert_called_once_with(111)
+        launcher.reset_mock()
+        utils.kill_heat(launcher)
+        launcher.kill_heat.assert_called_once_with(111)
+
+    @mock.patch('tripleoclient.heat_launcher.HeatPodLauncher')
+    @mock.patch('tripleoclient.heat_launcher.HeatNativeLauncher')
+    @mock.patch('tripleoclient.heat_launcher.HeatContainerLauncher')
+    def test_get_heat_launcher(self, mock_container, mock_native, mock_pod):
+        utils.get_heat_launcher('pod', 1, 2, 3, a='a', b='b', c='c')
+        mock_pod.assert_called_once_with(1, 2, 3, a='a', b='b', c='c')
+        utils.get_heat_launcher('native', 1, 2, 3, a='a', b='b', c='c')
+        mock_native.assert_called_once_with(1, 2, 3, a='a', b='b', c='c')
+        utils.get_heat_launcher('container', 1, 2, 3, a='a', b='b', c='c')
+        mock_container.assert_called_once_with(1, 2, 3, a='a', b='b', c='c')
+
+    def test_heat_api_port(self):
+        test_port = utils.test_heat_api_port.__wrapped__
+        mock_socket = mock.Mock()
+        host = '1.1.1.1'
+        port = 1234
+        test_port(mock_socket, host, port)
+        mock_socket.connect.assert_called_once_with((host, port))
+
+    @mock.patch('tripleoclient.utils.test_heat_api_port')
+    @mock.patch('tripleo_common.utils.heat.local_orchestration_client')
+    @mock.patch('socket.socket')
+    @mock.patch('tripleoclient.utils.get_heat_launcher')
+    def test_launch_heat(self, mock_get_heat_launcher, mock_socket,
+                         mock_local_client, mock_test_port):
+        utils._local_orchestration_client = 'client'
+        self.assertEqual('client', utils.launch_heat())
+        mock_get_heat_launcher.assert_not_called()
+
+        utils._local_orchestration_client = None
+        mock_launcher = mock.Mock()
+        mock_launcher.api_port = 1234
+        mock_get_heat_launcher.return_value = mock_launcher
+        mock_socket.return_value = 'socket'
+        utils.launch_heat()
+        mock_get_heat_launcher.assert_called_once()
+        mock_launcher.check_database.assert_called_once_with()
+        mock_launcher.check_message_bus.assert_called_once_with()
+        mock_launcher.heat_db_sync.assert_called_once_with(False)
+        mock_launcher.launch_heat.assert_called_once_with()
+        mock_test_port.assert_called_once_with(
+            'socket', mock_launcher.host,
+            int(mock_launcher.api_port))
+        mock_launcher.wait_for_message_queue.assert_called_once_with()
+        mock_local_client.assert_called_once_with(
+            mock_launcher.host,
+            mock_launcher.api_port)
+
+
+class TestHeatNativeLauncher(base.TestCase):
+    def setUp(self):
+        super(TestHeatNativeLauncher, self).setUp()
+        self.run = mock.patch('subprocess.run').start()
+        self.popen = mock.patch('subprocess.Popen').start()
+        self.mock_popen = mock.Mock()
+        self.mock_popen.communicate.return_value = ("", "")
+        self.popen.return_value = self.mock_popen
+        self.getpwnam = mock.patch('pwd.getpwnam').start()
+        self.getgrnam = mock.patch('grp.getgrnam').start()
+        self.chown = mock.patch('os.chown').start()
+
+        self.templates_dir = mock.patch(
+            'tripleoclient.heat_launcher.DEFAULT_TEMPLATES_DIR',
+            os.path.join(os.path.dirname(__file__),
+                         '..', '..', 'templates')).start()
+        self.heat_dir = self.useFixture(fixtures.TempDir()).path
+        self.tmp_dir = self.useFixture(fixtures.TempDir()).path
+
+        self.addCleanup(mock.patch.stopall)
+
+    def get_launcher(self, **kwargs):
+        return heat_launcher.HeatNativeLauncher(
+                heat_dir=self.heat_dir,
+                use_tmp_dir=True,
+                use_root=True,
+                **kwargs)
+
+    def test_heat_dir_no_exist(self):
+        shutil.rmtree(self.heat_dir)
+        launcher = self.get_launcher()
+        self.assertNotEqual(self.heat_dir, launcher.install_dir)
+
+    @mock.patch('tempfile.mkdtemp')
+    def test_get_launcher(self, mock_mkdtemp):
+        mock_mkdtemp.return_value = self.tmp_dir
+
+        def test_install_dir():
+            mock_mkdtemp.assert_not_called()
+            return ("", "")
+
+        # Test that tempfile.mkdtemp is *not* called before the tmpfs is setup,
+        # otherwise the tmpfs will cause the temp dir to be lost
+        self.mock_popen.communicate.side_effect = test_install_dir
+        self.get_launcher()

tripleoclient/utils.py

@@ -2659,11 +2659,11 @@ def get_heat_launcher(heat_type, *args, **kwargs):
         return heat_launcher.HeatPodLauncher(*args, **kwargs)
 
 
-def kill_heat(launcher, backup_db=True):
+def kill_heat(launcher):
     global _heat_pid
     if _heat_pid:
         LOG.debug("Attempting to kill heat pid %s" % _heat_pid)
-    launcher.kill_heat(_heat_pid, backup_db)
+    launcher.kill_heat(_heat_pid)
 
 
 def rm_heat(launcher, backup_db=False):

tripleoclient/v1/tripleo_launch_heat.py

@@ -31,7 +31,7 @@ from tripleoclient import utils
 class LaunchHeat(command.Command):
     """Launch ephemeral Heat process."""
 
-    log = logging.getLogger(__name__ + ".Deploy")
+    log = logging.getLogger("tripleoclient")
     auth_required = False
     heat_pid = None
 
@@ -43,8 +43,8 @@ class LaunchHeat(command.Command):
         when cleanup is requested.
 
         """
-        self.log.info("Attempting to kill ephemeral heat")
         if parsed_args.heat_type == "native":
+            self.log.info("Attempting to kill ephemeral heat")
             if self.heat_pid:
                 self.log.info("Using heat pid: %s" % self.heat_pid)
                 self.heat_launcher.kill_heat(self.heat_pid)
@@ -52,8 +52,6 @@ class LaunchHeat(command.Command):
                 self.heat_pid = None
             else:
                 self.log.info("No heat pid set, can't kill.")
-        else:
-            self.heat_launcher.kill_heat(None, backup_db=True)
 
         return 0
 
@@ -137,7 +135,7 @@ class LaunchHeat(command.Command):
             help=_('If specified and --heat-type is container or pod '
                    'any existing container or pod of a previous '
                    'ephemeral Heat process will be deleted first. '
-                   'Ignored if --heat-type is native.')
+                   'Ignored if --heat-type is native or --kill.')
         )
         parser.add_argument(
             '--skip-heat-pull',
@@ -190,6 +188,11 @@ class LaunchHeat(command.Command):
         else:
             heat_type = parsed_args.heat_type
 
+        if parsed_args.kill:
+            rm_heat = True
+        else:
+            rm_heat = parsed_args.rm_heat
+
         self.heat_launcher = utils.get_heat_launcher(
             heat_type, parsed_args.heat_api_port,
             parsed_args.heat_container_image,
@@ -199,7 +202,7 @@ class LaunchHeat(command.Command):
             parsed_args.heat_dir,
             False,
             False,
-            parsed_args.rm_heat,
+            rm_heat,
             parsed_args.skip_heat_pull)
 
         if parsed_args.kill: