Wait for Heat message queues

Adds a wait before utils.launch_heat returns to ensure that the
necessary rabbitmq queues have been created. Without the wait, there is
a race condition between the queues being created and the first
heatclient command being executed, which can lead to a
oslo_messaging.exceptions.MessagingTimeout exception.

Also fixes a minor issue in the choices for the --heat-type option for
overcloud deploy. The choice should be "installed" and not "system".

Change-Id: Ib63813b63c37fa2cee57a211535d43d605131529
Signed-off-by: James Slagle <jslagle@redhat.com>

tripleoclient/exceptions.py

@@ -135,3 +135,7 @@ class CellExportError(Base):
 
 class BannedParameters(Base):
     """Some of the environment parameters provided should be removed"""
+
+
+class HeatPodMessageQueueException(Base):
+    """Heat messaging queue not created"""

tripleoclient/heat_launcher.py

@@ -27,11 +27,15 @@ import tempfile
 
 import jinja2
 from oslo_utils import timeutils
+from tenacity import retry, retry_if_exception_type
+from tenacity.stop import stop_after_attempt, stop_after_delay
+from tenacity.wait import wait_fixed
 
 from tripleoclient.constants import (DEFAULT_HEAT_CONTAINER,
                                      DEFAULT_HEAT_API_CONTAINER,
                                      DEFAULT_HEAT_ENGINE_CONTAINER,
                                      DEFAULT_TEMPLATES_DIR)
+from tripleoclient.exceptions import HeatPodMessageQueueException
 
 log = logging.getLogger(__name__)
 
@@ -627,6 +631,19 @@ class HeatPodLauncher(HeatContainerLauncher):
     def _get_num_engine_workers(self):
         return int(multiprocessing.cpu_count() / 2)
 
+    @retry(retry=retry_if_exception_type(HeatPodMessageQueueException),
+           reraise=True,
+           stop=(stop_after_delay(10) | stop_after_attempt(10)),
+           wait=wait_fixed(0.5))
+    def wait_for_message_queue(self):
+        output = subprocess.check_output([
+            'sudo', 'podman', 'exec', '-it', 'rabbitmq',
+            'rabbitmqctl', 'list_queues'
+        ])
+        if 'heat' not in str(output):
+            msg = "Message queue for ephemeral heat not created in time."
+            raise HeatPodMessageQueueException(msg)
+
     def _write_heat_config(self):
         heat_config_tmpl_path = os.path.join(DEFAULT_TEMPLATES_DIR,
                                              "ephemeral-heat",

tripleoclient/utils.py

@@ -2637,6 +2637,7 @@ def launch_heat(launcher=None, restore_db=False):
     # Wait for the API to be listening
     heat_api_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     test_heat_api_port(heat_api_socket, launcher.host, int(launcher.api_port))
+    launcher.wait_for_message_queue()
 
     _local_orchestration_client = tc_heat_utils.local_orchestration_client(
         launcher.host, launcher.api_port)

tripleoclient/v1/overcloud_deploy.py

@@ -957,7 +957,7 @@ class DeployOvercloud(command.Command):
             '--heat-type',
             action='store',
             default='installed',
-            choices=['system', 'pod', 'container', 'native'],
+            choices=['installed', 'pod', 'container', 'native'],
             help=_('The type of Heat process to use to execute '
                    'the deployment.\n'
                    'installed (Default): Use the system installed '