From a35f79839087689b85bbb48eb46448f1e29c1237 Mon Sep 17 00:00:00 2001 From: James Slagle Date: Tue, 11 May 2021 17:15:07 -0400 Subject: [PATCH] Wait for Heat message queues Adds a wait before utils.launch_heat returns to ensure that the necessary rabbitmq queues have been created. Without the wait, there is a race condition between the queues being created and the first heatclient command being executed, which can lead to a oslo_messaging.exceptions.MessagingTimeout exception. Also fixes a minor issue in the choices for the --heat-type option for overcloud deploy. The choice should be "installed" and not "system". Change-Id: Ib63813b63c37fa2cee57a211535d43d605131529 Signed-off-by: James Slagle --- tripleoclient/exceptions.py | 4 ++++ tripleoclient/heat_launcher.py | 17 +++++++++++++++++ tripleoclient/utils.py | 1 + tripleoclient/v1/overcloud_deploy.py | 2 +- 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tripleoclient/exceptions.py b/tripleoclient/exceptions.py index 65c3eeaf2..f4af206a5 100644 --- a/tripleoclient/exceptions.py +++ b/tripleoclient/exceptions.py @@ -135,3 +135,7 @@ class CellExportError(Base): class BannedParameters(Base): """Some of the environment parameters provided should be removed""" + + +class HeatPodMessageQueueException(Base): + """Heat messaging queue not created""" diff --git a/tripleoclient/heat_launcher.py b/tripleoclient/heat_launcher.py index a6e21faf7..5fc5db054 100644 --- a/tripleoclient/heat_launcher.py +++ b/tripleoclient/heat_launcher.py @@ -27,11 +27,15 @@ import tempfile import jinja2 from oslo_utils import timeutils +from tenacity import retry, retry_if_exception_type +from tenacity.stop import stop_after_attempt, stop_after_delay +from tenacity.wait import wait_fixed from tripleoclient.constants import (DEFAULT_HEAT_CONTAINER, DEFAULT_HEAT_API_CONTAINER, DEFAULT_HEAT_ENGINE_CONTAINER, DEFAULT_TEMPLATES_DIR) +from tripleoclient.exceptions import HeatPodMessageQueueException log = logging.getLogger(__name__) @@ -627,6 +631,19 @@ class HeatPodLauncher(HeatContainerLauncher): def _get_num_engine_workers(self): return int(multiprocessing.cpu_count() / 2) + @retry(retry=retry_if_exception_type(HeatPodMessageQueueException), + reraise=True, + stop=(stop_after_delay(10) | stop_after_attempt(10)), + wait=wait_fixed(0.5)) + def wait_for_message_queue(self): + output = subprocess.check_output([ + 'sudo', 'podman', 'exec', '-it', 'rabbitmq', + 'rabbitmqctl', 'list_queues' + ]) + if 'heat' not in str(output): + msg = "Message queue for ephemeral heat not created in time." + raise HeatPodMessageQueueException(msg) + def _write_heat_config(self): heat_config_tmpl_path = os.path.join(DEFAULT_TEMPLATES_DIR, "ephemeral-heat", diff --git a/tripleoclient/utils.py b/tripleoclient/utils.py index d5ef44e1f..02a4a9a82 100644 --- a/tripleoclient/utils.py +++ b/tripleoclient/utils.py @@ -2637,6 +2637,7 @@ def launch_heat(launcher=None, restore_db=False): # Wait for the API to be listening heat_api_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) test_heat_api_port(heat_api_socket, launcher.host, int(launcher.api_port)) + launcher.wait_for_message_queue() _local_orchestration_client = tc_heat_utils.local_orchestration_client( launcher.host, launcher.api_port) diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py index ac6054ef7..f3ca308c1 100644 --- a/tripleoclient/v1/overcloud_deploy.py +++ b/tripleoclient/v1/overcloud_deploy.py @@ -957,7 +957,7 @@ class DeployOvercloud(command.Command): '--heat-type', action='store', default='installed', - choices=['system', 'pod', 'container', 'native'], + choices=['installed', 'pod', 'container', 'native'], help=_('The type of Heat process to use to execute ' 'the deployment.\n' 'installed (Default): Use the system installed '