Browse Source

Wait for Heat message queues

Adds a wait before utils.launch_heat returns to ensure that the
necessary rabbitmq queues have been created. Without the wait, there is
a race condition between the queues being created and the first
heatclient command being executed, which can lead to a
oslo_messaging.exceptions.MessagingTimeout exception.

Also fixes a minor issue in the choices for the --heat-type option for
overcloud deploy. The choice should be "installed" and not "system".

Change-Id: Ib63813b63c37fa2cee57a211535d43d605131529
Signed-off-by: James Slagle <jslagle@redhat.com>
changes/34/790834/4
James Slagle 3 months ago
parent
commit
a35f798390
  1. 4
      tripleoclient/exceptions.py
  2. 17
      tripleoclient/heat_launcher.py
  3. 1
      tripleoclient/utils.py
  4. 2
      tripleoclient/v1/overcloud_deploy.py

4
tripleoclient/exceptions.py

@ -135,3 +135,7 @@ class CellExportError(Base):
class BannedParameters(Base):
"""Some of the environment parameters provided should be removed"""
class HeatPodMessageQueueException(Base):
"""Heat messaging queue not created"""

17
tripleoclient/heat_launcher.py

@ -27,11 +27,15 @@ import tempfile
import jinja2
from oslo_utils import timeutils
from tenacity import retry, retry_if_exception_type
from tenacity.stop import stop_after_attempt, stop_after_delay
from tenacity.wait import wait_fixed
from tripleoclient.constants import (DEFAULT_HEAT_CONTAINER,
DEFAULT_HEAT_API_CONTAINER,
DEFAULT_HEAT_ENGINE_CONTAINER,
DEFAULT_TEMPLATES_DIR)
from tripleoclient.exceptions import HeatPodMessageQueueException
log = logging.getLogger(__name__)
@ -627,6 +631,19 @@ class HeatPodLauncher(HeatContainerLauncher):
def _get_num_engine_workers(self):
return int(multiprocessing.cpu_count() / 2)
@retry(retry=retry_if_exception_type(HeatPodMessageQueueException),
reraise=True,
stop=(stop_after_delay(10) | stop_after_attempt(10)),
wait=wait_fixed(0.5))
def wait_for_message_queue(self):
output = subprocess.check_output([
'sudo', 'podman', 'exec', '-it', 'rabbitmq',
'rabbitmqctl', 'list_queues'
])
if 'heat' not in str(output):
msg = "Message queue for ephemeral heat not created in time."
raise HeatPodMessageQueueException(msg)
def _write_heat_config(self):
heat_config_tmpl_path = os.path.join(DEFAULT_TEMPLATES_DIR,
"ephemeral-heat",

1
tripleoclient/utils.py

@ -2637,6 +2637,7 @@ def launch_heat(launcher=None, restore_db=False):
# Wait for the API to be listening
heat_api_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
test_heat_api_port(heat_api_socket, launcher.host, int(launcher.api_port))
launcher.wait_for_message_queue()
_local_orchestration_client = tc_heat_utils.local_orchestration_client(
launcher.host, launcher.api_port)

2
tripleoclient/v1/overcloud_deploy.py

@ -957,7 +957,7 @@ class DeployOvercloud(command.Command):
'--heat-type',
action='store',
default='installed',
choices=['system', 'pod', 'container', 'native'],
choices=['installed', 'pod', 'container', 'native'],
help=_('The type of Heat process to use to execute '
'the deployment.\n'
'installed (Default): Use the system installed '

Loading…
Cancel
Save