x/tobiko
Code Issues Proposed changes

Merge "Fix retries on get_pcs_resources_table method"

Browse Source
This commit is contained in:
Zuul 2024-02-26 09:48:43 +00:00 committed by Gerrit Code Review
commit c1c001e323
1 changed files with 6 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
tobiko/tripleo/pacemaker.py
View File

@ -3,7 +3,6 @@ from __future__ import absolute_import
import enum import enum
import io import io
import time import time
import typing
from oslo_log import log from oslo_log import log
import pandas import pandas
@ -41,12 +40,9 @@ def get_pcs_resources_table(timeout=720, interval=2) -> pandas.DataFrame:
:return: dataframe of pcs resources stats table :return: dataframe of pcs resources stats table
""" """
failures: typing.List[str] = []
start = time.time()
# prevent pcs table read failure while pacemaker is starting # prevent pcs table read failure while pacemaker is starting
while time.time() - start < timeout: for attempt in tobiko.retry(timeout=timeout,
failures = [] interval=interval):
try: try:
output = run_pcs_status(options=['resources'], grep_str='ocf') output = run_pcs_status(options=['resources'], grep_str='ocf')
# remove the first column when it only includes '*' characters # remove the first column when it only includes '*' characters
@ -56,18 +52,12 @@ def get_pcs_resources_table(timeout=720, interval=2) -> pandas.DataFrame:
stream, delim_whitespace=True, header=None) stream, delim_whitespace=True, header=None)
table.columns = ['resource', 'resource_type', 'resource_state', table.columns = ['resource', 'resource_type', 'resource_state',
'overcloud_node'] 'overcloud_node']
except ValueError: except (ValueError, sh.ShellCommandFailed, sh.ShellTimeoutExpired):
pcs_status_raw = run_pcs_status() if attempt.is_last:
failures.append(f'pcs status table import failed : ' raise
f'pcs status stdout:\n {pcs_status_raw}') LOG.exception('Failed to obtain pcs status table - Retrying...')
LOG.info('Retrying , timeout at: {}'
.format(timeout-(time.time() - start)))
time.sleep(interval)
else: else:
break break
# exhausted all retries
if failures:
tobiko.fail('pcs status table import error\n' + '\n'.join(failures))
LOG.debug("Got pcs status :\n%s", table) LOG.debug("Got pcs status :\n%s", table)
return table return table