x
/
tobiko
Code Issues Proposed changes

fix false : AssertionError: nova agents are unhealthy and 

: OvercloudProcessesException: not all overcloud processes are
in running state, process nova-scheduler is not running

Change-Id: I58f18127f08f04e74da3563b22a8c36ab576014d
This commit is contained in:
pinikomarov 2020-02-27 02:14:24 +02:00
parent 9bd7bf521e
commit f54c14b4b4
2 changed files with 62 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
tobiko/tripleo/nova.py
View File

@ -1,18 +1,39 @@
from __future__ import absolute_import
import time
from oslo_log import log
import tobiko
from tobiko.openstack import nova
LOG = log.getLogger(__name__)
def check_nova_services_health():
def check_nova_services_health(timeout=120, interval=2):
failures = []
nova_client = nova.get_nova_client()
services = nova_client.services.list()
start = time.time()
for service in services:
if not service.state == 'up':
failures.append('failed service: {}\n\n'.format(vars(service)))
while time.time() - start < timeout:
failures = []
nova_client = nova.get_nova_client()
services = nova_client.services.list()
for service in services:
if not service.state == 'up':
failures.append(
'failed service: {}\n\n'.format(vars(service)))
if failures:
LOG.info('Failed nova services:\n {}'.format(failures))
LOG.info('Not all nova services are up ..')
LOG.info('Retrying , timeout at: {}'
.format(timeout-(time.time() - start)))
time.sleep(interval)
else:
LOG.info([vars(service) for service in services])
LOG.info('All nova services are up!')
return
# exhausted all retries
if failures:
tobiko.fail(
'nova agents are unhealthy:\n{!s}', '\n'.join(failures))

52
tobiko/tripleo/processes.py
View File

@ -1,5 +1,7 @@
from __future__ import absolute_import
import time
from oslo_log import log
import pandas
import six
@ -110,21 +112,37 @@ class OvercloudProcessesStatus(object):
Checks that the oc_procs_df dataframe has all of the list procs
:return: Bool
"""
for process_name in self.processes_to_check:
# osp16/python3 process is "neutron-server:"
if process_name == 'neutron-server' and \
self.oc_procs_df.query('PROCESS=="{}"'.format(
process_name)).empty:
process_name = 'neutron-server:'
if not self.oc_procs_df.query('PROCESS=="{}"'.format(
process_name)).empty:
LOG.info("overcloud processes status checks: process {} is "
"in running state".format(process_name))
continue
else:
LOG.info("Failure : overcloud processes status checks: "
"process {} is not running ".format(process_name))
raise OvercloudProcessesException(
process_error="process {} is not running ".format(
for attempt_number in range(120):
try:
for process_name in self.processes_to_check:
# osp16/python3 process is "neutron-server:"
if process_name == 'neutron-server' and \
self.oc_procs_df.query('PROCESS=="{}"'.format(
process_name)).empty:
process_name = 'neutron-server:'
if not self.oc_procs_df.query('PROCESS=="{}"'.format(
process_name)).empty:
LOG.info("overcloud processes status checks: "
"process {} is "
"in running state".format(process_name))
continue
else:
LOG.info("Failure : overcloud processes status checks:"
"process {} is not running ".format(
process_name))
return True
raise OvercloudProcessesException(
process_error="process {} is not running ".format(
process_name))
# if all procs are running we can return true
return True
except OvercloudProcessesException:
LOG.info('Retrying overcloud processes checks attempt '
'{} of 360'.format(attempt_number))
time.sleep(1)
self.oc_procs_df = overcloud.get_overcloud_nodes_dataframe(
get_overcloud_node_processes_table)
# exhausted all retries
tobiko.fail('Not all overcloud processes are running !\n')