x
/
tobiko
Code Issues Proposed changes

start all nova instances if any, to not break containers before after test 

also make container state before after retryable

Change-Id: I2a0566e2bb185c172a095caea6a699b9e2b181ba
This commit is contained in:
pinikomarov 2020-03-03 23:51:07 +02:00
parent 1d90b2e3b3
commit 12a4f881ba
3 changed files with 43 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tobiko/tests/faults/ha/test_cloud_recovery.py
View File

@ -77,11 +77,12 @@ class RebootNodesTest(testtools.TestCase):
nodes_health_check()
computes_containers_dict_after = \
containers.list_containers(group='compute')
nova.start_all_instances()
containers.assert_equal_containers_state(
computes_containers_dict_before, computes_containers_dict_after)
# [..]
# more tests to folow
# more tests to follow
# run health checks
# os faults stop rabbitmq service on one controller
# faults stop rabbitmq service on one controller
# run health checks again

51
tobiko/tripleo/containers.py
View File

@ -1,5 +1,7 @@
from __future__ import absolute_import
import time
from oslo_log import log
import pandas
@ -192,34 +194,45 @@ def dataframe_difference(df1, df2, which=None):
def assert_equal_containers_state(expected_containers_list,
actual_containers_list):
actual_containers_list, timeout=120,
interval=2):
"""compare container with states from two lists"""
failures = []
start = time.time()
expected_containers_list_df = pandas.DataFrame(
get_container_states_list(expected_containers_list),
columns=['container_host', 'container_name', 'container_state'])
actual_containers_list_df = pandas.DataFrame(
get_container_states_list(actual_containers_list),
columns=['container_host', 'container_name', 'container_state'])
LOG.info('expected_containers_list_df: {} '.format(
expected_containers_list_df.to_string(index=False)))
LOG.info('actual_containers_list_df: {} '.format(
actual_containers_list_df.to_string(index=False)))
while time.time() - start < timeout:
# execute a dataframe diff between the excpected and actual containers
expected_containers_state_changed = \
dataframe_difference(expected_containers_list_df,
actual_containers_list_df)
# check for changed state containers
if not expected_containers_state_changed.empty:
failures.append('expected containers changed state ! : \n\n{}'.format(
expected_containers_state_changed.to_string(index=False)))
failures = []
actual_containers_list_df = pandas.DataFrame(
get_container_states_list(actual_containers_list),
columns=['container_host', 'container_name', 'container_state'])
LOG.info('expected_containers_list_df: {} '.format(
expected_containers_list_df.to_string(index=False)))
LOG.info('actual_containers_list_df: {} '.format(
actual_containers_list_df.to_string(index=False)))
# execute a dataframe diff between the excpected and actual containers
expected_containers_state_changed = \
dataframe_difference(expected_containers_list_df,
actual_containers_list_df)
# check for changed state containers
if not expected_containers_state_changed.empty:
failures.append('expected containers changed state ! : '
'\n\n{}'.format(expected_containers_state_changed.
to_string(index=False)))
LOG.info('container states mismatched:\n{}\n'.format(failures))
time.sleep(interval)
actual_containers_list = list_containers(group='compute')
else:
LOG.info("assert_equal_containers_state :"
" OK, all containers are on the same state")
return
if failures:
tobiko.fail('container states mismatched:\n{!s}', '\n'.join(failures))
else:
LOG.info("assert_equal_containers_state :"
" OK, all containers are on the same state")

8
tobiko/tripleo/nova.py
View File

@ -37,3 +37,11 @@ def check_nova_services_health(timeout=120, interval=2):
if failures:
tobiko.fail(
'nova agents are unhealthy:\n{!s}', '\n'.join(failures))
def start_all_instances():
"""try to start all stopped overcloud instances"""
nova_client = nova.get_nova_client()
servers = nova_client.servers.list()
for instance in servers:
nova.activate_server(instance)