add tripleo containers health checks

and use it in compute reboot test
done refactoring, unit tests are pasing , system tests are passing

related bz:
https://bugzilla.redhat.com/show_bug.cgi?id=1797892

Change-Id: I2816d03c26e8eeca92c99116e28be52f98aa5141
This commit is contained in:
pinikomarov 2020-02-09 13:21:52 +02:00 committed by Federico Ressi
parent 16c32e366b
commit b957f4a9c8
5 changed files with 198 additions and 36 deletions

View File

@ -30,7 +30,7 @@ def get_docker_client(base_urls=None, ssh_client=None):
def list_docker_containers(client=None, **kwargs):
try:
containers = docker_client(client).containers.list(**kwargs)
containers = docker_client(client).containers.list(all=True, **kwargs)
except _exception.DockerUrlNotFoundError:
return tobiko.Selection()
else:

View File

@ -26,6 +26,7 @@ from six.moves.urllib import parse
import tobiko
from tobiko import docker
from tobiko import podman
from tobiko.shell import ip
from tobiko.shell import ping
from tobiko.shell import sh
@ -97,6 +98,7 @@ def set_default_openstack_topology_class(topology_class):
class OpenStackTopologyNode(object):
_docker_client = None
_podman_client = None
def __init__(self, topology, name, public_ip, ssh_client):
self._topology = weakref.ref(topology)
@ -124,6 +126,14 @@ class OpenStackTopologyNode(object):
ssh_client=self.ssh_client)
return docker_client
@property
def podman_client(self):
podman_client = self._podman_client
if not podman_client:
self._podman_client = podman_client = podman.get_podman_client(
ssh_client=self.ssh_client)
return podman_client
def __repr__(self):
return "{cls!s}<name={name!r}>".format(cls=type(self).__name__,
name=self.name)

View File

@ -19,6 +19,7 @@ import six
import podman
import tobiko
from tobiko.podman import _exception
from tobiko.podman import _shell
@ -76,28 +77,31 @@ class PodmanClientFixture(tobiko.SharedFixture):
def setup_client(self):
# setup podman access via varlink
podman_client_setup_cmds = [
"sudo groupadd -f podman",
"sudo usermod -a -G podman heat-admin",
"sudo chmod o+w /etc/tmpfiles.d",
"sudo echo 'd /run/podman 0750 root heat-admin' > "
"/etc/tmpfiles.d/podman.conf",
"sudo cp /lib/systemd/system/io.podman.socket /etc/systemd/system/"
"io.podman.socket",
"sudo crudini --set /etc/systemd/system/io.podman.socket Socket "
"SocketMode 0660",
"sudo crudini --set /etc/systemd/system/io.podman.socket Socket"
" SocketGroup podman",
"sudo systemctl daemon-reload",
"sudo systemd-tmpfiles --create",
"sudo systemctl enable --now io.podman.socket",
"sudo chown -R root: /run/podman",
"sudo chmod g+rw /run/podman/io.podman",
"sudo systemctl start io.podman.socket"
]
podman_client_setup_cmds = \
"sudo test -f /var/varlink_client_access_setup || \
(sudo groupadd -f podman && \
sudo usermod -a -G podman heat-admin && \
sudo chmod -R o=wxr /etc/tmpfiles.d && \
sudo echo 'd /run/podman 0770 root heat-admin' > \
/etc/tmpfiles.d/podman.conf && \
sudo cp /lib/systemd/system/io.podman.socket \
/etc/systemd/system/io.podman.socket && \
sudo crudini --set /etc/systemd/system/io.podman.socket Socket \
SocketMode 0660 && \
sudo crudini --set /etc/systemd/system/io.podman.socket Socket \
SocketGroup podman && \
sudo systemctl daemon-reload && \
sudo systemd-tmpfiles --create && \
sudo systemctl enable --now io.podman.socket && \
sudo chmod 777 /run/podman && \
sudo chown -R root: /run/podman && \
sudo chmod g+rw /run/podman/io.podman && \
sudo chmod 777 /run/podman/io.podman && \
sudo setenforce 0 && \
sudo systemctl start io.podman.socket && \
sudo touch /var/varlink_client_access_setup)"
for cmd in podman_client_setup_cmds:
sh.execute(cmd, ssh_client=self.ssh_client)
sh.execute(podman_client_setup_cmds, ssh_client=self.ssh_client)
client = self.client
if client is None:
@ -105,6 +109,9 @@ class PodmanClientFixture(tobiko.SharedFixture):
return client
def create_client(self):
for _ in range(360):
try:
podman_remote_socket = self.discover_podman_socket()
podman_remote_socket_uri = 'unix:/tmp/podman.sock'
@ -118,6 +125,9 @@ class PodmanClientFixture(tobiko.SharedFixture):
identity_file='~/.ssh/id_rsa')
client.system.ping()
return client
except (ConnectionRefusedError, ConnectionResetError):
# retry
self.create_client()
def connect(self):
return tobiko.setup_fixture(self).client

View File

@ -6,6 +6,7 @@ from tobiko.shell import sh
from tobiko.tests.faults.ha import cloud_disruptions
from tobiko.tripleo import pacemaker
from tobiko.tripleo import processes
from tobiko.tripleo import containers
from tobiko.openstack import stacks
import tobiko
@ -17,13 +18,12 @@ def nodes_health_check():
# TODO:
# Test existing created servers
# ServerStackResourcesTest().test_server_create()
# check vm create with ssh and ping checks
def check_vm_create(stack_name):
'''stack_name: unique stack name ,
so that each time a new vm is created'''
"""stack_name: unique stack name ,
so that each time a new vm is created"""
# create a vm
stack = stacks.CirrosServerStackFixture(
stack_name=stack_name)
@ -61,8 +61,14 @@ class RebootNodesTest(testtools.TestCase):
def test_reboot_computes_recovery(self):
nodes_health_check()
computes_containers_dict_before = \
containers.list_containers(group='compute')
cloud_disruptions.reset_all_compute_nodes(hard_reset=True)
nodes_health_check()
computes_containers_dict_after = \
containers.list_containers(group='compute')
containers.assert_equal_containers_state(
computes_containers_dict_before, computes_containers_dict_after)
check_vm_create(stack_name=self.id())
# [..]

View File

@ -0,0 +1,136 @@
from __future__ import absolute_import
from oslo_log import log
import pandas
import tobiko
from tobiko import podman
from tobiko import docker
from tobiko.openstack import topology
LOG = log.getLogger(__name__)
def container_runtime():
"""check what container runtime is running
and return a handle to it"""
ssh_client = topology.list_openstack_nodes(group='controller')[
0].ssh_client
if docker.is_docker_running(ssh_client=ssh_client):
return docker
else:
return podman
container_runtime_type = container_runtime()
def list_node_containers(client=None):
"""returns a list of containers and their run state"""
if container_runtime_type == podman:
return container_runtime_type.list_podman_containers(client=client)
elif container_runtime_type == docker:
return container_runtime_type.list_docker_containers(client=client)
def get_container_client(ssh_client=None):
"""returns a list of containers and their run state"""
if container_runtime_type == podman:
return container_runtime_type.get_podman_client(
ssh_client=ssh_client).connect()
elif container_runtime_type == docker:
return container_runtime_type.get_docker_client(
ssh_client=ssh_client).connect()
def list_containers(group=None):
"""get list of containers in running state
from specified node group
returns : a list of overcloud_node's running containers"""
# moved here from topology
# reason : Workaround for :
# AttributeError: module 'tobiko.openstack.topology' has no
# attribute 'container_runtime'
containers_list = tobiko.Selection()
openstack_nodes = topology.list_openstack_nodes(group=group)
for node in openstack_nodes:
ssh_client = node.ssh_client
container_client = get_container_client(ssh_client)
node_containers_list = list_node_containers(client=container_client)
containers_list.extend(node_containers_list)
return containers_list
def comparable_container_keys(container):
"""returns the tuple : 'container_host','container_name',
'container_state'
"""
if container_runtime_type == podman:
return (container._client._context.hostname, # pylint: disable=W0212
container.data['names'], container.data['status'])
elif container_runtime_type == docker:
return (container.attrs['Config']['Hostname'],
container.attrs['State']['Status'], container.attrs['Name'])
def get_container_states_list(containers_list):
container_states_list = tobiko.Selection()
container_states_list.extend([comparable_container_keys(container) for
container in containers_list])
return container_states_list
def dataframe_difference(df1, df2, which=None):
"""Find rows which are different between two DataFrames."""
comparison_df = df1.merge(df2,
indicator='same_state',
how='outer')
if which is None:
diff_df = comparison_df[comparison_df['same_state'] != 'both']
else:
diff_df = comparison_df[comparison_df['same_state'] == which]
return diff_df
def assert_equal_containers_state(expected_containers_list,
actual_containers_list):
"""compare container with states from two lists"""
failures = []
expected_containers_list_df = pandas.DataFrame(
get_container_states_list(expected_containers_list),
columns=['container_host', 'container_name', 'container_state'])
actual_containers_list_df = pandas.DataFrame(
get_container_states_list(actual_containers_list),
columns=['container_host', 'container_name', 'container_state'])
LOG.info('expected_containers_list_df: {} '.format(
expected_containers_list_df.to_string(index=False)))
LOG.info('actual_containers_list_df: {} '.format(
actual_containers_list_df.to_string(index=False)))
# execute a dataframe diff between the excpected and actual containers
expected_containers_state_changed = \
dataframe_difference(expected_containers_list_df,
actual_containers_list_df)
# check for changed state containers
if not expected_containers_state_changed.empty:
failures.append('expected containers changed state ! : \n\n{}'.format(
expected_containers_state_changed.to_string(index=False)))
if failures:
tobiko.fail('container states mismatched:\n{!s}', '\n'.join(failures))
else:
LOG.info("assert_equal_containers_state :"
" OK, all containers are on the same state")