diff --git a/tobiko/openstack/topology/__init__.py b/tobiko/openstack/topology/__init__.py index 7bf54e73a..dd7b3c43f 100644 --- a/tobiko/openstack/topology/__init__.py +++ b/tobiko/openstack/topology/__init__.py @@ -63,3 +63,5 @@ set_default_openstack_topology_class = ( _topology.set_default_openstack_topology_class) verify_osp_version = _topology.verify_osp_version get_config_setting = _topology.get_config_setting +node_name_from_hostname = _topology.node_name_from_hostname +remove_duplications = _topology.remove_duplications diff --git a/tobiko/openstack/topology/_topology.py b/tobiko/openstack/topology/_topology.py index 17d550155..f91c49e10 100644 --- a/tobiko/openstack/topology/_topology.py +++ b/tobiko/openstack/topology/_topology.py @@ -311,6 +311,8 @@ class OpenStackTopology(tobiko.SharedFixture): has_containers = False + container_runtime_cmd = 'docker' + config_file_mappings = { 'ml2_conf.ini': '/etc/neutron/plugins/ml2/ml2_conf.ini', 'bgp-agent.conf': '/etc/ovn-bgp-agent/bgp-agent.conf' @@ -396,6 +398,15 @@ class OpenStackTopology(tobiko.SharedFixture): ssh_client=node.ssh_client) return digger + def assert_containers_running(self, expected_containers, + group=None, + full_name=True, bool_check=False, + nodenames=None): + pass + + def list_containers_df(self, group=None): + pass + def discover_nodes(self): self.discover_ssh_proxy_jump_node() self.discover_configured_nodes() diff --git a/tobiko/podified/_topology.py b/tobiko/podified/_topology.py index 20fe4ac23..356d05efd 100644 --- a/tobiko/podified/_topology.py +++ b/tobiko/podified/_topology.py @@ -22,6 +22,7 @@ from tobiko.openstack import neutron from tobiko.openstack import topology from tobiko.podified import _edpm from tobiko.podified import _openshift +from tobiko.podified import containers from tobiko import rhosp from tobiko.shell import ssh @@ -67,9 +68,29 @@ class PodifiedTopology(rhosp.RhospTopology): neutron.FRR: 'frr' } - def __init__(self): - super(PodifiedTopology, self).__init__() - self.ocp_workers = {} + sidecar_container_list = [ + 'neutron-haproxy-ovnmeta', + 'neutron-dnsmasq-qdhcp' + ] + + @property + def ignore_containers_list(self): + return self.sidecar_container_list + + def assert_containers_running(self, expected_containers, + group=None, + full_name=True, bool_check=False, + nodenames=None): + group = group or ALL_COMPUTES_GROUP_NAME + return containers.assert_containers_running( + group=group, + expected_containers=expected_containers, + full_name=full_name, + bool_check=bool_check, + nodenames=nodenames) + + def list_containers_df(self, group=None): + return containers.list_containers_df(group) def add_node(self, hostname: typing.Optional[str] = None, diff --git a/tobiko/podified/containers.py b/tobiko/podified/containers.py new file mode 100644 index 000000000..75d0437fc --- /dev/null +++ b/tobiko/podified/containers.py @@ -0,0 +1,351 @@ +from __future__ import absolute_import + +import functools +import os +import time +import typing + +from oslo_log import log +import pandas + +import tobiko +from tobiko import config +from tobiko.openstack import neutron +from tobiko.openstack import topology +from tobiko import rhosp as rhosp_topology +from tobiko.rhosp import containers as rhosp_containers + + +CONF = config.CONF +LOG = log.getLogger(__name__) + + +class ContainerRuntimeFixture(tobiko.SharedFixture): + + runtime: typing.Optional[rhosp_containers.ContainerRuntime] = None + + def setup_fixture(self): + self.runtime = self.get_runtime() + + def cleanup_fixture(self): + self.runtime = None + + @staticmethod + def get_runtime() -> typing.Optional[rhosp_containers.ContainerRuntime]: + """return handle to the container runtime""" + return rhosp_containers.PODMAN_RUNTIME + + +def get_container_runtime() -> rhosp_containers.ContainerRuntime: + runtime = tobiko.setup_fixture(ContainerRuntimeFixture).runtime + return runtime + + +def get_container_runtime_name() -> str: + return get_container_runtime().runtime_name + + +def is_docker() -> bool: + return False + + +def is_podman() -> bool: + return True + + +def has_container_runtime() -> bool: + return True + + +@functools.lru_cache() +def list_node_containers(ssh_client): + """returns a list of containers and their run state""" + return get_container_runtime().list_containers(ssh_client=ssh_client) + + +def get_container_client(ssh_client=None): + """returns a list of containers and their run state""" + return get_container_runtime().get_client(ssh_client=ssh_client) + + +def list_containers_df(group=None): + actual_containers_list = list_containers(group) + return pandas.DataFrame( + get_container_states_list(actual_containers_list), + columns=['container_host', 'container_name', 'container_state']) + + +def list_containers(group=None): + """get list of containers in running state + from specified node group + returns : a list of overcloud_node's running containers""" + + # moved here from topology + # reason : Workaround for : + # AttributeError: module 'tobiko.openstack.topology' has no + # attribute 'container_runtime' + + if group is None: + group = 'compute' + containers_list = tobiko.Selection() + openstack_nodes = topology.list_openstack_nodes(group=group) + + for node in openstack_nodes: + LOG.debug(f"List containers for node {node.name}") + node_containers_list = list_node_containers(ssh_client=node.ssh_client) + containers_list.extend(node_containers_list) + return containers_list + + +expected_containers_file = os.path.expanduser( + '~/expected_containers_list_df.csv') + + +def save_containers_state_to_file(expected_containers_list,): + expected_containers_list_df = pandas.DataFrame( + get_container_states_list(expected_containers_list), + columns=['container_host', 'container_name', 'container_state']) + expected_containers_list_df.to_csv( + expected_containers_file) + return expected_containers_file + + +def assert_containers_running(group, expected_containers, full_name=True, + bool_check=False, nodenames=None): + + """assert that all containers specified in the list are running + on the specified openstack group(controller or compute etc..) + if bool_check is True then return only True or false without failing""" + + if is_docker(): + LOG.info('not checking common containers since we are on docker') + return + + failures = [] + + openstack_nodes = topology.list_openstack_nodes(group=group, + hostnames=nodenames) + for node in openstack_nodes: + node_containers = list_node_containers(ssh_client=node.ssh_client) + containers_list_df = pandas.DataFrame( + get_container_states_list(node_containers), + columns=['container_host', 'container_name', 'container_state']) + # check that the containers are present + LOG.info('node: {} containers list : {}'.format( + node.name, containers_list_df.to_string(index=False))) + for container in expected_containers: + # get container attrs dataframe + if full_name: + container_attrs = containers_list_df.query( + 'container_name == "{}"'.format(container)) + else: + container_attrs = containers_list_df[ + containers_list_df['container_name']. + str.contains(container)] + # check if the container exists + LOG.info('checking container: {}'.format(container)) + if container_attrs.empty: + failures.append( + 'expected container {} not found on node {} ! : \n\n'. + format(container, node.name)) + # if container exists, check it is running + else: + # only one running container is expected + container_running_attrs = container_attrs.query( + 'container_state=="running"') + if container_running_attrs.empty: + failures.append( + 'expected container {} is not running on node {} , ' + 'its state is {}! : \n\n'.format( + container, node.name, + container_attrs.container_state.values.item())) + elif len(container_running_attrs) > 1: + failures.append( + 'only one running container {} was expected on ' + 'node {}, but got {}! : \n\n'.format( + container, node.name, + len(container_running_attrs))) + + if not bool_check and failures: + tobiko.fail( + 'container states mismatched:\n{}'.format('\n'.join(failures)), + rhosp_containers.ContainerMismatchException) + + elif bool_check and failures: + return False + + else: + LOG.info('All specified containers are in running state! ') + return True + + +def assert_ovn_containers_running(): + if not neutron.has_ovn(): + LOG.info("Networking OVN not configured") + return + ovn_containers = ['ovn_metadata_agent', + 'ovn_controller'] + groups = ['edpm-compute', 'edpm-networker'] + for group in groups: + assert_containers_running(group, ovn_containers, full_name=False) + LOG.info("Networking OVN containers verified in running state") + + +def get_container_states_list(containers_list, + include_container_objects=False): + container_states_list = tobiko.Selection() + container_states_list.extend([comparable_container_keys( + container, include_container_objects=include_container_objects) for + container in containers_list]) + return container_states_list + + +def comparable_container_keys(container, include_container_objects=False): + """returns the tuple : 'container_host','container_name', + 'container_state, container object if specified' + """ + # Differenciate between podman_ver3 with podman-py from earlier api + if include_container_objects: + return (rhosp_topology.ip_to_hostname( + container.client.base_url.netloc.rsplit('_')[1]), + container.attrs['Names'][0], container.attrs['State'], + container) + else: + return (rhosp_topology.ip_to_hostname( + container.client.base_url.netloc.rsplit('_')[1]), + container.attrs['Names'][0], + container.attrs['State']) + + +@functools.lru_cache() +def list_containers_objects_df(): + containers_list = list_containers() + containers_objects_list_df = pandas.DataFrame( + get_container_states_list( + containers_list, include_container_objects=True), + columns=['container_host', 'container_name', + 'container_state', 'container_object']) + return containers_objects_list_df + + +def get_edpm_container(container_name=None, container_host=None, + partial_container_name=None): + """gets an container object by name on specified host + container""" + con_obj_df = list_containers_objects_df() + if partial_container_name and container_host: + con_obj_df = con_obj_df[con_obj_df['container_name'].str.contains( + partial_container_name)] + contaniner_obj = con_obj_df.query( + 'container_host == "{container_host}"'.format( + container_host=container_host))['container_object'] + elif container_host: + contaniner_obj = con_obj_df.query( + 'container_name == "{container_name}"' + ' and container_host == "{container_host}"'. + format(container_host=container_host, + container_name=container_name)).container_object + else: + contaniner_obj = con_obj_df.query( + 'container_name == "{container_name}"'. + format(container_name=container_name)).container_object + if not contaniner_obj.empty: + return contaniner_obj.values[0] + else: + tobiko.fail('container {} not found!'.format(container_name)) + + +def action_on_container(action: str, + container_name=None, + container_host=None, + partial_container_name=None): + """take a container and perform an action on it + actions are as defined in : podman/libs/containers.py:14/164""" + + LOG.debug(f"Executing '{action}' action on container " + f"'{container_name}@{container_host}'...") + container = get_edpm_container( + container_name=container_name, + container_host=container_host, + partial_container_name=partial_container_name) + + container_class: typing.Type = type(container) + # we get the specified action as function from podman lib + action_method: typing.Optional[typing.Callable] = getattr( + container_class, action, None) + if action_method is None: + raise TypeError(f"Unsupported container action for class :" + f" {container_class}") + if not callable(action_method): + raise TypeError( + f"Attribute '{container_class.__qualname__}.{action}' value " + f" is not a method: {action_method!r}") + LOG.debug(f"Calling '{action_method}' action on container " + f"'{container}'") + return action_method(container) + + +def assert_equal_containers_state(expected_containers_list=None, + timeout=120, interval=2, + recreate_expected=False): + + """compare all edpm container states with using two lists: + one is current , the other some past list + first time this method runs it creates a file holding overcloud + containers' states: ~/expected_containers_list_df.csv' + second time it creates a current containers states list and + compares them, they must be identical""" + + # if we have a file or an explicit variable use that , otherwise create + # and return + if recreate_expected or (not expected_containers_list and + not os.path.exists(expected_containers_file)): + save_containers_state_to_file(list_containers()) + return + + elif expected_containers_list: + expected_containers_list_df = pandas.DataFrame( + get_container_states_list(expected_containers_list), + columns=['container_host', 'container_name', 'container_state']) + + elif os.path.exists(expected_containers_file): + expected_containers_list_df = pandas.read_csv( + expected_containers_file) + + failures = [] + start = time.time() + error_info = 'Output explanation: left_only is the original state, ' \ + 'right_only is the new state' + + while time.time() - start < timeout: + + failures = [] + actual_containers_list_df = list_containers_df() + + LOG.info('expected_containers_list_df: {} '.format( + expected_containers_list_df.to_string(index=False))) + LOG.info('actual_containers_list_df: {} '.format( + actual_containers_list_df.to_string(index=False))) + + # execute a `dataframe` diff between the expected and actual containers + expected_containers_state_changed = \ + rhosp_containers.dataframe_difference( + expected_containers_list_df, + actual_containers_list_df) + # check for changed state containerstopology + if not expected_containers_state_changed.empty: + failures.append('expected containers changed state ! : ' + '\n\n{}\n{}'.format( + expected_containers_state_changed. + to_string(index=False), error_info)) + LOG.info('container states mismatched:\n{}\n'.format(failures)) + time.sleep(interval) + # clear cache to obtain new data + list_node_containers.cache_clear() + else: + LOG.info("assert_equal_containers_state :" + " OK, all containers are on the same state") + return + if failures: + tobiko.fail('container states mismatched:\n{!s}', '\n'.join( + failures)) diff --git a/tobiko/rhosp/__init__.py b/tobiko/rhosp/__init__.py index 2e05fb878..5c6b9ca9e 100644 --- a/tobiko/rhosp/__init__.py +++ b/tobiko/rhosp/__init__.py @@ -21,3 +21,5 @@ RhospNode = _topology.RhospNode get_rhosp_release = _version_utils.get_rhosp_release get_rhosp_version = _version_utils.get_rhosp_version + +ip_to_hostname = _topology.ip_to_hostname diff --git a/tobiko/rhosp/_topology.py b/tobiko/rhosp/_topology.py index 93b314271..af4a21c5f 100644 --- a/tobiko/rhosp/_topology.py +++ b/tobiko/rhosp/_topology.py @@ -27,6 +27,27 @@ from tobiko.shell import ssh LOG = log.getLogger(__name__) +def get_ip_to_nodes_dict(group, openstack_nodes=None): + if not openstack_nodes: + openstack_nodes = topology.list_openstack_nodes(group=group) + ip_to_nodes_dict = {str(node.public_ip): node.name for node in + openstack_nodes} + return ip_to_nodes_dict + + +def ip_to_hostname(oc_ip, group=None): + ip_to_nodes_dict = get_ip_to_nodes_dict(group) + oc_ipv6 = oc_ip.replace(".", ":") + if netaddr.valid_ipv4(oc_ip) or netaddr.valid_ipv6(oc_ip): + return ip_to_nodes_dict[oc_ip] + elif netaddr.valid_ipv6(oc_ipv6): + LOG.debug("The provided string was a modified IPv6 address: %s", + oc_ip) + return ip_to_nodes_dict[oc_ipv6] + else: + tobiko.fail("wrong IP value provided %s" % oc_ip) + + class RhospTopology(topology.OpenStackTopology): """Base topology for Red Hat OpenStack deployments. @@ -36,6 +57,11 @@ class RhospTopology(topology.OpenStackTopology): """ has_containers = True + container_runtime_cmd = 'podman' + + @property + def ignore_containers_list(self): + return None class RhospNode(topology.OpenStackTopologyNode): diff --git a/tobiko/rhosp/containers.py b/tobiko/rhosp/containers.py new file mode 100644 index 000000000..9eaac5adb --- /dev/null +++ b/tobiko/rhosp/containers.py @@ -0,0 +1,123 @@ +from __future__ import absolute_import + +import abc +import re +import typing + +from oslo_log import log + +import tobiko +from tobiko.openstack import topology +from tobiko import podman +from tobiko import docker +from tobiko.shell import ssh + + +LOG = log.getLogger(__name__) + + +class ContainerRuntime(abc.ABC): + runtime_name: str + version_pattern: typing.Pattern + + def match_version(self, version: str) -> bool: + for version_line in version.splitlines(): + if self.version_pattern.match(version_line) is not None: + return True + return False + + def get_client(self, ssh_client): + for attempt in tobiko.retry(timeout=60.0, + interval=5.0): + try: + client = self._get_client(ssh_client=ssh_client) + break + # TODO chose a better exception type + except Exception: + if attempt.is_last: + raise + LOG.debug('Unable to connect to docker server', + exc_info=1) + ssh.reset_default_ssh_port_forward_manager() + else: + raise RuntimeError("Broken retry loop") + return client + + def _get_client(self, ssh_client): + raise NotImplementedError + + def list_containers(self, ssh_client): + raise NotImplementedError + + +class DockerContainerRuntime(ContainerRuntime): + runtime_name = 'docker' + version_pattern = re.compile('Docker version .*', re.IGNORECASE) + + def _get_client(self, ssh_client): + return docker.get_docker_client(ssh_client=ssh_client, + sudo=True).connect() + + def list_containers(self, ssh_client): + client = self.get_client(ssh_client=ssh_client) + return docker.list_docker_containers(client=client) + + +class PodmanContainerRuntime(ContainerRuntime): + runtime_name = 'podman' + version_pattern = re.compile('Podman version .*', re.IGNORECASE) + + def _get_client(self, ssh_client): + return podman.get_podman_client(ssh_client=ssh_client).connect() + + def list_containers(self, ssh_client): + client = self.get_client(ssh_client=ssh_client) + return podman.list_podman_containers(client=client) + + +DOCKER_RUNTIME = DockerContainerRuntime() +PODMAN_RUNTIME = PodmanContainerRuntime() +CONTAINER_RUNTIMES = [PODMAN_RUNTIME, DOCKER_RUNTIME] + + +class ContainerMismatchException(tobiko.TobikoException): + pass + + +def remove_containers_from_comparison(comparable_containers_df): + """remove any containers if comparing them with previous status is not + necessary or makes no sense + """ + os_topology = topology.get_openstack_topology() + for row in comparable_containers_df.iterrows(): + for ignore_container in os_topology.ignore_containers_list: + if ignore_container in str(row): + LOG.info(f'container {ignore_container} has changed state, ' + 'but that\'s ok - it will be ignored and the test ' + f'will not fail due to this: {str(row)}') + # if a pcs resource is found , we drop that row + comparable_containers_df.drop(row[0], inplace=True) + # this row was already dropped, go to next row + break + + +def dataframe_difference(df1, df2, which=None): + """Find rows which are different between two DataFrames.""" + comparison_df = df1.merge(df2, + indicator='same_state', + how='outer') + # return only non identical rows + if which is None: + diff_df = comparison_df[comparison_df['same_state'] != 'both'] + + else: + diff_df = comparison_df[comparison_df['same_state'] == which] + + # if the list of different state containers includes sidecar containers, + # ignore them because the existence of these containers depends on the + # created resources + # if the list of different state containers includes pacemaker resources, + # ignore them since the sanity and fault tests check pacemaker status too + remove_containers_from_comparison(diff_df) + + return diff_df diff --git a/tobiko/tests/faults/neutron/test_agents.py b/tobiko/tests/faults/neutron/test_agents.py index 9cf70c2af..85445df33 100644 --- a/tobiko/tests/faults/neutron/test_agents.py +++ b/tobiko/tests/faults/neutron/test_agents.py @@ -31,8 +31,6 @@ from tobiko.openstack import tests from tobiko.openstack import topology from tobiko.shell import ping from tobiko.shell import sh -from tobiko.tripleo import containers -from tobiko.tripleo import overcloud LOG = log.getLogger(__name__) @@ -69,10 +67,7 @@ class BaseAgentTest(testtools.TestCase): @property def container_runtime_name(self): - if overcloud.has_overcloud(): - return containers.get_container_runtime_name() - else: - return 'docker' + return topology.get_openstack_topology().container_runtime_cmd def get_agent_container_name(self, agent_name): try: @@ -88,8 +83,9 @@ class BaseAgentTest(testtools.TestCase): self.agent_name) if not self.container_name: return - oc_containers_df = containers.list_containers_df().query( - f'container_name == "{self.container_name}"') + oc_containers_df = \ + topology.get_openstack_topology().list_containers_df().query( + f'container_name == "{self.container_name}"') LOG.debug( f"{self.container_name} container found:\n{oc_containers_df}") @@ -166,8 +162,7 @@ class BaseAgentTest(testtools.TestCase): ''' self._do_agent_action(START, hosts) if self.container_name: - containers.assert_containers_running( - group='overcloud', + topology.get_openstack_topology().assert_containers_running( expected_containers=[self.container_name], nodenames=hosts or self.hosts) @@ -181,8 +176,7 @@ class BaseAgentTest(testtools.TestCase): ''' self._do_agent_action(RESTART, hosts) if self.container_name: - containers.assert_containers_running( - group='overcloud', + topology.get_openstack_topology().assert_containers_running( expected_containers=[self.container_name], nodenames=hosts or self.hosts) @@ -213,8 +207,7 @@ class BaseAgentTest(testtools.TestCase): sudo=True) if self.container_name: - containers.assert_containers_running( - group='overcloud', + topology.get_openstack_topology().assert_containers_running( expected_containers=[self.container_name], nodenames=hosts or self.hosts) diff --git a/tobiko/tests/functional/tripleo/test_containers.py b/tobiko/tests/functional/tripleo/test_containers.py index 032773af0..466f53633 100644 --- a/tobiko/tests/functional/tripleo/test_containers.py +++ b/tobiko/tests/functional/tripleo/test_containers.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import testtools +from tobiko.rhosp import containers as rhosp_containers from tobiko.tripleo import containers @@ -23,7 +24,7 @@ class RuntimeRuntimeTest(testtools.TestCase): def test_get_container_runtime(self): runtime = containers.get_container_runtime() - self.assertIsInstance(runtime, containers.ContainerRuntime) + self.assertIsInstance(runtime, rhosp_containers.ContainerRuntime) def test_list_containers(self): containers_list = containers.list_containers() diff --git a/tobiko/tests/sanity/containers/test_containers.py b/tobiko/tests/sanity/containers/test_containers.py index a71b1da79..bb71b3992 100644 --- a/tobiko/tests/sanity/containers/test_containers.py +++ b/tobiko/tests/sanity/containers/test_containers.py @@ -26,6 +26,7 @@ import pandas import tobiko from tobiko.openstack import neutron from tobiko.openstack import topology +from tobiko.rhosp import containers as rhosp_containers from tobiko.shell import sh from tobiko import tripleo from tobiko.tripleo import containers @@ -34,8 +35,15 @@ from tobiko.tripleo import containers LOG = log.getLogger(__name__) +class BaseContainersHealtTest(testtools.TestCase): + + def _assert_containers_running(self, group, expected_containers): + topology.get_openstack_topology().assert_containers_running( + group=group, expected_containers=expected_containers) + + @tripleo.skip_if_missing_overcloud -class ContainersHealthTest(testtools.TestCase): +class ContainersHealthTest(BaseContainersHealtTest): # TODO(eolivare): refactor this class, because it replicates some code from # tobiko/tripleo/containers.py and its tests may be duplicating what # test_0vercloud_health_check already covers when it calls @@ -49,158 +57,150 @@ class ContainersHealthTest(testtools.TestCase): def test_cinder_api(self): """check that all common tripleo containers are running""" - containers.assert_containers_running('controller', ['cinder_api']) + self._assert_containers_running('controller', ['cinder_api']) @tripleo.skip_if_ceph_rgw() def test_swift_rsync(self): - containers.assert_containers_running('controller', ['swift_rsync']) + self._assert_containers_running('controller', ['swift_rsync']) @tripleo.skip_if_ceph_rgw() def test_swift_proxy(self): - containers.assert_containers_running('controller', ['swift_proxy']) + self._assert_containers_running('controller', ['swift_proxy']) @tripleo.skip_if_ceph_rgw() def test_swift_object_updater(self): - containers.assert_containers_running('controller', - ['swift_object_updater']) + self._assert_containers_running('controller', ['swift_object_updater']) @tripleo.skip_if_ceph_rgw() def test_swift_object_server(self): - containers.assert_containers_running('controller', - ['swift_object_server']) + self._assert_containers_running('controller', ['swift_object_server']) @tripleo.skip_if_ceph_rgw() def test_swift_object_replicator(self): - containers.assert_containers_running('controller', - ['swift_object_replicator']) + self._assert_containers_running('controller', + ['swift_object_replicator']) @tripleo.skip_if_ceph_rgw() def test_swift_object_expirer(self): - containers.assert_containers_running('controller', - ['swift_object_expirer']) + self._assert_containers_running('controller', ['swift_object_expirer']) @tripleo.skip_if_ceph_rgw() def test_swift_object_auditor(self): - containers.assert_containers_running('controller', - ['swift_object_auditor']) + self._assert_containers_running('controller', ['swift_object_auditor']) @tripleo.skip_if_ceph_rgw() def test_swift_container_updater(self): - containers.assert_containers_running('controller', - ['swift_container_updater']) + self._assert_containers_running('controller', + ['swift_container_updater']) @tripleo.skip_if_ceph_rgw() def test_swift_container_server(self): - containers.assert_containers_running('controller', - ['swift_container_server']) + self._assert_containers_running('controller', + ['swift_container_server']) @tripleo.skip_if_ceph_rgw() def test_swift_container_replicator(self): - containers.assert_containers_running('controller', - ['swift_container_replicator']) + self._assert_containers_running('controller', + ['swift_container_replicator']) @tripleo.skip_if_ceph_rgw() def test_swift_container_auditor(self): - containers.assert_containers_running('controller', - ['swift_container_auditor']) + self._assert_containers_running('controller', + ['swift_container_auditor']) @tripleo.skip_if_ceph_rgw() def test_swift_account_server(self): - containers.assert_containers_running('controller', - ['swift_account_server']) + self._assert_containers_running('controller', ['swift_account_server']) @tripleo.skip_if_ceph_rgw() def test_swift_account_replicator(self): - containers.assert_containers_running('controller', - ['swift_account_replicator']) + self._assert_containers_running('controller', + ['swift_account_replicator']) @tripleo.skip_if_ceph_rgw() def test_swift_account_reaper(self): - containers.assert_containers_running('controller', - ['swift_account_reaper']) + self._assert_containers_running('controller', ['swift_account_reaper']) @tripleo.skip_if_ceph_rgw() def test_swift_account_auditor(self): - containers.assert_containers_running('controller', - ['swift_account_auditor']) + self._assert_containers_running('controller', + ['swift_account_auditor']) def test_nova_vnc_proxy(self): - containers.assert_containers_running('controller', ['nova_vnc_proxy']) + self._assert_containers_running('controller', ['nova_vnc_proxy']) def test_nova_scheduler(self): - containers.assert_containers_running('controller', ['nova_scheduler']) + self._assert_containers_running('controller', ['nova_scheduler']) def test_nova_metadata(self): - containers.assert_containers_running('controller', ['nova_metadata']) + self._assert_containers_running('controller', ['nova_metadata']) def test_nova_conductor(self): - containers.assert_containers_running('controller', ['nova_conductor']) + self._assert_containers_running('controller', ['nova_conductor']) def test_nova_api_cron(self): - containers.assert_containers_running('controller', ['nova_api_cron']) + self._assert_containers_running('controller', ['nova_api_cron']) def test_nova_api(self): - containers.assert_containers_running('controller', ['nova_api']) + self._assert_containers_running('controller', ['nova_api']) def test_neutron_api(self): - containers.assert_containers_running('controller', ['neutron_api']) + self._assert_containers_running('controller', ['neutron_api']) def test_memcached(self): - containers.assert_containers_running('controller', ['memcached']) + self._assert_containers_running('controller', ['memcached']) def test_controller_logrotate_crond(self): - containers.assert_containers_running('controller', ['logrotate_crond']) + self._assert_containers_running('controller', ['logrotate_crond']) def test_keystone(self): - containers.assert_containers_running('controller', ['keystone']) + self._assert_containers_running('controller', ['keystone']) def test_controller_iscsid(self): - containers.assert_containers_running('controller', ['iscsid']) + self._assert_containers_running('controller', ['iscsid']) def test_horizon(self): - containers.assert_containers_running('controller', ['horizon']) + self._assert_containers_running('controller', ['horizon']) def test_heat_engine(self): - containers.assert_containers_running('controller', ['heat_engine']) + self._assert_containers_running('controller', ['heat_engine']) def test_heat_api_cron(self): - containers.assert_containers_running('controller', ['heat_api_cron']) + self._assert_containers_running('controller', ['heat_api_cron']) def test_heat_api_cfn(self): - containers.assert_containers_running('controller', ['heat_api_cfn']) + self._assert_containers_running('controller', ['heat_api_cfn']) def test_heat_api(self): - containers.assert_containers_running('controller', ['heat_api']) + self._assert_containers_running('controller', ['heat_api']) def test_glance_api(self): - containers.assert_containers_running('controller', ['glance_api']) + self._assert_containers_running('controller', ['glance_api']) def test_cinder_scheduler(self): - containers.assert_containers_running('controller', - ['cinder_scheduler']) + self._assert_containers_running('controller', ['cinder_scheduler']) def test_cinder_api_cron(self): - containers.assert_containers_running('controller', ['cinder_api_cron']) + self._assert_containers_running('controller', ['cinder_api_cron']) def test_compute_iscsid(self): - containers.assert_containers_running('compute', ['iscsid']) + self._assert_containers_running('compute', ['iscsid']) def test_compute_logrotate_crond(self): - containers.assert_containers_running('compute', ['logrotate_crond']) + self._assert_containers_running('compute', ['logrotate_crond']) def test_nova_compute(self): - containers.assert_containers_running('compute', ['nova_compute']) + self._assert_containers_running('compute', ['nova_compute']) def test_nova_libvirt(self): nova_libvirt = containers.get_libvirt_container_name() - containers.assert_containers_running('compute', [nova_libvirt]) + self._assert_containers_running('compute', [nova_libvirt]) def test_nova_migration_target(self): - containers.assert_containers_running('compute', - ['nova_migration_target']) + self._assert_containers_running('compute', ['nova_migration_target']) def test_nova_virtlogd(self): - containers.assert_containers_running('compute', ['nova_virtlogd']) + self._assert_containers_running('compute', ['nova_virtlogd']) def test_ovn_containers_running(self): containers.assert_ovn_containers_running() @@ -244,8 +244,8 @@ class ContainersHealthTest(testtools.TestCase): # execute a `dataframe` diff between the expected # and actual containers expected_containers_state_changed = \ - containers.dataframe_difference(expected_containers_list_df, - actual_containers_list_df) + rhosp_containers.dataframe_difference( + expected_containers_list_df, actual_containers_list_df) # check for changed state containerstopology if not expected_containers_state_changed.empty: failures.append('expected containers changed state ! : ' diff --git a/tobiko/tripleo/_topology.py b/tobiko/tripleo/_topology.py index 97d1a04d3..6949874d1 100644 --- a/tobiko/tripleo/_topology.py +++ b/tobiko/tripleo/_topology.py @@ -31,6 +31,7 @@ from tobiko.shell import sh from tobiko.shell import ssh from tobiko.tripleo import _overcloud from tobiko.tripleo import _undercloud +from tobiko.tripleo import containers CONF = config.CONF LOG = log.getLogger(__name__) @@ -84,12 +85,47 @@ class TripleoTopology(rhosp.RhospTopology): neutron.SERVER: '/var/log/containers/neutron/server.log*', } + _container_runtime_cmd = None + + pcs_resource_list = ['haproxy', 'galera', 'redis', 'ovn-dbs', 'cinder', + 'rabbitmq', 'manila', 'ceph', 'pacemaker'] + + sidecar_container_list = [ + 'neutron-haproxy-ovnmeta', 'neutron-haproxy-qrouter', + 'neutron-dnsmasq-qdhcp', 'neutron-keepalived-qrouter', 'neutron-radvd'] + + @property + def container_runtime_cmd(self): + if self._container_runtime_cmd is None: + self._container_runtime_cmd = \ + containers.get_container_runtime_name() + return self._container_runtime_cmd + + @property + def ignore_containers_list(self): + return self.pcs_resource_list + self.sidecar_container_list + def create_node(self, name, ssh_client, **kwargs): return TripleoTopologyNode(topology=self, name=name, ssh_client=ssh_client, **kwargs) + def assert_containers_running(self, expected_containers, + group=None, + full_name=True, bool_check=False, + nodenames=None): + group = group or 'overcloud' + return containers.assert_containers_running( + group=group, + expected_containers=expected_containers, + full_name=full_name, + bool_check=bool_check, + nodenames=nodenames) + + def list_containers_df(self, group=None): + return containers.list_containers_df(group) + def discover_nodes(self): self.discover_ssh_proxy_jump_node() self.discover_undercloud_nodes() @@ -237,32 +273,11 @@ def setup_tripleo_topology(): topology.set_default_openstack_topology_class(TripleoTopology) -def get_ip_to_nodes_dict(openstack_nodes=None): - if not openstack_nodes: - openstack_nodes = topology.list_openstack_nodes(group='overcloud') - ip_to_nodes_dict = {str(node.public_ip): node.name for node in - openstack_nodes} - return ip_to_nodes_dict - - def str_is_not_ip(check_str): letters = re.compile('[A-Za-z]') return bool(letters.match(check_str)) -def ip_to_hostname(oc_ip): - ip_to_nodes_dict = get_ip_to_nodes_dict() - oc_ipv6 = oc_ip.replace(".", ":") - if netaddr.valid_ipv4(oc_ip) or netaddr.valid_ipv6(oc_ip): - return ip_to_nodes_dict[oc_ip] - elif netaddr.valid_ipv6(oc_ipv6): - LOG.debug("The provided string was a modified IPv6 address: %s", - oc_ip) - return ip_to_nodes_dict[oc_ipv6] - else: - tobiko.fail("wrong IP value provided %s" % oc_ip) - - def actual_node_groups(groups): """return only existing node groups""" return set(groups).intersection(topology.list_openstack_node_groups()) diff --git a/tobiko/tripleo/containers.py b/tobiko/tripleo/containers.py index 75e743fc7..f9961f2f4 100644 --- a/tobiko/tripleo/containers.py +++ b/tobiko/tripleo/containers.py @@ -1,6 +1,5 @@ from __future__ import absolute_import -import abc import functools import os import re @@ -13,86 +12,21 @@ import pandas import tobiko from tobiko import config from tobiko import podman -from tobiko import docker from tobiko.openstack import neutron from tobiko.openstack import topology +from tobiko import rhosp as rhosp_topology +from tobiko.rhosp import containers as rhosp_containers from tobiko.shell import sh -from tobiko.shell import ssh from tobiko.tripleo import overcloud -from tobiko.tripleo import topology as tripleo_topology CONF = config.CONF LOG = log.getLogger(__name__) -class ContainerRuntime(abc.ABC): - runtime_name: str - version_pattern: typing.Pattern - - def match_version(self, version: str) -> bool: - for version_line in version.splitlines(): - if self.version_pattern.match(version_line) is not None: - return True - return False - - def get_client(self, ssh_client): - for attempt in tobiko.retry(timeout=60.0, - interval=5.0): - try: - client = self._get_client(ssh_client=ssh_client) - break - # TODO chose a better exception type - except Exception: - if attempt.is_last: - raise - LOG.debug('Unable to connect to docker server', - exc_info=1) - ssh.reset_default_ssh_port_forward_manager() - else: - raise RuntimeError("Broken retry loop") - return client - - def _get_client(self, ssh_client): - raise NotImplementedError - - def list_containers(self, ssh_client): - raise NotImplementedError - - -class DockerContainerRuntime(ContainerRuntime): - runtime_name = 'docker' - version_pattern = re.compile('Docker version .*', re.IGNORECASE) - - def _get_client(self, ssh_client): - return docker.get_docker_client(ssh_client=ssh_client, - sudo=True).connect() - - def list_containers(self, ssh_client): - client = self.get_client(ssh_client=ssh_client) - return docker.list_docker_containers(client=client) - - -class PodmanContainerRuntime(ContainerRuntime): - runtime_name = 'podman' - version_pattern = re.compile('Podman version .*', re.IGNORECASE) - - def _get_client(self, ssh_client): - return podman.get_podman_client(ssh_client=ssh_client).connect() - - def list_containers(self, ssh_client): - client = self.get_client(ssh_client=ssh_client) - return podman.list_podman_containers(client=client) - - -DOCKER_RUNTIME = DockerContainerRuntime() -PODMAN_RUNTIME = PodmanContainerRuntime() -CONTAINER_RUNTIMES = [PODMAN_RUNTIME, DOCKER_RUNTIME] - - class ContainerRuntimeFixture(tobiko.SharedFixture): - runtime: typing.Optional[ContainerRuntime] = None + runtime: typing.Optional[rhosp_containers.ContainerRuntime] = None def setup_fixture(self): if overcloud.has_overcloud(): @@ -102,7 +36,7 @@ class ContainerRuntimeFixture(tobiko.SharedFixture): self.runtime = None @staticmethod - def get_runtime() -> typing.Optional[ContainerRuntime]: + def get_runtime() -> typing.Optional[rhosp_containers.ContainerRuntime]: """check what container runtime is running and return a handle to it""" # TODO THIS LOCKS SSH CLIENT TO CONTROLLER @@ -112,7 +46,7 @@ class ContainerRuntimeFixture(tobiko.SharedFixture): ssh_client=node.ssh_client) except sh.ShellCommandFailed: continue - for runtime in CONTAINER_RUNTIMES: + for runtime in rhosp_containers.CONTAINER_RUNTIMES: for version in [result.stdout, result.stderr]: if runtime.match_version(version): return runtime @@ -121,7 +55,7 @@ class ContainerRuntimeFixture(tobiko.SharedFixture): "controller node") -def get_container_runtime() -> ContainerRuntime: +def get_container_runtime() -> rhosp_containers.ContainerRuntime: runtime = tobiko.setup_fixture(ContainerRuntimeFixture).runtime return runtime @@ -200,10 +134,6 @@ def save_containers_state_to_file(expected_containers_list,): return expected_containers_file -class ContainerMismatchException(tobiko.TobikoException): - pass - - def assert_containers_running(group, expected_containers, full_name=True, bool_check=False, nodenames=None): @@ -263,7 +193,7 @@ def assert_containers_running(group, expected_containers, full_name=True, if not bool_check and failures: tobiko.fail( 'container states mismatched:\n{}'.format('\n'.join(failures)), - ContainerMismatchException) + rhosp_containers.ContainerMismatchException) elif bool_check and failures: return False @@ -469,22 +399,22 @@ def comparable_container_keys(container, include_container_objects=False): # Differenciate between podman_ver3 with podman-py from earlier api if is_podman(): if podman.Podman_Version_3(): - con_host_name_stat_obj_tuple = (tripleo_topology.ip_to_hostname( + con_host_name_stat_obj_tuple = (rhosp_topology.ip_to_hostname( container.client.base_url.netloc.rsplit('_')[1]), container.attrs[ 'Names'][0], container.attrs['State'], container) - con_host_name_stat_tuple = (tripleo_topology.ip_to_hostname( + con_host_name_stat_tuple = (rhosp_topology.ip_to_hostname( container.client.base_url.netloc.rsplit('_')[1]), container.attrs[ 'Names'][0], container.attrs['State']) else: - con_host_name_stat_obj_tuple = (tripleo_topology.ip_to_hostname( + con_host_name_stat_obj_tuple = (rhosp_topology.ip_to_hostname( container._client._context.hostname), # pylint: disable=W0212 container.data['names'], container.data['status'], container) - con_host_name_stat_tuple = (tripleo_topology.ip_to_hostname( + con_host_name_stat_tuple = (rhosp_topology.ip_to_hostname( container._client._context.hostname), # pylint: disable=W0212 container.data['names'], container.data['status']) @@ -584,54 +514,6 @@ def get_container_states_list(containers_list, return container_states_list -pcs_resource_list = ['haproxy', 'galera', 'redis', 'ovn-dbs', 'cinder', - 'rabbitmq', 'manila', 'ceph', 'pacemaker'] - - -sidecar_container_list = [ - 'neutron-haproxy-ovnmeta', 'neutron-haproxy-qrouter', - 'neutron-dnsmasq-qdhcp', 'neutron-keepalived-qrouter', 'neutron-radvd'] - - -def remove_containers_from_comparison(comparable_containers_df): - """remove any containers if comparing them with previous status is not - necessary or makes no sense - """ - ignore_containers_list = pcs_resource_list + sidecar_container_list - for row in comparable_containers_df.iterrows(): - for ignore_container in ignore_containers_list: - if ignore_container in str(row): - LOG.info(f'container {ignore_container} has changed state, ' - 'but that\'s ok - it will be ignored and the test ' - f'will not fail due to this: {str(row)}') - # if a pcs resource is found , we drop that row - comparable_containers_df.drop(row[0], inplace=True) - # this row was already dropped, go to next row - break - - -def dataframe_difference(df1, df2, which=None): - """Find rows which are different between two DataFrames.""" - comparison_df = df1.merge(df2, - indicator='same_state', - how='outer') - # return only non identical rows - if which is None: - diff_df = comparison_df[comparison_df['same_state'] != 'both'] - - else: - diff_df = comparison_df[comparison_df['same_state'] == which] - - # if the list of different state containers includes sidecar containers, - # ignore them because the existence of these containers depends on the - # created resources - # if the list of different state containers includes pacemaker resources, - # ignore them since the sanity and fault tests check pacemaker status too - remove_containers_from_comparison(diff_df) - - return diff_df - - def assert_equal_containers_state(expected_containers_list=None, timeout=120, interval=2, recreate_expected=False): @@ -676,8 +558,9 @@ def assert_equal_containers_state(expected_containers_list=None, # execute a `dataframe` diff between the expected and actual containers expected_containers_state_changed = \ - dataframe_difference(expected_containers_list_df, - actual_containers_list_df) + rhosp_containers.dataframe_difference( + expected_containers_list_df, + actual_containers_list_df) # check for changed state containerstopology if not expected_containers_state_changed.empty: failures.append('expected containers changed state ! : '