From 0181e3da6586746c6433c99a459afffa99a4e51a Mon Sep 17 00:00:00 2001 From: pkomarov Date: Mon, 11 Nov 2019 14:51:11 +0200 Subject: [PATCH] add ability to get list of nodes running systemctl services Change-Id: I411528e26a792d3906beeab02052a5cd6b74b085 --- .../functional/tripleo/test_overcloud.py | 18 +++ tobiko/tripleo/overcloud.py | 20 +++ tobiko/tripleo/services.py | 115 ++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 tobiko/tripleo/services.py diff --git a/tobiko/tests/functional/tripleo/test_overcloud.py b/tobiko/tests/functional/tripleo/test_overcloud.py index 2cf6472bd..00df3e448 100644 --- a/tobiko/tests/functional/tripleo/test_overcloud.py +++ b/tobiko/tests/functional/tripleo/test_overcloud.py @@ -24,6 +24,7 @@ from tobiko import config from tobiko.openstack import nova from tobiko.tripleo import overcloud from tobiko.tripleo import pacemaker +from tobiko.tripleo import services import tobiko @@ -102,3 +103,20 @@ class OvercloudPacemakerTest(testtools.TestCase): def test_pacemaker_resources_health(self): pcs_health = pacemaker.PacemakerResourcesStatus() self.assertTrue(pcs_health.all_healthy) + + +@overcloud.skip_if_missing_overcloud +class OvercloudServicesTest(testtools.TestCase): + + """ + Assert that a subset of overcloud services are in running state + across the overcloud nodes + """ + def test_get_services_resource_table(self): + oss = services.OvercloudServicesStatus() + self.assertIsInstance(oss.oc_services_df, + pd.DataFrame) + + def test_overcloud_services(self): + oss = services.OvercloudServicesStatus() + self.assertTrue(oss.basic_overcloud_services_running) diff --git a/tobiko/tripleo/overcloud.py b/tobiko/tripleo/overcloud.py index fa8cf10f1..6d7f5ca44 100644 --- a/tobiko/tripleo/overcloud.py +++ b/tobiko/tripleo/overcloud.py @@ -16,6 +16,7 @@ from __future__ import absolute_import import io import os +import pandas import six import tobiko @@ -163,3 +164,22 @@ def setup_overcloud_keystone_crederntials(): if has_overcloud(): keystone.DEFAULT_KEYSTONE_CREDENTIALS_FIXTURES.append( OvercloudKeystoneCredentialsFixture) + + +def get_overcloud_nodes_dataframe(oc_node_df_function): + """ + :param oc_node_df_function : a function that queries a oc node + using a cli command and returns a datraframe with an added + hostname field. + + This function concats oc nodes dataframes into a unified overcloud + dataframe, seperated by hostname field + + :return: dataframe of all overcloud nodes processes + """ + oc_nodes_selection = list_overcloud_nodes() + oc_nodes_names = [node.name for node in oc_nodes_selection] + oc_nodes_dfs = [oc_node_df_function(node_name) for + node_name in oc_nodes_names] + oc_procs_df = pandas.concat(oc_nodes_dfs, ignore_index=True) + return oc_procs_df diff --git a/tobiko/tripleo/services.py b/tobiko/tripleo/services.py new file mode 100644 index 000000000..a720ee18a --- /dev/null +++ b/tobiko/tripleo/services.py @@ -0,0 +1,115 @@ +from __future__ import absolute_import + +from oslo_log import log +import pandas +import six + +import tobiko +from tobiko.tripleo import overcloud +from tobiko.shell import sh + + +LOG = log.getLogger(__name__) + + +class OvercloudServiceException(tobiko.TobikoException): + message = "not all overcloud nodes services are in active state" + + +def get_overcloud_node_services_table(hostname): + """ + get services table from overcloud node + + returns : +auditd.service|loaded|active|running|SecurityAuditingService +auth-rpcgss-module.service|loaded|inactivedead|KernelModulesupportingRPCSEC_GSS +blk-availability.service|loaded|active|exited|Availabilityofblockdevices +brandbot.service|loaded|inactivedead|FlexibleBrandingService +certmonger.service|loaded|active|running|CertificatemonitoringandPKIenrollment +cinder-lvm-losetup.service|loaded|inactivedead|CinderLVMlosetup +cloud-config.service|loaded|active|exited|Applythesettingsspecifiedincloud-con +cloud-final.service|loaded|active|exited|Executeclouduser/finalscripts +cloud-init-local.service|loaded|active|exited|Initialcloud-initjob(pre-network) +cloud-init.service|loaded|active|exited|Initialcloud-initjob(metadataservicecr) + + :return: dataframe of overcloud node services + """ + + ssh_client = overcloud.overcloud_ssh_client(hostname) + output = sh.execute( + "systemctl -a --no-pager --plain --no-legend|" + "sed \'s/\\s\\s/|/g\'|sed \'s/||*/|/g\'|sed \'s@ @@g\'", + ssh_client=ssh_client).stdout + stream = six.StringIO(output) + table = pandas.read_csv(stream, sep='|', header=None, skiprows=0) + table.replace(to_replace=' ', value="", regex=True, inplace=True) + table.columns = ['UNIT', 'loaded_state', 'active_state', + 'low_level_state', 'UNIT_DESCRIPTION'] + table['overcloud_node'] = hostname + + LOG.debug("Got overcloud nodes services status :\n%s", table) + return table + + +def get_overcloud_nodes_running_service(service): + """ + Check what nodes are running the specified service or unit + process: exact str of a process name as seen in systemctl -a + :return: list of overcloud nodes + """ + oc_procs_df = overcloud.get_overcloud_nodes_dataframe( + get_overcloud_node_services_table) + oc_nodes_running_service = oc_procs_df.query('UNIT=="{}"'.format(service))[ + 'overcloud_node'].unique() + return oc_nodes_running_service + + +def check_if_process_running_on_overcloud(process): + """ + Check what nodes are running the specifies + process: exact str of a process name as seen in ps -axw -o "%c" + :return: list of overcloud nodes + """ + oc_services_df = overcloud.get_overcloud_nodes_dataframe( + get_overcloud_node_services_table) + if not oc_services_df.query('UNIT=="{}"'.format(process)).empty: + return True + else: + return False + + +class OvercloudServicesStatus(object): + """ + class to handle services checks, + checks that all of these are running in the overcloud: + 'corosync.service','iptables.service','network.service','ntpd.service', + 'pacemaker.service','rpcbind.service','sshd.service' + + """ + def __init__(self): + self.services_to_check = ['corosync.service', 'iptables.service', + 'network.service', 'ntpd.service', + 'pacemaker.service', 'rpcbind.service', + 'sshd.service'] + + self.oc_services_df = overcloud.get_overcloud_nodes_dataframe( + get_overcloud_node_services_table) + + @property + def basic_overcloud_services_running(self): + """ + Checks that the oc_services dataframe has all of the list services + running + :return: Bool + """ + for service_name in self.services_to_check: + if not self.oc_services_df.query('UNIT=="{}"'.format( + service_name)).empty: + LOG.info("overcloud processes status checks: process {} is " + "in running state".format(service_name)) + continue + else: + LOG.info("Failure : overcloud processes status checks: " + "process {} is not running ".format(service_name)) + raise OvercloudServiceException() + return True