From a575b79451490bba300e9fae13215bc64c48541a Mon Sep 17 00:00:00 2001 From: Eduardo Olivares Date: Thu, 19 Dec 2024 12:25:08 +0100 Subject: [PATCH] Use run_on_master_controller to run command when possible This method logs the command executed and its output, so it could help to troubleshoot some issues. Change-Id: If1de40baad8d7d956724b4d666e66169b23708d3 --- .../tests/scenario/base.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/whitebox_neutron_tempest_plugin/tests/scenario/base.py b/whitebox_neutron_tempest_plugin/tests/scenario/base.py index 057021f..f8e5f9b 100644 --- a/whitebox_neutron_tempest_plugin/tests/scenario/base.py +++ b/whitebox_neutron_tempest_plugin/tests/scenario/base.py @@ -112,6 +112,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase): @classmethod def run_on_master_controller(cls, cmd): + LOG.debug("Command: %s", cmd) if WB_CONF.openstack_type == 'podified': output = cls.proxy_host_client.exec_command(cmd) if WB_CONF.openstack_type == 'devstack': @@ -268,7 +269,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase): nodes = [] inventory_data = yaml.safe_load( - cls.proxy_host_client.exec_command( + cls.run_on_master_controller( 'cat ' + WB_CONF.proxy_host_inventory_path)) is_crc = False ocps = inventory_data['all']['children']['ocps'] @@ -277,11 +278,11 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase): is_crc = True else: # create ocp_node_yaml_list - ocp_node_list = cls.proxy_host_client.exec_command( + ocp_node_list = cls.run_on_master_controller( "{} get nodes -o name".format(cls.OC)).splitlines() ocp_node_yaml_list = [] for ocp_node in ocp_node_list: - output = cls.proxy_host_client.exec_command( + output = cls.run_on_master_controller( "{} get {} -o yaml".format(cls.OC, ocp_node)) ocp_node_yaml_list.append(yaml.safe_load(output)) @@ -324,7 +325,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase): cls.keys_data = { 'id_cifw_key': id_cifw_key} if hasattr(cls, 'ocp_nodes_key_path'): - devscripts_key = cls.proxy_host_client.exec_command( + devscripts_key = cls.run_on_master_controller( 'cat ' + cls.ocp_nodes_key_path) cls.keys_data['devscripts_key'] = devscripts_key for host in cls.nodes_data: @@ -381,7 +382,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase): else: filters = "cut -d'/' -f 2 | grep {}".format(service) - pods_output = cls.proxy_host_client.exec_command( + pods_output = cls.run_on_master_controller( "{} | {}; true".format(pods_list, filters)) return [pod.strip() for pod in pods_output.splitlines()] @@ -391,10 +392,10 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase): # since it's the only service that existing tests are using if service == 'neutron': pod = cls.get_pods_of_service(service)[0] - return cls.proxy_host_client.exec_command( + return cls.run_on_master_controller( '{} rsh {} find {} -type f'.format( cls.OC, pod, os.path.split( - cls.neutron_conf)[0])).strip().split('\n') + cls.neutron_conf)[0])).split('\n') # TODO(mblue): next gen computes configuration set should be done too, # 'oc patch' for data plane would need more steps and triggers deployment @@ -435,17 +436,17 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase): "'cp /etc/neutron/neutron.conf.d/{2} /tmp/ && " "chmod g+w /tmp/{2}'").format( cls.OC, service_pod, custom_file) - cls.proxy_host_client.exec_command(copy_config) + cls.run_on_master_controller(copy_config) for config_option in config_list: combine_conf_cmd = ( "{0} rsh {1} bash -c '" "crudini --set /tmp/{2} {3} {4} {5}'").format( cls.OC, service_pod, custom_file, config_option.section, config_option.parameter, config_option.value) - cls.proxy_host_client.exec_command(combine_conf_cmd) + cls.run_on_master_controller(combine_conf_cmd) read_conf_cmd = "{0} rsh {1} bash -c 'cat /tmp/{2}'".format( cls.OC, service_pod, custom_file) - combined_conf = cls.proxy_host_client.exec_command(read_conf_cmd) + combined_conf = cls.run_on_master_controller(read_conf_cmd) combined_conf_ind = combined_conf.replace('\n', '\n' + 8 * ' ') patch_buffer = ( 'spec:\n' @@ -459,7 +460,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase): cmd = ("{0} patch $({0} get oscp -o name) --type=merge " "--patch '".format(cls.OC) + patch_buffer + "'") LOG.debug("Set configuration command:\n%s", cmd) - output = cls.proxy_host_client.exec_command(cmd) + output = cls.run_on_master_controller(cmd) LOG.debug("Output:\n%s", output) if not cfg_change and '(no change)' in output: # No config change done, no pod replacement. @@ -1077,8 +1078,7 @@ class TrafficFlowTest(BaseTempestWhiteboxTestCase): cmd = ("{} get pods --field-selector=status.phase=Running " "-o custom-columns=NODE:.spec.nodeName,NAME:.metadata.name " "-l service=ovn-controller-ovs".format(cls.OC)) - output = cls.proxy_host_client.exec_command( - cmd).strip().splitlines() + output = cls.run_on_master_controller(cmd).splitlines() for line in output: for node in cls.nodes: # split('.')[0] ensures that we always compare short names. @@ -1281,7 +1281,7 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase): @classmethod def get_podified_ovn_db_cmd(cls, db): # use the first pod from the list, in case of multiple replicas - db_pod = cls.proxy_host_client.exec_command( + db_pod = cls.run_on_master_controller( '{} get pods -l service=ovsdbserver-{} -o name'.format( cls.OC, db)).splitlines()[0].strip() command_items = [cls.OC, 'rsh', db_pod, 'ovn-{}ctl'.format(db)] @@ -1290,7 +1290,7 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase): # leader DB instance cmd = '{} get ovndbcluster ovndbcluster-{} '.format(cls.OC, db) cmd += '-o jsonpath="{.status.internalDbAddress}"' - all_db_addresses = cls.proxy_host_client.exec_command(cmd).strip() + all_db_addresses = cls.run_on_master_controller(cmd) command_items.append('--db="{}"'.format(all_db_addresses)) # obtain the ovsdb-server command running on this pod @@ -1303,7 +1303,7 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase): '-- setsid', 'ps -o command -C ovsdb-server --no-headers -ww', '|| true')) - ovsdb_server_cmd = cls.proxy_host_client.exec_command(cmd) + ovsdb_server_cmd = cls.run_on_master_controller(cmd) assert 'ovsdb-server' in ovsdb_server_cmd, \ "command '{}' returned unexpected output: {}".format( cmd, ovsdb_server_cmd) @@ -1355,14 +1355,14 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase): def get_router_gateway_chassis_by_id(self, chassis_id): res = self.run_on_master_controller( - self.sbctl + " get chassis " + chassis_id + " hostname").rstrip() + self.sbctl + " get chassis " + chassis_id + " hostname") return res.replace('"', '').split('.')[0] def get_router_port_gateway_mtu(self, router_port_id): cmd = (self.nbctl + " get logical_router_port lrp-" + router_port_id + " options:gateway_mtu") return int( - self.run_on_master_controller(cmd).rstrip().strip('"')) + self.run_on_master_controller(cmd).strip('"')) def get_item_uuid(self, db, item, search_string): ovn_db = self.sbctl if db == 'sb' else self.nbctl @@ -1428,7 +1428,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase): def resource_setup(cls): super(BaseDisruptiveTempestTestCase, cls).resource_setup() try: - cls.proxy_host_client.exec_command( + cls.run_on_master_controller( "timeout 10 ssh {} virsh list".format(WB_CONF.hypervisor_host)) cls.hypervisor_host = WB_CONF.hypervisor_host return @@ -1438,12 +1438,12 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase): ".ssh/config file.", WB_CONF.hypervisor_host) # Depending on ci-fmw version and/or setup, .ssh/config file could # include an entry for either hypervisor or hypervisor-1 - host = cls.proxy_host_client.exec_command( + host = cls.run_on_master_controller( r"grep 'Host.*\ \(hypervisor\|hypervisor-1\)$' ~/.ssh/config " - "| cut -d' ' -f 2").strip() + "| cut -d' ' -f 2") try: - cls.proxy_host_client.exec_command( + cls.run_on_master_controller( "timeout 10 ssh {} virsh list".format(host)) except lib_exceptions.SSHExecCommandFailed: raise cls.skipException( @@ -1457,20 +1457,20 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase): cmd = ("timeout 10 ssh {} sudo virsh list --all --name " "| grep -w {}").format( cls.hypervisor_host, host) - return cls.proxy_host_client.exec_command(cmd).strip() + return cls.run_on_master_controller(cmd) @classmethod def is_host_state_is_shut_off(cls, host): cmd = ("timeout 10 ssh {} virsh list --state-shutoff | grep -w {} " "|| true".format(cls.hypervisor_host, host)) - output = cls.proxy_host_client.exec_command(cmd) + output = cls.run_on_master_controller(cmd) return host in output @classmethod def is_host_loginable(cls, host): cmd = "timeout 10 ssh {} ssh {} hostname || true".format( cls.hypervisor_host, host) - output = cls.proxy_host_client.exec_command(cmd) + output = cls.run_on_master_controller(cmd) return host in output @classmethod @@ -1479,7 +1479,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase): raise cls.skipException("Power operations are not allowed") cmd = "timeout 10 ssh {} sudo virsh destroy {}".format( cls.hypervisor_host, cls.find_host_virsh_name(host)) - cls.proxy_host_client.exec_command(cmd) + cls.run_on_master_controller(cmd) common_utils.wait_until_true( lambda: cls.is_host_state_is_shut_off(host), timeout=30, sleep=5) @@ -1490,7 +1490,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase): raise cls.skipException("Power operations are not allowed") cmd = "timeout 10 ssh {} sudo virsh start {}".format( cls.hypervisor_host, cls.find_host_virsh_name(host)) - cls.proxy_host_client.exec_command(cmd) + cls.run_on_master_controller(cmd) # TODO(rsafrono): implement and apply additional health checks common_utils.wait_until_true( lambda: cls.is_host_loginable(host), @@ -1502,7 +1502,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase): raise cls.skipException("Power operations are not allowed") cmd = "timeout 10 ssh {} sudo virsh reboot {}".format( cls.hypervisor_host, cls.find_host_virsh_name(host)) - cls.proxy_host_client.exec_command(cmd) + cls.run_on_master_controller(cmd) common_utils.wait_until_true( lambda: cls.is_host_loginable(host), timeout=120, sleep=5) @@ -1510,9 +1510,9 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase): def ensure_overcloud_nodes_active(self): """Checks all openstack nodes are up, otherwise activates them. """ - hosts = self.proxy_host_client.exec_command( + hosts = self.run_on_master_controller( "timeout 10 ssh {} sudo virsh list --all --name".format( - self.hypervisor_host)).strip().split() + self.hypervisor_host)).split() for host in hosts: if self.is_host_state_is_shut_off(host): self.power_on_host(host)