Use run_on_master_controller to run command when possible
This method logs the command executed and its output, so it could help to troubleshoot some issues. Change-Id: If1de40baad8d7d956724b4d666e66169b23708d3
This commit is contained in:
Eduardo Olivares
parent
3429e220a5
commit
a575b79451
@ -112,6 +112,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
|
||||
|
||||
@classmethod
|
||||
def run_on_master_controller(cls, cmd):
|
||||
LOG.debug("Command: %s", cmd)
|
||||
if WB_CONF.openstack_type == 'podified':
|
||||
output = cls.proxy_host_client.exec_command(cmd)
|
||||
if WB_CONF.openstack_type == 'devstack':
|
||||
@ -268,7 +269,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
|
||||
|
||||
nodes = []
|
||||
inventory_data = yaml.safe_load(
|
||||
cls.proxy_host_client.exec_command(
|
||||
cls.run_on_master_controller(
|
||||
'cat ' + WB_CONF.proxy_host_inventory_path))
|
||||
is_crc = False
|
||||
ocps = inventory_data['all']['children']['ocps']
|
||||
@ -277,11 +278,11 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
|
||||
is_crc = True
|
||||
else:
|
||||
# create ocp_node_yaml_list
|
||||
ocp_node_list = cls.proxy_host_client.exec_command(
|
||||
ocp_node_list = cls.run_on_master_controller(
|
||||
"{} get nodes -o name".format(cls.OC)).splitlines()
|
||||
ocp_node_yaml_list = []
|
||||
for ocp_node in ocp_node_list:
|
||||
output = cls.proxy_host_client.exec_command(
|
||||
output = cls.run_on_master_controller(
|
||||
"{} get {} -o yaml".format(cls.OC, ocp_node))
|
||||
ocp_node_yaml_list.append(yaml.safe_load(output))
|
||||
|
||||
@ -324,7 +325,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
|
||||
cls.keys_data = {
|
||||
'id_cifw_key': id_cifw_key}
|
||||
if hasattr(cls, 'ocp_nodes_key_path'):
|
||||
devscripts_key = cls.proxy_host_client.exec_command(
|
||||
devscripts_key = cls.run_on_master_controller(
|
||||
'cat ' + cls.ocp_nodes_key_path)
|
||||
cls.keys_data['devscripts_key'] = devscripts_key
|
||||
for host in cls.nodes_data:
|
||||
@ -381,7 +382,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
|
||||
else:
|
||||
filters = "cut -d'/' -f 2 | grep {}".format(service)
|
||||
|
||||
pods_output = cls.proxy_host_client.exec_command(
|
||||
pods_output = cls.run_on_master_controller(
|
||||
"{} | {}; true".format(pods_list, filters))
|
||||
return [pod.strip() for pod in pods_output.splitlines()]
|
||||
|
||||
@ -391,10 +392,10 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
|
||||
# since it's the only service that existing tests are using
|
||||
if service == 'neutron':
|
||||
pod = cls.get_pods_of_service(service)[0]
|
||||
return cls.proxy_host_client.exec_command(
|
||||
return cls.run_on_master_controller(
|
||||
'{} rsh {} find {} -type f'.format(
|
||||
cls.OC, pod, os.path.split(
|
||||
cls.neutron_conf)[0])).strip().split('\n')
|
||||
cls.neutron_conf)[0])).split('\n')
|
||||
|
||||
# TODO(mblue): next gen computes configuration set should be done too,
|
||||
# 'oc patch' for data plane would need more steps and triggers deployment
|
||||
@ -435,17 +436,17 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
|
||||
"'cp /etc/neutron/neutron.conf.d/{2} /tmp/ && "
|
||||
"chmod g+w /tmp/{2}'").format(
|
||||
cls.OC, service_pod, custom_file)
|
||||
cls.proxy_host_client.exec_command(copy_config)
|
||||
cls.run_on_master_controller(copy_config)
|
||||
for config_option in config_list:
|
||||
combine_conf_cmd = (
|
||||
"{0} rsh {1} bash -c '"
|
||||
"crudini --set /tmp/{2} {3} {4} {5}'").format(
|
||||
cls.OC, service_pod, custom_file, config_option.section,
|
||||
config_option.parameter, config_option.value)
|
||||
cls.proxy_host_client.exec_command(combine_conf_cmd)
|
||||
cls.run_on_master_controller(combine_conf_cmd)
|
||||
read_conf_cmd = "{0} rsh {1} bash -c 'cat /tmp/{2}'".format(
|
||||
cls.OC, service_pod, custom_file)
|
||||
combined_conf = cls.proxy_host_client.exec_command(read_conf_cmd)
|
||||
combined_conf = cls.run_on_master_controller(read_conf_cmd)
|
||||
combined_conf_ind = combined_conf.replace('\n', '\n' + 8 * ' ')
|
||||
patch_buffer = (
|
||||
'spec:\n'
|
||||
@ -459,7 +460,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
|
||||
cmd = ("{0} patch $({0} get oscp -o name) --type=merge "
|
||||
"--patch '".format(cls.OC) + patch_buffer + "'")
|
||||
LOG.debug("Set configuration command:\n%s", cmd)
|
||||
output = cls.proxy_host_client.exec_command(cmd)
|
||||
output = cls.run_on_master_controller(cmd)
|
||||
LOG.debug("Output:\n%s", output)
|
||||
if not cfg_change and '(no change)' in output:
|
||||
# No config change done, no pod replacement.
|
||||
@ -1077,8 +1078,7 @@ class TrafficFlowTest(BaseTempestWhiteboxTestCase):
|
||||
cmd = ("{} get pods --field-selector=status.phase=Running "
|
||||
"-o custom-columns=NODE:.spec.nodeName,NAME:.metadata.name "
|
||||
"-l service=ovn-controller-ovs".format(cls.OC))
|
||||
output = cls.proxy_host_client.exec_command(
|
||||
cmd).strip().splitlines()
|
||||
output = cls.run_on_master_controller(cmd).splitlines()
|
||||
for line in output:
|
||||
for node in cls.nodes:
|
||||
# split('.')[0] ensures that we always compare short names.
|
||||
@ -1281,7 +1281,7 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase):
|
||||
@classmethod
|
||||
def get_podified_ovn_db_cmd(cls, db):
|
||||
# use the first pod from the list, in case of multiple replicas
|
||||
db_pod = cls.proxy_host_client.exec_command(
|
||||
db_pod = cls.run_on_master_controller(
|
||||
'{} get pods -l service=ovsdbserver-{} -o name'.format(
|
||||
cls.OC, db)).splitlines()[0].strip()
|
||||
command_items = [cls.OC, 'rsh', db_pod, 'ovn-{}ctl'.format(db)]
|
||||
@ -1290,7 +1290,7 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase):
|
||||
# leader DB instance
|
||||
cmd = '{} get ovndbcluster ovndbcluster-{} '.format(cls.OC, db)
|
||||
cmd += '-o jsonpath="{.status.internalDbAddress}"'
|
||||
all_db_addresses = cls.proxy_host_client.exec_command(cmd).strip()
|
||||
all_db_addresses = cls.run_on_master_controller(cmd)
|
||||
command_items.append('--db="{}"'.format(all_db_addresses))
|
||||
|
||||
# obtain the ovsdb-server command running on this pod
|
||||
@ -1303,7 +1303,7 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase):
|
||||
'-- setsid',
|
||||
'ps -o command -C ovsdb-server --no-headers -ww',
|
||||
'|| true'))
|
||||
ovsdb_server_cmd = cls.proxy_host_client.exec_command(cmd)
|
||||
ovsdb_server_cmd = cls.run_on_master_controller(cmd)
|
||||
assert 'ovsdb-server' in ovsdb_server_cmd, \
|
||||
"command '{}' returned unexpected output: {}".format(
|
||||
cmd, ovsdb_server_cmd)
|
||||
@ -1355,14 +1355,14 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase):
|
||||
|
||||
def get_router_gateway_chassis_by_id(self, chassis_id):
|
||||
res = self.run_on_master_controller(
|
||||
self.sbctl + " get chassis " + chassis_id + " hostname").rstrip()
|
||||
self.sbctl + " get chassis " + chassis_id + " hostname")
|
||||
return res.replace('"', '').split('.')[0]
|
||||
|
||||
def get_router_port_gateway_mtu(self, router_port_id):
|
||||
cmd = (self.nbctl + " get logical_router_port lrp-" + router_port_id +
|
||||
" options:gateway_mtu")
|
||||
return int(
|
||||
self.run_on_master_controller(cmd).rstrip().strip('"'))
|
||||
self.run_on_master_controller(cmd).strip('"'))
|
||||
|
||||
def get_item_uuid(self, db, item, search_string):
|
||||
ovn_db = self.sbctl if db == 'sb' else self.nbctl
|
||||
@ -1428,7 +1428,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
|
||||
def resource_setup(cls):
|
||||
super(BaseDisruptiveTempestTestCase, cls).resource_setup()
|
||||
try:
|
||||
cls.proxy_host_client.exec_command(
|
||||
cls.run_on_master_controller(
|
||||
"timeout 10 ssh {} virsh list".format(WB_CONF.hypervisor_host))
|
||||
cls.hypervisor_host = WB_CONF.hypervisor_host
|
||||
return
|
||||
@ -1438,12 +1438,12 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
|
||||
".ssh/config file.", WB_CONF.hypervisor_host)
|
||||
# Depending on ci-fmw version and/or setup, .ssh/config file could
|
||||
# include an entry for either hypervisor or hypervisor-1
|
||||
host = cls.proxy_host_client.exec_command(
|
||||
host = cls.run_on_master_controller(
|
||||
r"grep 'Host.*\ \(hypervisor\|hypervisor-1\)$' ~/.ssh/config "
|
||||
"| cut -d' ' -f 2").strip()
|
||||
"| cut -d' ' -f 2")
|
||||
|
||||
try:
|
||||
cls.proxy_host_client.exec_command(
|
||||
cls.run_on_master_controller(
|
||||
"timeout 10 ssh {} virsh list".format(host))
|
||||
except lib_exceptions.SSHExecCommandFailed:
|
||||
raise cls.skipException(
|
||||
@ -1457,20 +1457,20 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
|
||||
cmd = ("timeout 10 ssh {} sudo virsh list --all --name "
|
||||
"| grep -w {}").format(
|
||||
cls.hypervisor_host, host)
|
||||
return cls.proxy_host_client.exec_command(cmd).strip()
|
||||
return cls.run_on_master_controller(cmd)
|
||||
|
||||
@classmethod
|
||||
def is_host_state_is_shut_off(cls, host):
|
||||
cmd = ("timeout 10 ssh {} virsh list --state-shutoff | grep -w {} "
|
||||
"|| true".format(cls.hypervisor_host, host))
|
||||
output = cls.proxy_host_client.exec_command(cmd)
|
||||
output = cls.run_on_master_controller(cmd)
|
||||
return host in output
|
||||
|
||||
@classmethod
|
||||
def is_host_loginable(cls, host):
|
||||
cmd = "timeout 10 ssh {} ssh {} hostname || true".format(
|
||||
cls.hypervisor_host, host)
|
||||
output = cls.proxy_host_client.exec_command(cmd)
|
||||
output = cls.run_on_master_controller(cmd)
|
||||
return host in output
|
||||
|
||||
@classmethod
|
||||
@ -1479,7 +1479,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
|
||||
raise cls.skipException("Power operations are not allowed")
|
||||
cmd = "timeout 10 ssh {} sudo virsh destroy {}".format(
|
||||
cls.hypervisor_host, cls.find_host_virsh_name(host))
|
||||
cls.proxy_host_client.exec_command(cmd)
|
||||
cls.run_on_master_controller(cmd)
|
||||
common_utils.wait_until_true(
|
||||
lambda: cls.is_host_state_is_shut_off(host),
|
||||
timeout=30, sleep=5)
|
||||
@ -1490,7 +1490,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
|
||||
raise cls.skipException("Power operations are not allowed")
|
||||
cmd = "timeout 10 ssh {} sudo virsh start {}".format(
|
||||
cls.hypervisor_host, cls.find_host_virsh_name(host))
|
||||
cls.proxy_host_client.exec_command(cmd)
|
||||
cls.run_on_master_controller(cmd)
|
||||
# TODO(rsafrono): implement and apply additional health checks
|
||||
common_utils.wait_until_true(
|
||||
lambda: cls.is_host_loginable(host),
|
||||
@ -1502,7 +1502,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
|
||||
raise cls.skipException("Power operations are not allowed")
|
||||
cmd = "timeout 10 ssh {} sudo virsh reboot {}".format(
|
||||
cls.hypervisor_host, cls.find_host_virsh_name(host))
|
||||
cls.proxy_host_client.exec_command(cmd)
|
||||
cls.run_on_master_controller(cmd)
|
||||
common_utils.wait_until_true(
|
||||
lambda: cls.is_host_loginable(host),
|
||||
timeout=120, sleep=5)
|
||||
@ -1510,9 +1510,9 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
|
||||
def ensure_overcloud_nodes_active(self):
|
||||
"""Checks all openstack nodes are up, otherwise activates them.
|
||||
"""
|
||||
hosts = self.proxy_host_client.exec_command(
|
||||
hosts = self.run_on_master_controller(
|
||||
"timeout 10 ssh {} sudo virsh list --all --name".format(
|
||||
self.hypervisor_host)).strip().split()
|
||||
self.hypervisor_host)).split()
|
||||
for host in hosts:
|
||||
if self.is_host_state_is_shut_off(host):
|
||||
self.power_on_host(host)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user