diff --git a/tobiko/tests/faults/ha/test_cloud_recovery.py b/tobiko/tests/faults/ha/test_cloud_recovery.py index 40730fe57..b43b6047a 100644 --- a/tobiko/tests/faults/ha/test_cloud_recovery.py +++ b/tobiko/tests/faults/ha/test_cloud_recovery.py @@ -24,6 +24,7 @@ def overcloud_health_checks(passive_checks_only=False, # create a uniq stack check_vm_create() nova.start_all_instances() + nova.check_computes_vms_running_via_virsh() containers.list_node_containers.cache_clear() containers.assert_all_tripleo_containers_running() containers.assert_equal_containers_state() diff --git a/tobiko/tripleo/nova.py b/tobiko/tripleo/nova.py index 5ed41c6f6..5eb7628aa 100644 --- a/tobiko/tripleo/nova.py +++ b/tobiko/tripleo/nova.py @@ -7,8 +7,12 @@ from oslo_log import log import pandas import tobiko +from tobiko.tripleo import overcloud from tobiko.shell import ping +from tobiko.shell import sh from tobiko.openstack import nova +from tobiko.openstack import topology +from tobiko.tripleo import containers LOG = log.getLogger(__name__) @@ -171,3 +175,46 @@ def check_vm_evacuations(vms_df_old=None, compute_host=None, timeout=600, def get_stack_server_id(stack): return stack.server_details.id + + +def get_fqdn_from_topology_node(topology_node): + return sh.execute("hostname -f", ssh_client=topology_node.ssh_client, + expect_exit_status=None).stdout.strip() + + +def check_vm_running_via_virsh(topology_compute, vm_id): + """check that a vm is in running state via virsh command, + return false if not""" + if vm_id in get_vm_uuid_list_running_via_virsh(topology_compute): + return True + else: + return False + + +def get_vm_uuid_list_running_via_virsh(topology_compute): + if overcloud.has_overcloud(): + container_runtime = containers.get_container_runtime_name() + command = f"sudo {container_runtime} exec nova_libvirt " \ + f"sh -c 'for i in `virsh list --name --state-running` " \ + f";do virsh domuuid $i;done'" + else: + command = "for i in `sudo virsh list --name --state-running` " \ + ";do virsh domuuid $i;done'" + return sh.execute(command, + ssh_client=topology_compute.ssh_client).stdout.split() + + +def check_computes_vms_running_via_virsh(): + """check all vms are running via virsh list command""" + for compute in topology.list_openstack_nodes(group='compute'): + hostname = get_fqdn_from_topology_node(compute) + retry = tobiko.retry(timeout=120, interval=5) + for vm_id in get_compute_vms_df(hostname)['vm_id'].to_list(): + for _ in retry: + if check_vm_running_via_virsh(compute, vm_id): + LOG.info(f"{vm_id} is running ok on " + f"{compute.hostname}") + break + else: + LOG.info(f"{vm_id} is not in running state on " + f"{compute.hostname}")