diff --git a/doc/system_tests.rst b/doc/system_tests.rst index 94fb90eee..ea7e93bf6 100644 --- a/doc/system_tests.rst +++ b/doc/system_tests.rst @@ -141,6 +141,11 @@ vCenter/DVS .. automodule:: system_test.tests.vcenter.test_vcenter_dvs :members: +vCenter/DVS failover +-------------------- +.. automodule:: system_test.tests.vcenter.test_vcenter_failover + :members: + Helpers ======= diff --git a/fuelweb_test/helpers/os_actions.py b/fuelweb_test/helpers/os_actions.py index 4cba72432..ebe446a42 100644 --- a/fuelweb_test/helpers/os_actions.py +++ b/fuelweb_test/helpers/os_actions.py @@ -95,7 +95,7 @@ class OpenStackActions(common.Common): :param name: server name, if None -> test-serv + random suffix :param security_groups: list, if None -> ssh + icmp v4 & icmp v6 :param flavor_id: micro_flavor if None - :param net_id: network id, could be omitted. + :param net_id: network id, could be omitted :param timeout: int=100 :param image: TestVM if None. :return: Server, in started state diff --git a/system_test/actions/base.py b/system_test/actions/base.py index 44745971e..804fc8a24 100644 --- a/system_test/actions/base.py +++ b/system_test/actions/base.py @@ -260,6 +260,7 @@ class BaseActions(PrepareActions, HealthCheckActions, PluginsActions, actions_order = None cluster_id = None scale_step = 0 + power_step = 0 def _add_node(self, nodes_list): """Add nodes to Environment""" @@ -463,7 +464,7 @@ class BaseActions(PrepareActions, HealthCheckActions, PluginsActions, action: add For remove nodes with role use scale_nodes in yaml with action delete - in step::: + in step:: scale_nodes: - - roles: @@ -498,9 +499,78 @@ class BaseActions(PrepareActions, HealthCheckActions, PluginsActions, if 'compute-vmware' in node['roles']: self.del_vmware_nova_compute() else: - logger.error("Unknow scale action: {}".format(node['action'])) + logger.error("Unknown scale action: {}".format(node['action'])) self.scale_step += 1 + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def manage_nodes_power(self): + """Manage power of node + + To power off node with role use manage_nodes_power in yaml with action + power_off in step:: + + manage_nodes_power: + - - roles: + - controller + node_number: 0 + action: power_off + + To power on node with role use manage_nodes_power in yaml with action + power_on in step:: + + manage_nodes_power: + - - roles: + - controller + node_number: 0 + action: power_on + + To restart node with role use manage_nodes_power in yaml with action + warm_restart or cold_restart in step:: + + manage_nodes_power: + - - roles: + - controller + node_number: 0 + action: warm_restart + + Example of cold restarting two different nodes with the same role:: + + manage_nodes_power: + - - roles: + - controller + node_number: 0 + action: cold_restart + - - roles: + - controller + node_number: 1 + action: cold_restart + + """ + power_actions = { + 'power_off_warm': self.fuel_web.warm_shutdown_nodes, + 'power_on_warm': self.fuel_web.warm_start_nodes, + 'reboot_warm': self.fuel_web.warm_restart_nodes, + 'reboot_cold': self.fuel_web.cold_restart_nodes + } + + step_config = self.env_config['manage_nodes_power'][self.power_step] + for node in step_config: + power_action = power_actions.get(node['action'], None) + node_number = node['node_number'] + if power_action: + ng_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( + cluster_id=self.cluster_id, roles=[node['roles'][0]]) + + dev_node = self.fuel_web.get_devops_node_by_nailgun_fqdn( + ng_nodes[node_number]['fqdn']) + + power_action([dev_node]) + else: + logger.error("Unknown power switch action: " + "{}".format(node['action'])) + self.power_step += 1 + def add_vmware_nova_compute(self, nova_computes): vmware_attr = \ self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id) diff --git a/system_test/actions/vcenter_actions.py b/system_test/actions/vcenter_actions.py index 10076067c..98bb61efe 100644 --- a/system_test/actions/vcenter_actions.py +++ b/system_test/actions/vcenter_actions.py @@ -12,10 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -from time import sleep +import itertools from random import randrange +from time import sleep from devops.helpers import helpers + from proboscis import SkipTest from proboscis.asserts import assert_equal from proboscis.asserts import assert_not_equal @@ -24,17 +26,24 @@ from proboscis.asserts import assert_true from fuelweb_test.helpers.os_actions import OpenStackActions from fuelweb_test.helpers.ssh_manager import SSHManager from fuelweb_test.settings import NEUTRON -from system_test import logger -from system_test import deferred_decorator +from fuelweb_test.settings import SERVTEST_PASSWORD +from fuelweb_test.settings import SERVTEST_TENANT +from fuelweb_test.settings import SERVTEST_USERNAME from system_test import action +from system_test import deferred_decorator +from system_test import logger from system_test.helpers.decorators import make_snapshot_if_step_fail # pylint: disable=no-member class VMwareActions(object): - """VMware vCenter/DVS related actions""" + """VMware vCenter/DVS related actions.""" plugin_version = None + vms_to_ping = [] # instances which should ping each other + vip_contr = None # controller with VIP resources + primary_ctlr_ng = None # nailgun primary controller + os_conn = None vcenter_az = 'vcenter' cinder_az = 'vcenter-cinder' vmware_image = 'TestVM-VMDK' @@ -47,8 +56,7 @@ class VMwareActions(object): @deferred_decorator([make_snapshot_if_step_fail]) @action def configure_dvs_plugin(self): - """Configure DVS plugin""" - + """Configure DVS plugin.""" msg = "Plugin couldn't be enabled. Check plugin version. Test aborted" assert_true( self.fuel_web.check_plugin_exists( @@ -77,8 +85,7 @@ class VMwareActions(object): @deferred_decorator([make_snapshot_if_step_fail]) @action def configure_vcenter(self): - """Configure vCenter settings""" - + """Configure vCenter settings.""" vmware_vcenter = self.env_settings['vmware_vcenter'] vcenter_value = { @@ -160,7 +167,7 @@ class VMwareActions(object): @deferred_decorator([make_snapshot_if_step_fail]) @action def set_custom_node_names(self): - """Set custom node names""" + """Set custom node names.""" custom_hostnames = [] for node in self.fuel_web.client.list_cluster_nodes(self.cluster_id): custom_hostname = "{0}-{1}".format( @@ -170,7 +177,8 @@ class VMwareActions(object): @staticmethod def get_nova_conf_dict(az, nova): - """ + """Return nova conf_dict. + :param az: vcenter az (api), dict :param nova: nova (api), dict :return: dict @@ -188,8 +196,7 @@ class VMwareActions(object): @deferred_decorator([make_snapshot_if_step_fail]) @action def check_nova_conf(self): - """Verify nova-compute vmware configuration""" - + """Verify nova-compute vmware configuration.""" nodes = self.fuel_web.client.list_cluster_nodes(self.cluster_id) vmware_attr = self.fuel_web.client.get_cluster_vmware_attributes( self.cluster_id) @@ -231,8 +238,7 @@ class VMwareActions(object): @deferred_decorator([make_snapshot_if_step_fail]) @action def check_nova_srv(self): - """Verify nova-compute service for each vSphere cluster""" - + """Verify nova-compute service for each vSphere cluster.""" vmware_attr = self.fuel_web.client.get_cluster_vmware_attributes( self.cluster_id) az = vmware_attr['editable']['value']['availability_zones'][0] @@ -251,8 +257,7 @@ class VMwareActions(object): @deferred_decorator([make_snapshot_if_step_fail]) @action def check_cinder_vmware_srv(self): - """Verify cinder-vmware service""" - + """Verify cinder-vmware service.""" ctrl_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( self.cluster_id, ["controller"]) cmd = '. openrc; cinder-manage service list | grep vcenter | ' \ @@ -263,7 +268,7 @@ class VMwareActions(object): @deferred_decorator([make_snapshot_if_step_fail]) @action def deploy_changes(self): - """Deploy environment""" + """Deploy environment.""" if self.cluster_id is None: raise SkipTest() @@ -272,8 +277,7 @@ class VMwareActions(object): @action def create_and_attach_empty_volume(self): - """Create and attach to instance empty volume""" - + """Create and attach to instance empty volume.""" mount_point = '/dev/sdb' public_ip = self.fuel_web.get_public_vip(self.cluster_id) @@ -324,8 +328,7 @@ class VMwareActions(object): @action def create_bootable_volume_and_run_instance(self): - """Create bootable volume and launch instance from it""" - + """Create bootable volume and launch instance from it.""" public_ip = self.fuel_web.get_public_vip(self.cluster_id) os_conn = OpenStackActions(public_ip) @@ -350,9 +353,8 @@ class VMwareActions(object): @action def check_vmware_service_actions(self): - """Disable vmware host (cluster) and check instance creation - on enabled cluster""" - + """Disable vmware host (cluster) and check instance creation on + enabled cluster.""" public_ip = self.fuel_web.get_public_vip(self.cluster_id) os_conn = OpenStackActions(public_ip) @@ -371,7 +373,7 @@ class VMwareActions(object): for service in vmware_services: logger.info("Check {}".format(service.host)) os_conn.enable_nova_service(service) - vm = os_conn.create_server(image=image, + vm = os_conn.create_server(image=image, timeout=180, availability_zone=self.vcenter_az, net_id=net['id'], security_groups=[sg]) vm_host = getattr(vm, 'OS-EXT-SRV-ATTR:host') @@ -383,8 +385,7 @@ class VMwareActions(object): @action def upload_image(self): - """Upload vmdk image""" - + """Upload vmdk image.""" controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles( self.cluster_id, ["controller"])[0] @@ -407,8 +408,7 @@ class VMwareActions(object): @action def check_instance_creation(self): - """Create instance and check connection""" - + """Create instance and check connection.""" public_ip = self.fuel_web.get_public_vip(self.cluster_id) os_conn = OpenStackActions(public_ip) @@ -433,8 +433,7 @@ class VMwareActions(object): @action def create_instance_with_vmxnet3_adapter(self): - """Create instance with vmxnet3 adapter""" - + """Create instance with vmxnet3 adapter.""" public_ip = self.fuel_web.get_public_vip(self.cluster_id) os_conn = OpenStackActions(public_ip) @@ -467,8 +466,7 @@ class VMwareActions(object): @action def check_batch_instance_creation(self): - """Create several instance simultaneously""" - + """Create several instance simultaneously.""" count = 10 vm_name = 'vcenter_vm' @@ -500,8 +498,7 @@ class VMwareActions(object): @action def create_instance_with_different_disktype(self): - """Create instances with different disk type""" - + """Create instances with different disk type.""" public_ip = self.fuel_web.get_public_vip(self.cluster_id) os_conn = OpenStackActions(public_ip) @@ -535,8 +532,7 @@ class VMwareActions(object): @deferred_decorator([make_snapshot_if_step_fail]) @action def check_neutron_public(self): - """Check that public network was assigned to all nodes""" - + """Check that public network was assigned to all nodes.""" cluster = self.fuel_web.client.get_cluster(self.cluster_id) assert_equal(str(cluster['net_provider']), NEUTRON) os_conn = OpenStackActions( @@ -547,8 +543,7 @@ class VMwareActions(object): @deferred_decorator([make_snapshot_if_step_fail]) @action def check_gw_on_vmware_nodes(self): - """Check that default gw != fuel node ip""" - + """Check that default gw != fuel node ip.""" vmware_nodes = [] vmware_nodes.extend(self.fuel_web.get_nailgun_cluster_nodes_by_roles( self.cluster_id, ["compute-vmware"])) @@ -561,3 +556,285 @@ class VMwareActions(object): logger.debug('Default gw for node {0} is {1}'.format( node['name'], gw_ip['stdout_str'])) assert_not_equal(gw_ip['stdout_str'], self.fuel_web.admin_node_ip) + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def config_idatastore(self): + """Reconfigure vCenter settings with incorrect regex of Datastore.""" + vmware_attr = \ + self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id) + vcenter_data = vmware_attr['editable'] + vcenter_data['value']['availability_zones'][0]['nova_computes'][0][ + 'datastore_regex'] = '!@#$%^&*()' + + self.fuel_web.client.update_cluster_vmware_attributes(self.cluster_id, + vmware_attr) + logger.info("Datastore regex settings have been updated") + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def config_idc_glance(self): + """Reconfigure vCenter settings with incorrect Glance Datacenter.""" + vmware_attr = \ + self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id) + vcenter_data = vmware_attr['editable'] + vcenter_data['value']['glance']['datacenter'] = '!@#$%^&*()' + + self.fuel_web.client.update_cluster_vmware_attributes(self.cluster_id, + vmware_attr) + logger.info("Glance datacenter settings have been updated") + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def config_ids_glance(self): + """Reconfigure vCenter settings with incorrect Glance Datastore.""" + vmware_attr = \ + self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id) + vcenter_data = vmware_attr['editable'] + vcenter_data['value']['glance']['datastore'] = '!@#$%^&*()' + + self.fuel_web.client.update_cluster_vmware_attributes(self.cluster_id, + vmware_attr) + + logger.info("Glance datastore settings have been updated") + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def delete_instances(self): + """Delete created instances.""" + for srv in self.vms_to_ping: + logger.info('Started: delete existing VM "{}"'.format(srv.name)) + self.os_conn.nova.servers.delete(srv) + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def create_instances(self): + """Create instances with nova az and vcenter az.""" + os_ip = self.fuel_web.get_public_vip(self.cluster_id) + self.os_conn = OpenStackActions( + os_ip, SERVTEST_USERNAME, + SERVTEST_PASSWORD, + SERVTEST_TENANT + ) + vcenter_az = self.env_settings['vmware_vcenter']['settings']['az'] + net = self.os_conn.get_network(self.net_name) + sec_group = self.os_conn.create_sec_group_for_ssh() + + inst_count = 1 # amount of VMs to create on each az + vc_inst_name_prefix = 'vcenter-test' + vc_image = self.os_conn.get_image('TestVM-VMDK') + nova_inst_name_prefix = 'nova-test' + nova_image = self.os_conn.get_image('TestVM') + + logger.info( + 'Started: create {num} VM(s) with net="{net}", az="{az}", ' + 'image="{image}"'.format(num=inst_count, net=net['name'], + az=vcenter_az, image='TestVM-VMDK') + ) + self.os_conn.create_server( + name=vc_inst_name_prefix, + net_id=net['id'], + availability_zone=vcenter_az, + image=vc_image, + timeout=200, + security_groups=[sec_group], + min_count=inst_count + ) + + logger.info( + 'Started: create {num} VM(s) with net="{net}", az="{az}", ' + 'image="{image}"'.format(num=inst_count, net=net['name'], + az='nova', image='TestVM') + ) + self.os_conn.create_server( + name=nova_inst_name_prefix, + net_id=net['id'], + image=nova_image, + security_groups=[sec_group], + availability_zone='nova', + min_count=inst_count + ) + + servers = self.os_conn.nova.servers.list() + self.vms_to_ping = [srv for srv in servers if + srv.name.startswith(vc_inst_name_prefix) or + srv.name.startswith(nova_inst_name_prefix)] + + def _get_controller_with_vip(self): + """Return name of controller with VIPs.""" + for node in self.env.d_env.nodes().slaves: + ng_node = self.env.fuel_web.get_nailgun_node_by_devops_node(node) + if ng_node['online']: + hosts_vip = self.fuel_web.get_pacemaker_resource_location( + ng_node['devops_name'], 'vip__management') + logger.info('Now primary controller is ' + '{}'.format(hosts_vip[0].name)) + return hosts_vip[0].name + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def hard_reset_primary(self): + """Hard reboot of primary controller.""" + self.vip_contr = self._get_controller_with_vip() + + self.primary_ctlr_ng = self.fuel_web.get_nailgun_primary_node( + self.env.d_env.nodes().slaves[0]) + + self.fuel_web.cold_restart_nodes([self.primary_ctlr_ng]) + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def shutdown_primary(self): + """Shut down primary controller.""" + self.vip_contr = self._get_controller_with_vip() + + self.primary_ctlr_ng = self.fuel_web.get_nailgun_primary_node( + self.env.d_env.nodes().slaves[0]) + + self.primary_ctlr_ng.destroy() + + timeout = 60 * 10 + logger.info('Wait offline status for ' + '{ctrlr}'.format(ctrlr=self.primary_ctlr_ng.name)) + + helpers.wait(lambda: self.env.fuel_web.get_nailgun_node_by_devops_node( + self.primary_ctlr_ng)['online'] is not True, + timeout=timeout, + timeout_msg="Primary controller is still online") + logger.info('Primary controller is offline') + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def safe_reboot_primary(self): + """Safe reboot primary controller.""" + self.vip_contr = self._get_controller_with_vip() + + self.primary_ctlr_ng = self.fuel_web.get_nailgun_primary_node( + self.env.d_env.nodes().slaves[0]) + + self.fuel_web.warm_restart_nodes([self.primary_ctlr_ng]) + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def check_up_vips(self): + """Ensure that VIPs are moved to another controller.""" + vip_contr = self._get_controller_with_vip() + + assert_true(vip_contr and vip_contr != self.vip_contr, + 'VIPs have not been moved to another controller') + logger.info('VIPs have been moved to another controller') + + # @deferred_decorator([make_snapshot_if_step_fail]) + @action + def turn_on_primary(self): + """Turn on primary controller.""" + self.primary_ctlr_ng.start() + logger.info('Started: turn on primary controller ' + '{name}'.format(name=self.primary_ctlr_ng.name)) + + timeout = 60 * 10 + logger.info('Wait online status for ' + '{name}'.format(name=self.primary_ctlr_ng.name)) + + helpers.wait(lambda: self.env.fuel_web.get_nailgun_node_by_devops_node( + self.primary_ctlr_ng)['online'], timeout=timeout, + timeout_msg="Primary controller is still offline") + logger.info('Primary controller is online') + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def vcenter_ostf(self): + """Run vCenter OSTF tests.""" + self.fuel_web.run_ostf( + cluster_id=self.cluster_id, + test_sets=['smoke'], + should_fail=getattr(self, 'ostf_tests_should_failed', 0), + failed_test_name=getattr(self, 'failed_test_name', None)) + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def ostf_with_services_fail(self): + """Run OSTF tests (one should fail).""" + self.env.fuel_web.run_ostf( + self.cluster_id, should_fail=1, + failed_test_name='Check that required services are running') + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def fail_ostf(self): + """Run OSTF tests (must fail).""" + try: + self.env.fuel_web.run_ostf( + self.cluster_id, + test_sets=['sanity', 'smoke', 'ha']) + failed = False + except AssertionError: + failed = True + assert_true(failed, 'OSTF passed with incorrect parameters') + logger.info('OSTF failed') + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def fail_deploy_cluster(self): + """Deploy environment (must fail).""" + try: + self.fuel_web.deploy_cluster_wait(self.cluster_id) + failed = False + except AssertionError: + failed = True + assert_true(failed, 'Deploy passed with incorrect parameters') + logger.info('Deploy failed') + + def ping_from_instance(self, src_floating_ip, dst_ip, primary, + size=56, count=1): + """Verify ping between instances. + + :param src_floating_ip: floating ip address of instance + :param dst_ip: destination ip address + :param primary: name of the primary controller + :param size: number of data bytes to be sent + :param count: number of packets to be sent + """ + creds = ("cirros", "cubswin:)") + + with self.fuel_web.get_ssh_for_node(primary) as ssh: + command = "ping -s {0} -c {1} {2}".format(size, count, + dst_ip) + ping = self.os_conn.execute_through_host(ssh, src_floating_ip, + command, creds) + logger.info("Ping result is {}".format(ping['exit_code'])) + return 0 == ping['exit_code'] + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def check_vm_connect(self): + """Ensure connectivity between VMs.""" + if self.vip_contr: + primary_ctrl_name = self._get_controller_with_vip() + else: + primary_ctrl_name = self.fuel_web.get_nailgun_primary_node( + self.env.d_env.nodes().slaves[0]).name + + private_ips = {} + floating_ips = {} + + for srv in self.vms_to_ping: + floating = self.os_conn.assign_floating_ip(srv) + floating_ips[srv] = floating.ip + logger.info("Floating address {0} was associated with instance " + "{1}".format(floating_ips[srv], srv.name)) + + private_ips[srv] = self.os_conn.get_nova_instance_ip( + srv, net_name=self.net_name) + + for vm in itertools.combinations(self.vms_to_ping, 2): + logger.info('Try to ping from {src} ({src_vm}) to {dst} ' + '({dst_vm})'.format(src=floating_ips[vm[0]], + dst=private_ips[vm[1]], + src_vm=vm[0].name, + dst_vm=vm[1].name)) + + assert_true(self.ping_from_instance(floating_ips[vm[0]], + private_ips[vm[1]], + primary_ctrl_name), + 'Ping between VMs failed') diff --git a/system_test/tests/vcenter/test_vcenter_dvs.py b/system_test/tests/vcenter/test_vcenter_dvs.py index 20a0ea2ff..57e7462ae 100644 --- a/system_test/tests/vcenter/test_vcenter_dvs.py +++ b/system_test/tests/vcenter/test_vcenter_dvs.py @@ -14,22 +14,22 @@ from fuelweb_test.settings import DVS_PLUGIN_PATH from fuelweb_test.settings import DVS_PLUGIN_VERSION -from fuelweb_test.settings import VMWARE_IMG_NAME -from fuelweb_test.settings import VMWARE_IMG_URL from fuelweb_test.settings import VMWARE_IMG_LOGIN +from fuelweb_test.settings import VMWARE_IMG_NAME from fuelweb_test.settings import VMWARE_IMG_PASSWORD +from fuelweb_test.settings import VMWARE_IMG_URL from system_test import testcase -from system_test.tests import ActionTest from system_test.actions import BaseActions from system_test.actions import VMwareActions +from system_test.tests import ActionTest @testcase(groups=['system_test', 'system_test.vcenter', 'system_test.vcenter.deploy_vcenter_dvs_run_ostf']) class DeployWithVMware(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and dvs plugin + """Deploy cluster with vCenter and dvs plugin. Scenario: 1. Upload plugin to the master node @@ -64,7 +64,7 @@ class DeployWithVMware(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.scale_vcenter_dvs']) class ScaleWithVMware(ActionTest, BaseActions, VMwareActions): - """Deploy and scale cluster with vCenter and dvs plugin + """Deploy and scale cluster with vCenter and dvs plugin. Scenario: 1. Upload plugin to the master node @@ -143,7 +143,7 @@ class DeployWithVMwareRunPlatformOSTF(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.deploy_with_custom_hostname']) class DeployWithCustomHostname(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and custom hostname + """Deploy cluster with vCenter and custom hostname. Scenario: 1. Upload plugin to the master node @@ -180,7 +180,7 @@ class DeployWithCustomHostname(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.check_nova_config']) class CheckNovaConfig(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and custom hostname + """Deploy cluster with vCenter and custom hostname. Scenario: 1. Upload plugin to the master node @@ -217,7 +217,7 @@ class CheckNovaConfig(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.check_nova_srv']) class CheckNovaSrv(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and custom hostname + """Deploy cluster with vCenter and custom hostname. Scenario: 1. Upload plugin to the master node @@ -254,7 +254,7 @@ class CheckNovaSrv(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.check_cinder_srv']) class CheckCinderVmwareSrv(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and custom hostname + """Deploy cluster with vCenter and custom hostname. Scenario: 1. Upload plugin to the master node @@ -291,7 +291,7 @@ class CheckCinderVmwareSrv(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.attach_empty_volume']) class AttachEmptyVol(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and dvs plugin + """Deploy cluster with vCenter and dvs plugin. Scenario: 1. Upload plugin to the master node @@ -326,7 +326,7 @@ class AttachEmptyVol(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.bootable_vol']) class BootableVol(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and dvs plugin + """Deploy cluster with vCenter and dvs plugin. Scenario: 1. Upload plugin to the master node @@ -361,7 +361,7 @@ class BootableVol(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.disable_enable_compute_service']) class DisableEnableVMwareServices(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and dvs plugin + """Deploy cluster with vCenter and dvs plugin. Scenario: 1. Upload plugin to the master node @@ -396,7 +396,7 @@ class DisableEnableVMwareServices(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.upload_image']) class UploadImage(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and dvs plugin + """Deploy cluster with vCenter and dvs plugin. Scenario: 1. Upload plugin to the master node @@ -436,7 +436,7 @@ class UploadImage(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.vmxnet3']) class Vmxnet3(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and dvs plugin + """Deploy cluster with vCenter and dvs plugin. Scenario: 1. Upload plugin to the master node @@ -476,7 +476,7 @@ class Vmxnet3(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.create_batch_of_instances']) class CreateBatchInstances(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and dvs plugin + """Deploy cluster with vCenter and dvs plugin. Scenario: 1. Upload plugin to the master node @@ -511,7 +511,7 @@ class CreateBatchInstances(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.diff_disk_types']) class DiffDiskTypes(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and dvs plugin + """Deploy cluster with vCenter and dvs plugin. Scenario: 1. Upload plugin to the master node @@ -546,7 +546,7 @@ class DiffDiskTypes(ActionTest, BaseActions, VMwareActions): 'system_test.vcenter', 'system_test.vcenter.neutron_public_net']) class DeployNeutronPublicNet(ActionTest, BaseActions, VMwareActions): - """Deploy cluster with vCenter and dvs plugin + """Deploy cluster with vCenter and dvs plugin. Scenario: 1. Upload plugin to the master node diff --git a/system_test/tests/vcenter/test_vcenter_failover.py b/system_test/tests/vcenter/test_vcenter_failover.py new file mode 100644 index 000000000..4259f65a7 --- /dev/null +++ b/system_test/tests/vcenter/test_vcenter_failover.py @@ -0,0 +1,365 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE_2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuelweb_test.settings import DVS_PLUGIN_PATH +from fuelweb_test.settings import DVS_PLUGIN_VERSION + +from system_test import testcase +from system_test.actions import BaseActions +from system_test.actions import VMwareActions +from system_test.tests import ActionTest + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_reset_ctrl']) +class HardResetPrimaryWithVMware(ActionTest, BaseActions, VMwareActions): + """Hard reset primary controller and check vCenter functionality. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings (depends on yaml config) + 7. Deploy the cluster + 8. Create instances on Nova and vCenter + 9. Hard reset primary controller + 10. Wait 5-10 minutes + 11. Verify networks + 12. Ensure that VIPs are moved to other controller + 13. Ensure connectivity between VMs + 14. Run OSTF tests + + Duration 3h 00min + Snapshot vcenter_reset_ctrl + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter', + 'deploy_cluster', + 'create_instances', + 'hard_reset_primary', + 'network_check', + 'check_up_vips', + 'check_vm_connect', + 'delete_instances', + 'health_check_sanity_smoke_ha' + ] + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_shutdown_ctrl']) +class ShutdownPrimaryWithVMware(ActionTest, BaseActions, VMwareActions): + """Shutdown primary controller and check vCenter functionality. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings (depends on yaml config) + 7. Deploy the cluster + 8. Create instances on Nova and vCenter + 9. Shutdown primary controller + 10. Verify networks + 11. Ensure that VIPs are moved to other controller + 12. Ensure connectivity between VMs + 13. Run OSTF tests (one should fail) + 14. Turn on primary controller + 15. Wait 5-10 minutes + 16. Verify networks + 17. Run OSTF tests + + Duration 3h 00min + Snapshot vcenter_shutdown_ctrl + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter', + 'deploy_cluster', + 'create_instances', + 'shutdown_primary', + 'network_check', + 'check_up_vips', + 'check_vm_connect', + 'delete_instances', + 'ostf_with_services_fail', + 'turn_on_primary', + 'network_check', + 'health_check_sanity_smoke_ha' + ] + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_reboot_ctrl']) +class SafeRebootPrimaryWithVMware(ActionTest, BaseActions, VMwareActions): + """Safe reboot primary controller and check vCenter functionality. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings (depends on yaml config) + 7. Deploy the cluster + 8. Create instances on Nova and vCenter + 9. Safe reboot primary controller + 10. Wait 5-10 minutes + 11. Verify networks + 12. Ensure that VIPs are moved to other controller + 13. Ensure connectivity between VMs + 14. Run OSTF tests + + Duration 3h 00min + Snapshot vcenter_reboot_ctrl + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter', + 'deploy_cluster', + 'create_instances', + 'safe_reboot_primary', + 'network_check', + 'check_up_vips', + 'check_vm_connect', + 'delete_instances', + 'health_check_sanity_smoke_ha' + ] + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_shutdown_cindervmware']) +class ShutdownCinderNodeWithVMware(ActionTest, BaseActions, VMwareActions): + """Shutdown one of CinderVMDK node. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings (depends on yaml config) + 7. Deploy the cluster + 8. Create instances on KVM and vCenter + 9. Run all OSTF tests + 10. Shutdown one of CinderVMDK node + 11. Run vCenter OSTF tests + 12. Power on CinderVMDK node and wait for it to load + 13. Run vCenter OSTF tests + 14. Shutdown another CinderVMDK node + 15. Run vCenter OSTF tests + 16. Power on CinderVMDK node and wait for it to load + 17. Run all OSTF tests + + Duration 3h 00min + Snapshot vcenter_shutdown_cindervmware + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter', + 'deploy_cluster', + 'create_instances', + 'health_check_sanity_smoke_ha', + 'manage_nodes_power', + 'vcenter_ostf', + 'manage_nodes_power', + 'vcenter_ostf', + 'manage_nodes_power', + 'vcenter_ostf', + 'manage_nodes_power', + 'delete_instances', + 'health_check_sanity_smoke_ha' + ] + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_reboot_cindervmware']) +class RebootCinderNodeWithVMware(ActionTest, BaseActions, VMwareActions): + """Restart CinderVMware node. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings (depends on yaml config) + 7. Deploy the cluster + 8. Reboot CinderVMware node. + 9. Check CinderVMware services. + + Duration 3h 00min + Snapshot vcenter_reboot_cindervmware + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter', + 'deploy_cluster', + 'manage_nodes_power', + 'check_cinder_vmware_srv', + 'health_check' + ] + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_iname_glance_ds']) +class DeployINameDSWithVMware(ActionTest, BaseActions, VMwareActions): + """Deploy with controller and incorrect name of vCenter Glance Datastore. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings (depends on yaml config) + 7. Deploy the cluster (Deploy should fail) + + Duration 3h 00min + Snapshot vcenter_iname_glance_ds + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter', + 'config_ids_glance', + 'fail_deploy_cluster' + ] + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_iname_glance_dc']) +class DeployINameDCWithVMware(ActionTest, BaseActions, VMwareActions): + """Deploy with controller and incorrect name of vCenter Glance Datacenter. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings (depends on yaml config) + 7. Deploy the cluster (Deploy should fail) + + Duration 3h 00min + Snapshot vcenter_iname_glance_dc + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter', + 'config_idc_glance', + 'fail_deploy_cluster' + ] + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_idatastore']) +class DeployIDSWithVMware(ActionTest, BaseActions, VMwareActions): + """Deploy with controller and not correct regex of vCenter Datastore. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings (depends on yaml config) + 7. Redefine vmware settings with incorrect ds + 8. Deploy the cluster + 9. Run OSTF tests (should fail) + + Duration 3h 00min + Snapshot vcenter_idatastore + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter', + 'config_idatastore', + 'deploy_cluster', + 'fail_ostf' + ] diff --git a/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_dvs_failover.yaml b/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_dvs_failover.yaml new file mode 100644 index 000000000..271ab9e88 --- /dev/null +++ b/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_dvs_failover.yaml @@ -0,0 +1,37 @@ +--- +template: + name: 5 nodes on Neutron/VLAN with DVS plugin (3 controllers, 1 compute + cinder, 1 cinder-vmware) + slaves: 5 + cluster_template: + name: vcenter_dvs_failover + release: ubuntu + network: + !include cluster_configs/networks/neutron_vlan.yaml + settings: + components: + !include cluster_configs/settings/components/wo_components.yaml + storages: + !include cluster_configs/settings/storages/cinder_only.yaml + vmware_vcenter: + settings: + !include cluster_configs/settings/vmware/vcenter_main.yaml + nova-compute: + !include cluster_configs/settings/vmware/nova_compute/2clusters_ctrl.yaml + glance: + !include cluster_configs/settings/vmware/vcenter_glance.yaml + vmware_dvs: + !include cluster_configs/settings/vmware/dvs/dvs_main.yaml + nodes: + - roles: + - controller + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 3 + - roles: + - compute + - cinder + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 1 + - roles: + - cinder-vmware + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 1 \ No newline at end of file diff --git a/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_reboot_cvm.yaml b/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_reboot_cvm.yaml new file mode 100644 index 000000000..905081afb --- /dev/null +++ b/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_reboot_cvm.yaml @@ -0,0 +1,39 @@ +--- +template: + name: 1 controller, 1 cinder-vmware on Neutron/VLAN with DVS plugin + slaves: 2 + cluster_template: + name: vcenter_reboot_cindervmware + release: ubuntu + network: + !include cluster_configs/networks/neutron_vlan.yaml + settings: + components: + !include cluster_configs/settings/components/wo_components.yaml + storages: + !include cluster_configs/settings/storages/cinder_only.yaml + vmware_vcenter: + settings: + !include cluster_configs/settings/vmware/vcenter_main.yaml + nova-compute: + !include cluster_configs/settings/vmware/nova_compute/1cluster_ctrl.yaml + glance: + enable: false + vmware_dvs: + !include cluster_configs/settings/vmware/dvs/dvs_main.yaml + nodes: + - roles: + - controller + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 1 + - roles: + - cinder-vmware + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 1 + + + manage_nodes_power: + - - roles: + - cinder-vmware + node_number: 0 + action: reboot_warm \ No newline at end of file diff --git a/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_shutdown_cvm.yaml b/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_shutdown_cvm.yaml new file mode 100644 index 000000000..ec61fc01f --- /dev/null +++ b/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_shutdown_cvm.yaml @@ -0,0 +1,56 @@ +--- +template: + name: 1 controller, 1 compute + cinder, 2 cinder-vmware on Neutron/VLAN with DVS plugin + slaves: 4 + cluster_template: + name: vcenter_shutdown_cindervmware + release: ubuntu + network: + !include cluster_configs/networks/neutron_vlan.yaml + settings: + components: + !include cluster_configs/settings/components/wo_components.yaml + storages: + !include cluster_configs/settings/storages/cinder_only.yaml + vmware_vcenter: + settings: + !include cluster_configs/settings/vmware/vcenter_main.yaml + nova-compute: + !include cluster_configs/settings/vmware/nova_compute/1cluster_ctrl.yaml + glance: + enable: false + vmware_dvs: + !include cluster_configs/settings/vmware/dvs/dvs_main.yaml + nodes: + - roles: + - controller + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 1 + - roles: + - compute + - cinder + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 1 + - roles: + - cinder-vmware + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 2 + + + manage_nodes_power: + - - roles: + - cinder-vmware + node_number: 0 + action: power_off_warm + - - roles: + - cinder-vmware + node_number: 0 + action: power_on_warm + - - roles: + - cinder-vmware + node_number: 1 + action: power_off_warm + - - roles: + - cinder-vmware + node_number: 1 + action: power_on_warm \ No newline at end of file