Added failover tests for vCenter

Change-Id: I6b0a3691223fbd89fe46667243b48b1398e936c2
Closes-bug: #1575742

doc/system_tests.rst

@@ -141,6 +141,11 @@ vCenter/DVS
 .. automodule:: system_test.tests.vcenter.test_vcenter_dvs
     :members:
 
+vCenter/DVS failover
+--------------------
+.. automodule:: system_test.tests.vcenter.test_vcenter_failover
+    :members:
+
 Helpers
 =======
 

fuelweb_test/helpers/os_actions.py

@@ -95,7 +95,7 @@ class OpenStackActions(common.Common):
         :param name: server name, if None -> test-serv + random suffix
         :param security_groups: list, if None -> ssh + icmp v4 & icmp v6
         :param flavor_id: micro_flavor if None
-        :param net_id: network id, could be omitted.
+        :param net_id: network id, could be omitted
         :param timeout: int=100
         :param image: TestVM if None.
         :return: Server, in started state

system_test/actions/base.py

@@ -260,6 +260,7 @@ class BaseActions(PrepareActions, HealthCheckActions, PluginsActions,
     actions_order = None
     cluster_id = None
     scale_step = 0
+    power_step = 0
 
     def _add_node(self, nodes_list):
         """Add nodes to Environment"""
@@ -463,7 +464,7 @@ class BaseActions(PrepareActions, HealthCheckActions, PluginsActions,
               action: add
 
         For remove nodes with role use scale_nodes in yaml with action delete
-        in step:::
+        in step::
 
           scale_nodes:
           - - roles:
@@ -498,9 +499,78 @@ class BaseActions(PrepareActions, HealthCheckActions, PluginsActions,
                 if 'compute-vmware' in node['roles']:
                     self.del_vmware_nova_compute()
             else:
-                logger.error("Unknow scale action: {}".format(node['action']))
+                logger.error("Unknown scale action: {}".format(node['action']))
         self.scale_step += 1
 
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def manage_nodes_power(self):
+        """Manage power of node
+
+        To power off node with role use manage_nodes_power in yaml with action
+        power_off in step::
+
+          manage_nodes_power:
+          - - roles:
+              - controller
+              node_number: 0
+              action: power_off
+
+        To power on node with role use manage_nodes_power in yaml with action
+        power_on in step::
+
+          manage_nodes_power:
+          - - roles:
+              - controller
+              node_number: 0
+              action: power_on
+
+        To restart node with role use manage_nodes_power in yaml with action
+        warm_restart or cold_restart in step::
+
+          manage_nodes_power:
+          - - roles:
+              - controller
+              node_number: 0
+              action: warm_restart
+
+        Example of cold restarting two different nodes with the same role::
+
+          manage_nodes_power:
+          - - roles:
+              - controller
+              node_number: 0
+              action: cold_restart
+          - - roles:
+              - controller
+              node_number: 1
+              action: cold_restart
+
+        """
+        power_actions = {
+            'power_off_warm': self.fuel_web.warm_shutdown_nodes,
+            'power_on_warm': self.fuel_web.warm_start_nodes,
+            'reboot_warm': self.fuel_web.warm_restart_nodes,
+            'reboot_cold': self.fuel_web.cold_restart_nodes
+        }
+
+        step_config = self.env_config['manage_nodes_power'][self.power_step]
+        for node in step_config:
+            power_action = power_actions.get(node['action'], None)
+            node_number = node['node_number']
+            if power_action:
+                ng_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
+                    cluster_id=self.cluster_id, roles=[node['roles'][0]])
+
+                dev_node = self.fuel_web.get_devops_node_by_nailgun_fqdn(
+                    ng_nodes[node_number]['fqdn'])
+
+                power_action([dev_node])
+            else:
+                logger.error("Unknown power switch action: "
+                             "{}".format(node['action']))
+        self.power_step += 1
+
     def add_vmware_nova_compute(self, nova_computes):
         vmware_attr = \
             self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id)

system_test/actions/vcenter_actions.py

@@ -12,10 +12,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from time import sleep
+import itertools
 from random import randrange
+from time import sleep
 
 from devops.helpers import helpers
+
 from proboscis import SkipTest
 from proboscis.asserts import assert_equal
 from proboscis.asserts import assert_not_equal
@@ -24,17 +26,24 @@ from proboscis.asserts import assert_true
 from fuelweb_test.helpers.os_actions import OpenStackActions
 from fuelweb_test.helpers.ssh_manager import SSHManager
 from fuelweb_test.settings import NEUTRON
-from system_test import logger
+from fuelweb_test.settings import SERVTEST_PASSWORD
-from system_test import deferred_decorator
+from fuelweb_test.settings import SERVTEST_TENANT
+from fuelweb_test.settings import SERVTEST_USERNAME
 from system_test import action
+from system_test import deferred_decorator
+from system_test import logger
 from system_test.helpers.decorators import make_snapshot_if_step_fail
 
 
 # pylint: disable=no-member
 class VMwareActions(object):
-    """VMware vCenter/DVS related actions"""
+    """VMware vCenter/DVS related actions."""
 
     plugin_version = None
+    vms_to_ping = []  # instances which should ping each other
+    vip_contr = None  # controller with VIP resources
+    primary_ctlr_ng = None  # nailgun primary controller
+    os_conn = None
     vcenter_az = 'vcenter'
     cinder_az = 'vcenter-cinder'
     vmware_image = 'TestVM-VMDK'
@@ -47,8 +56,7 @@ class VMwareActions(object):
     @deferred_decorator([make_snapshot_if_step_fail])
     @action
     def configure_dvs_plugin(self):
-        """Configure DVS plugin"""
+        """Configure DVS plugin."""
-
         msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
         assert_true(
             self.fuel_web.check_plugin_exists(
@@ -77,8 +85,7 @@ class VMwareActions(object):
     @deferred_decorator([make_snapshot_if_step_fail])
     @action
     def configure_vcenter(self):
-        """Configure vCenter settings"""
+        """Configure vCenter settings."""
-
         vmware_vcenter = self.env_settings['vmware_vcenter']
 
         vcenter_value = {
@@ -160,7 +167,7 @@ class VMwareActions(object):
     @deferred_decorator([make_snapshot_if_step_fail])
     @action
     def set_custom_node_names(self):
-        """Set custom node names"""
+        """Set custom node names."""
         custom_hostnames = []
         for node in self.fuel_web.client.list_cluster_nodes(self.cluster_id):
             custom_hostname = "{0}-{1}".format(
@@ -170,7 +177,8 @@ class VMwareActions(object):
 
     @staticmethod
     def get_nova_conf_dict(az, nova):
-        """
+        """Return nova conf_dict.
+
         :param az: vcenter az (api), dict
         :param nova:  nova (api), dict
         :return: dict
@@ -188,8 +196,7 @@ class VMwareActions(object):
     @deferred_decorator([make_snapshot_if_step_fail])
     @action
     def check_nova_conf(self):
-        """Verify nova-compute vmware configuration"""
+        """Verify nova-compute vmware configuration."""
-
         nodes = self.fuel_web.client.list_cluster_nodes(self.cluster_id)
         vmware_attr = self.fuel_web.client.get_cluster_vmware_attributes(
             self.cluster_id)
@@ -231,8 +238,7 @@ class VMwareActions(object):
     @deferred_decorator([make_snapshot_if_step_fail])
     @action
     def check_nova_srv(self):
-        """Verify nova-compute service for each vSphere cluster"""
+        """Verify nova-compute service for each vSphere cluster."""
-
         vmware_attr = self.fuel_web.client.get_cluster_vmware_attributes(
             self.cluster_id)
         az = vmware_attr['editable']['value']['availability_zones'][0]
@@ -251,8 +257,7 @@ class VMwareActions(object):
     @deferred_decorator([make_snapshot_if_step_fail])
     @action
     def check_cinder_vmware_srv(self):
-        """Verify cinder-vmware service"""
+        """Verify cinder-vmware service."""
-
         ctrl_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
             self.cluster_id, ["controller"])
         cmd = '. openrc; cinder-manage service list | grep vcenter | ' \
@@ -263,7 +268,7 @@ class VMwareActions(object):
     @deferred_decorator([make_snapshot_if_step_fail])
     @action
     def deploy_changes(self):
-        """Deploy environment"""
+        """Deploy environment."""
         if self.cluster_id is None:
             raise SkipTest()
 
@@ -272,8 +277,7 @@ class VMwareActions(object):
 
     @action
     def create_and_attach_empty_volume(self):
-        """Create and attach to instance empty volume"""
+        """Create and attach to instance empty volume."""
-
         mount_point = '/dev/sdb'
 
         public_ip = self.fuel_web.get_public_vip(self.cluster_id)
@@ -324,8 +328,7 @@ class VMwareActions(object):
 
     @action
     def create_bootable_volume_and_run_instance(self):
-        """Create bootable volume and launch instance from it"""
+        """Create bootable volume and launch instance from it."""
-
         public_ip = self.fuel_web.get_public_vip(self.cluster_id)
         os_conn = OpenStackActions(public_ip)
 
@@ -350,9 +353,8 @@ class VMwareActions(object):
 
     @action
     def check_vmware_service_actions(self):
-        """Disable vmware host (cluster) and check instance creation
+        """Disable vmware host (cluster) and check instance creation on
-        on enabled cluster"""
+        enabled cluster."""
-
         public_ip = self.fuel_web.get_public_vip(self.cluster_id)
         os_conn = OpenStackActions(public_ip)
 
@@ -371,7 +373,7 @@ class VMwareActions(object):
         for service in vmware_services:
             logger.info("Check {}".format(service.host))
             os_conn.enable_nova_service(service)
-            vm = os_conn.create_server(image=image,
+            vm = os_conn.create_server(image=image, timeout=180,
                                        availability_zone=self.vcenter_az,
                                        net_id=net['id'], security_groups=[sg])
             vm_host = getattr(vm, 'OS-EXT-SRV-ATTR:host')
@@ -383,8 +385,7 @@ class VMwareActions(object):
 
     @action
     def upload_image(self):
-        """Upload vmdk image"""
+        """Upload vmdk image."""
-
         controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
             self.cluster_id, ["controller"])[0]
 
@@ -407,8 +408,7 @@ class VMwareActions(object):
 
     @action
     def check_instance_creation(self):
-        """Create instance and check connection"""
+        """Create instance and check connection."""
-
         public_ip = self.fuel_web.get_public_vip(self.cluster_id)
         os_conn = OpenStackActions(public_ip)
 
@@ -433,8 +433,7 @@ class VMwareActions(object):
 
     @action
     def create_instance_with_vmxnet3_adapter(self):
-        """Create instance with vmxnet3 adapter"""
+        """Create instance with vmxnet3 adapter."""
-
         public_ip = self.fuel_web.get_public_vip(self.cluster_id)
         os_conn = OpenStackActions(public_ip)
 
@@ -467,8 +466,7 @@ class VMwareActions(object):
 
     @action
     def check_batch_instance_creation(self):
-        """Create several instance simultaneously"""
+        """Create several instance simultaneously."""
-
         count = 10
         vm_name = 'vcenter_vm'
 
@@ -500,8 +498,7 @@ class VMwareActions(object):
 
     @action
     def create_instance_with_different_disktype(self):
-        """Create instances with different disk type"""
+        """Create instances with different disk type."""
-
         public_ip = self.fuel_web.get_public_vip(self.cluster_id)
         os_conn = OpenStackActions(public_ip)
 
@@ -535,8 +532,7 @@ class VMwareActions(object):
     @deferred_decorator([make_snapshot_if_step_fail])
     @action
     def check_neutron_public(self):
-        """Check that public network was assigned to all nodes"""
+        """Check that public network was assigned to all nodes."""
-
         cluster = self.fuel_web.client.get_cluster(self.cluster_id)
         assert_equal(str(cluster['net_provider']), NEUTRON)
         os_conn = OpenStackActions(
@@ -547,8 +543,7 @@ class VMwareActions(object):
     @deferred_decorator([make_snapshot_if_step_fail])
     @action
     def check_gw_on_vmware_nodes(self):
-        """Check that default gw != fuel node ip"""
+        """Check that default gw != fuel node ip."""
-
         vmware_nodes = []
         vmware_nodes.extend(self.fuel_web.get_nailgun_cluster_nodes_by_roles(
             self.cluster_id, ["compute-vmware"]))
@@ -561,3 +556,285 @@ class VMwareActions(object):
             logger.debug('Default gw for node {0} is {1}'.format(
                 node['name'], gw_ip['stdout_str']))
             assert_not_equal(gw_ip['stdout_str'], self.fuel_web.admin_node_ip)
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def config_idatastore(self):
+        """Reconfigure vCenter settings with incorrect regex of Datastore."""
+        vmware_attr = \
+            self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id)
+        vcenter_data = vmware_attr['editable']
+        vcenter_data['value']['availability_zones'][0]['nova_computes'][0][
+            'datastore_regex'] = '!@#$%^&*()'
+
+        self.fuel_web.client.update_cluster_vmware_attributes(self.cluster_id,
+                                                              vmware_attr)
+        logger.info("Datastore regex settings have been updated")
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def config_idc_glance(self):
+        """Reconfigure vCenter settings with incorrect Glance Datacenter."""
+        vmware_attr = \
+            self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id)
+        vcenter_data = vmware_attr['editable']
+        vcenter_data['value']['glance']['datacenter'] = '!@#$%^&*()'
+
+        self.fuel_web.client.update_cluster_vmware_attributes(self.cluster_id,
+                                                              vmware_attr)
+        logger.info("Glance datacenter settings have been updated")
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def config_ids_glance(self):
+        """Reconfigure vCenter settings with incorrect Glance Datastore."""
+        vmware_attr = \
+            self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id)
+        vcenter_data = vmware_attr['editable']
+        vcenter_data['value']['glance']['datastore'] = '!@#$%^&*()'
+
+        self.fuel_web.client.update_cluster_vmware_attributes(self.cluster_id,
+                                                              vmware_attr)
+
+        logger.info("Glance datastore settings have been updated")
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def delete_instances(self):
+        """Delete created instances."""
+        for srv in self.vms_to_ping:
+            logger.info('Started: delete existing VM "{}"'.format(srv.name))
+            self.os_conn.nova.servers.delete(srv)
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def create_instances(self):
+        """Create instances with nova az and vcenter az."""
+        os_ip = self.fuel_web.get_public_vip(self.cluster_id)
+        self.os_conn = OpenStackActions(
+            os_ip, SERVTEST_USERNAME,
+            SERVTEST_PASSWORD,
+            SERVTEST_TENANT
+        )
+        vcenter_az = self.env_settings['vmware_vcenter']['settings']['az']
+        net = self.os_conn.get_network(self.net_name)
+        sec_group = self.os_conn.create_sec_group_for_ssh()
+
+        inst_count = 1  # amount of VMs to create on each az
+        vc_inst_name_prefix = 'vcenter-test'
+        vc_image = self.os_conn.get_image('TestVM-VMDK')
+        nova_inst_name_prefix = 'nova-test'
+        nova_image = self.os_conn.get_image('TestVM')
+
+        logger.info(
+            'Started: create {num} VM(s) with net="{net}", az="{az}", '
+            'image="{image}"'.format(num=inst_count, net=net['name'],
+                                     az=vcenter_az, image='TestVM-VMDK')
+        )
+        self.os_conn.create_server(
+            name=vc_inst_name_prefix,
+            net_id=net['id'],
+            availability_zone=vcenter_az,
+            image=vc_image,
+            timeout=200,
+            security_groups=[sec_group],
+            min_count=inst_count
+        )
+
+        logger.info(
+            'Started: create {num} VM(s) with net="{net}", az="{az}", '
+            'image="{image}"'.format(num=inst_count, net=net['name'],
+                                     az='nova', image='TestVM')
+        )
+        self.os_conn.create_server(
+            name=nova_inst_name_prefix,
+            net_id=net['id'],
+            image=nova_image,
+            security_groups=[sec_group],
+            availability_zone='nova',
+            min_count=inst_count
+        )
+
+        servers = self.os_conn.nova.servers.list()
+        self.vms_to_ping = [srv for srv in servers if
+                            srv.name.startswith(vc_inst_name_prefix) or
+                            srv.name.startswith(nova_inst_name_prefix)]
+
+    def _get_controller_with_vip(self):
+        """Return name of controller with VIPs."""
+        for node in self.env.d_env.nodes().slaves:
+            ng_node = self.env.fuel_web.get_nailgun_node_by_devops_node(node)
+            if ng_node['online']:
+                hosts_vip = self.fuel_web.get_pacemaker_resource_location(
+                    ng_node['devops_name'], 'vip__management')
+                logger.info('Now primary controller is '
+                            '{}'.format(hosts_vip[0].name))
+                return hosts_vip[0].name
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def hard_reset_primary(self):
+        """Hard reboot of primary controller."""
+        self.vip_contr = self._get_controller_with_vip()
+
+        self.primary_ctlr_ng = self.fuel_web.get_nailgun_primary_node(
+            self.env.d_env.nodes().slaves[0])
+
+        self.fuel_web.cold_restart_nodes([self.primary_ctlr_ng])
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def shutdown_primary(self):
+        """Shut down primary controller."""
+        self.vip_contr = self._get_controller_with_vip()
+
+        self.primary_ctlr_ng = self.fuel_web.get_nailgun_primary_node(
+            self.env.d_env.nodes().slaves[0])
+
+        self.primary_ctlr_ng.destroy()
+
+        timeout = 60 * 10
+        logger.info('Wait offline status for '
+                    '{ctrlr}'.format(ctrlr=self.primary_ctlr_ng.name))
+
+        helpers.wait(lambda: self.env.fuel_web.get_nailgun_node_by_devops_node(
+                     self.primary_ctlr_ng)['online'] is not True,
+                     timeout=timeout,
+                     timeout_msg="Primary controller is still online")
+        logger.info('Primary controller is offline')
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def safe_reboot_primary(self):
+        """Safe reboot primary controller."""
+        self.vip_contr = self._get_controller_with_vip()
+
+        self.primary_ctlr_ng = self.fuel_web.get_nailgun_primary_node(
+            self.env.d_env.nodes().slaves[0])
+
+        self.fuel_web.warm_restart_nodes([self.primary_ctlr_ng])
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def check_up_vips(self):
+        """Ensure that VIPs are moved to another controller."""
+        vip_contr = self._get_controller_with_vip()
+
+        assert_true(vip_contr and vip_contr != self.vip_contr,
+                    'VIPs have not been moved to another controller')
+        logger.info('VIPs have been moved to another controller')
+
+    # @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def turn_on_primary(self):
+        """Turn on primary controller."""
+        self.primary_ctlr_ng.start()
+        logger.info('Started: turn on primary controller '
+                    '{name}'.format(name=self.primary_ctlr_ng.name))
+
+        timeout = 60 * 10
+        logger.info('Wait online status for '
+                    '{name}'.format(name=self.primary_ctlr_ng.name))
+
+        helpers.wait(lambda: self.env.fuel_web.get_nailgun_node_by_devops_node(
+                     self.primary_ctlr_ng)['online'], timeout=timeout,
+                     timeout_msg="Primary controller is still offline")
+        logger.info('Primary controller is online')
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def vcenter_ostf(self):
+        """Run vCenter OSTF tests."""
+        self.fuel_web.run_ostf(
+            cluster_id=self.cluster_id,
+            test_sets=['smoke'],
+            should_fail=getattr(self, 'ostf_tests_should_failed', 0),
+            failed_test_name=getattr(self, 'failed_test_name', None))
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def ostf_with_services_fail(self):
+        """Run OSTF tests (one should fail)."""
+        self.env.fuel_web.run_ostf(
+            self.cluster_id, should_fail=1,
+            failed_test_name='Check that required services are running')
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def fail_ostf(self):
+        """Run OSTF tests (must fail)."""
+        try:
+            self.env.fuel_web.run_ostf(
+                self.cluster_id,
+                test_sets=['sanity', 'smoke', 'ha'])
+            failed = False
+        except AssertionError:
+            failed = True
+        assert_true(failed, 'OSTF passed with incorrect parameters')
+        logger.info('OSTF failed')
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def fail_deploy_cluster(self):
+        """Deploy environment (must fail)."""
+        try:
+            self.fuel_web.deploy_cluster_wait(self.cluster_id)
+            failed = False
+        except AssertionError:
+            failed = True
+        assert_true(failed, 'Deploy passed with incorrect parameters')
+        logger.info('Deploy failed')
+
+    def ping_from_instance(self, src_floating_ip, dst_ip, primary,
+                           size=56, count=1):
+        """Verify ping between instances.
+
+        :param src_floating_ip: floating ip address of instance
+        :param dst_ip: destination ip address
+        :param primary: name of the primary controller
+        :param size: number of data bytes to be sent
+        :param count: number of packets to be sent
+        """
+        creds = ("cirros", "cubswin:)")
+
+        with self.fuel_web.get_ssh_for_node(primary) as ssh:
+            command = "ping -s {0} -c {1} {2}".format(size, count,
+                                                      dst_ip)
+            ping = self.os_conn.execute_through_host(ssh, src_floating_ip,
+                                                     command, creds)
+            logger.info("Ping result is {}".format(ping['exit_code']))
+            return 0 == ping['exit_code']
+
+    @deferred_decorator([make_snapshot_if_step_fail])
+    @action
+    def check_vm_connect(self):
+        """Ensure connectivity between VMs."""
+        if self.vip_contr:
+            primary_ctrl_name = self._get_controller_with_vip()
+        else:
+            primary_ctrl_name = self.fuel_web.get_nailgun_primary_node(
+                self.env.d_env.nodes().slaves[0]).name
+
+        private_ips = {}
+        floating_ips = {}
+
+        for srv in self.vms_to_ping:
+            floating = self.os_conn.assign_floating_ip(srv)
+            floating_ips[srv] = floating.ip
+            logger.info("Floating address {0} was associated with instance "
+                        "{1}".format(floating_ips[srv], srv.name))
+
+            private_ips[srv] = self.os_conn.get_nova_instance_ip(
+                srv, net_name=self.net_name)
+
+        for vm in itertools.combinations(self.vms_to_ping, 2):
+            logger.info('Try to ping from {src} ({src_vm}) to {dst} '
+                        '({dst_vm})'.format(src=floating_ips[vm[0]],
+                                            dst=private_ips[vm[1]],
+                                            src_vm=vm[0].name,
+                                            dst_vm=vm[1].name))
+
+            assert_true(self.ping_from_instance(floating_ips[vm[0]],
+                                                private_ips[vm[1]],
+                                                primary_ctrl_name),
+                        'Ping between VMs failed')

system_test/tests/vcenter/test_vcenter_dvs.py

@@ -14,22 +14,22 @@
 
 from fuelweb_test.settings import DVS_PLUGIN_PATH
 from fuelweb_test.settings import DVS_PLUGIN_VERSION
-from fuelweb_test.settings import VMWARE_IMG_NAME
-from fuelweb_test.settings import VMWARE_IMG_URL
 from fuelweb_test.settings import VMWARE_IMG_LOGIN
+from fuelweb_test.settings import VMWARE_IMG_NAME
 from fuelweb_test.settings import VMWARE_IMG_PASSWORD
+from fuelweb_test.settings import VMWARE_IMG_URL
 
 from system_test import testcase
-from system_test.tests import ActionTest
 from system_test.actions import BaseActions
 from system_test.actions import VMwareActions
+from system_test.tests import ActionTest
 
 
 @testcase(groups=['system_test',
                   'system_test.vcenter',
                   'system_test.vcenter.deploy_vcenter_dvs_run_ostf'])
 class DeployWithVMware(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and dvs plugin
+    """Deploy cluster with vCenter and dvs plugin.
 
     Scenario:
         1. Upload plugin to the master node
@@ -64,7 +64,7 @@ class DeployWithVMware(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.scale_vcenter_dvs'])
 class ScaleWithVMware(ActionTest, BaseActions, VMwareActions):
-    """Deploy and scale cluster with vCenter and dvs plugin
+    """Deploy and scale cluster with vCenter and dvs plugin.
 
     Scenario:
         1. Upload plugin to the master node
@@ -143,7 +143,7 @@ class DeployWithVMwareRunPlatformOSTF(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.deploy_with_custom_hostname'])
 class DeployWithCustomHostname(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and custom hostname
+    """Deploy cluster with vCenter and custom hostname.
 
     Scenario:
         1. Upload plugin to the master node
@@ -180,7 +180,7 @@ class DeployWithCustomHostname(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.check_nova_config'])
 class CheckNovaConfig(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and custom hostname
+    """Deploy cluster with vCenter and custom hostname.
 
     Scenario:
         1. Upload plugin to the master node
@@ -217,7 +217,7 @@ class CheckNovaConfig(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.check_nova_srv'])
 class CheckNovaSrv(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and custom hostname
+    """Deploy cluster with vCenter and custom hostname.
 
     Scenario:
         1. Upload plugin to the master node
@@ -254,7 +254,7 @@ class CheckNovaSrv(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.check_cinder_srv'])
 class CheckCinderVmwareSrv(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and custom hostname
+    """Deploy cluster with vCenter and custom hostname.
 
     Scenario:
         1. Upload plugin to the master node
@@ -291,7 +291,7 @@ class CheckCinderVmwareSrv(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.attach_empty_volume'])
 class AttachEmptyVol(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and dvs plugin
+    """Deploy cluster with vCenter and dvs plugin.
 
     Scenario:
         1. Upload plugin to the master node
@@ -326,7 +326,7 @@ class AttachEmptyVol(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.bootable_vol'])
 class BootableVol(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and dvs plugin
+    """Deploy cluster with vCenter and dvs plugin.
 
     Scenario:
         1. Upload plugin to the master node
@@ -361,7 +361,7 @@ class BootableVol(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.disable_enable_compute_service'])
 class DisableEnableVMwareServices(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and dvs plugin
+    """Deploy cluster with vCenter and dvs plugin.
 
     Scenario:
         1. Upload plugin to the master node
@@ -396,7 +396,7 @@ class DisableEnableVMwareServices(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.upload_image'])
 class UploadImage(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and dvs plugin
+    """Deploy cluster with vCenter and dvs plugin.
 
     Scenario:
         1. Upload plugin to the master node
@@ -436,7 +436,7 @@ class UploadImage(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.vmxnet3'])
 class Vmxnet3(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and dvs plugin
+    """Deploy cluster with vCenter and dvs plugin.
 
     Scenario:
         1. Upload plugin to the master node
@@ -476,7 +476,7 @@ class Vmxnet3(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.create_batch_of_instances'])
 class CreateBatchInstances(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and dvs plugin
+    """Deploy cluster with vCenter and dvs plugin.
 
     Scenario:
         1. Upload plugin to the master node
@@ -511,7 +511,7 @@ class CreateBatchInstances(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.diff_disk_types'])
 class DiffDiskTypes(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and dvs plugin
+    """Deploy cluster with vCenter and dvs plugin.
 
     Scenario:
         1. Upload plugin to the master node
@@ -546,7 +546,7 @@ class DiffDiskTypes(ActionTest, BaseActions, VMwareActions):
                   'system_test.vcenter',
                   'system_test.vcenter.neutron_public_net'])
 class DeployNeutronPublicNet(ActionTest, BaseActions, VMwareActions):
-    """Deploy cluster with vCenter and dvs plugin
+    """Deploy cluster with vCenter and dvs plugin.
 
     Scenario:
         1. Upload plugin to the master node

system_test/tests/vcenter/test_vcenter_failover.py

@@ -0,0 +1,365 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE_2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from fuelweb_test.settings import DVS_PLUGIN_PATH
+from fuelweb_test.settings import DVS_PLUGIN_VERSION
+
+from system_test import testcase
+from system_test.actions import BaseActions
+from system_test.actions import VMwareActions
+from system_test.tests import ActionTest
+
+
+@testcase(groups=['system_test',
+                  'system_test.vcenter',
+                  'system_test.vcenter.vcenter_reset_ctrl'])
+class HardResetPrimaryWithVMware(ActionTest, BaseActions, VMwareActions):
+    """Hard reset primary controller and check vCenter functionality.
+
+    Scenario:
+        1. Upload plugin to the master node
+        2. Install plugin
+        3. Create cluster
+        4. Configure dvs settings (depends on yaml config)
+        5. Add nodes (depends on yaml config)
+        6. Configure vmware settings (depends on yaml config)
+        7. Deploy the cluster
+        8. Create instances on Nova and vCenter
+        9. Hard reset primary controller
+        10. Wait 5-10 minutes
+        11. Verify networks
+        12. Ensure that VIPs are moved to other controller
+        13. Ensure connectivity between VMs
+        14. Run OSTF tests
+
+    Duration 3h 00min
+    Snapshot vcenter_reset_ctrl
+    """
+
+    plugin_name = "fuel-plugin-vmware-dvs"
+    plugin_path = DVS_PLUGIN_PATH
+    plugin_version = DVS_PLUGIN_VERSION
+
+    actions_order = [
+        'prepare_env_with_plugin',
+        'create_env',
+        'configure_dvs_plugin',
+        'add_nodes',
+        'configure_vcenter',
+        'deploy_cluster',
+        'create_instances',
+        'hard_reset_primary',
+        'network_check',
+        'check_up_vips',
+        'check_vm_connect',
+        'delete_instances',
+        'health_check_sanity_smoke_ha'
+    ]
+
+
+@testcase(groups=['system_test',
+                  'system_test.vcenter',
+                  'system_test.vcenter.vcenter_shutdown_ctrl'])
+class ShutdownPrimaryWithVMware(ActionTest, BaseActions, VMwareActions):
+    """Shutdown primary controller and check vCenter functionality.
+
+    Scenario:
+        1. Upload plugin to the master node
+        2. Install plugin
+        3. Create cluster
+        4. Configure dvs settings (depends on yaml config)
+        5. Add nodes (depends on yaml config)
+        6. Configure vmware settings (depends on yaml config)
+        7. Deploy the cluster
+        8. Create instances on Nova and vCenter
+        9. Shutdown primary controller
+        10. Verify networks
+        11. Ensure that VIPs are moved to other controller
+        12. Ensure connectivity between VMs
+        13. Run OSTF tests (one should fail)
+        14. Turn on primary controller
+        15. Wait 5-10 minutes
+        16. Verify networks
+        17. Run OSTF tests
+
+    Duration 3h 00min
+    Snapshot vcenter_shutdown_ctrl
+    """
+
+    plugin_name = "fuel-plugin-vmware-dvs"
+    plugin_path = DVS_PLUGIN_PATH
+    plugin_version = DVS_PLUGIN_VERSION
+
+    actions_order = [
+        'prepare_env_with_plugin',
+        'create_env',
+        'configure_dvs_plugin',
+        'add_nodes',
+        'configure_vcenter',
+        'deploy_cluster',
+        'create_instances',
+        'shutdown_primary',
+        'network_check',
+        'check_up_vips',
+        'check_vm_connect',
+        'delete_instances',
+        'ostf_with_services_fail',
+        'turn_on_primary',
+        'network_check',
+        'health_check_sanity_smoke_ha'
+    ]
+
+
+@testcase(groups=['system_test',
+                  'system_test.vcenter',
+                  'system_test.vcenter.vcenter_reboot_ctrl'])
+class SafeRebootPrimaryWithVMware(ActionTest, BaseActions, VMwareActions):
+    """Safe reboot primary controller and check vCenter functionality.
+
+    Scenario:
+        1. Upload plugin to the master node
+        2. Install plugin
+        3. Create cluster
+        4. Configure dvs settings (depends on yaml config)
+        5. Add nodes (depends on yaml config)
+        6. Configure vmware settings (depends on yaml config)
+        7. Deploy the cluster
+        8. Create instances on Nova and vCenter
+        9. Safe reboot primary controller
+        10. Wait 5-10 minutes
+        11. Verify networks
+        12. Ensure that VIPs are moved to other controller
+        13. Ensure connectivity between VMs
+        14. Run OSTF tests
+
+    Duration 3h 00min
+    Snapshot vcenter_reboot_ctrl
+    """
+
+    plugin_name = "fuel-plugin-vmware-dvs"
+    plugin_path = DVS_PLUGIN_PATH
+    plugin_version = DVS_PLUGIN_VERSION
+
+    actions_order = [
+        'prepare_env_with_plugin',
+        'create_env',
+        'configure_dvs_plugin',
+        'add_nodes',
+        'configure_vcenter',
+        'deploy_cluster',
+        'create_instances',
+        'safe_reboot_primary',
+        'network_check',
+        'check_up_vips',
+        'check_vm_connect',
+        'delete_instances',
+        'health_check_sanity_smoke_ha'
+    ]
+
+
+@testcase(groups=['system_test',
+                  'system_test.vcenter',
+                  'system_test.vcenter.vcenter_shutdown_cindervmware'])
+class ShutdownCinderNodeWithVMware(ActionTest, BaseActions, VMwareActions):
+    """Shutdown one of CinderVMDK node.
+
+    Scenario:
+        1. Upload plugin to the master node
+        2. Install plugin
+        3. Create cluster
+        4. Configure dvs settings (depends on yaml config)
+        5. Add nodes (depends on yaml config)
+        6. Configure vmware settings (depends on yaml config)
+        7. Deploy the cluster
+        8. Create instances on KVM and vCenter
+        9. Run all OSTF tests
+        10. Shutdown one of CinderVMDK node
+        11. Run vCenter OSTF tests
+        12. Power on CinderVMDK node and wait for it to load
+        13. Run vCenter OSTF tests
+        14. Shutdown another CinderVMDK node
+        15. Run vCenter OSTF tests
+        16. Power on CinderVMDK node and wait for it to load
+        17. Run all OSTF tests
+
+    Duration 3h 00min
+    Snapshot vcenter_shutdown_cindervmware
+    """
+
+    plugin_name = "fuel-plugin-vmware-dvs"
+    plugin_path = DVS_PLUGIN_PATH
+    plugin_version = DVS_PLUGIN_VERSION
+
+    actions_order = [
+        'prepare_env_with_plugin',
+        'create_env',
+        'configure_dvs_plugin',
+        'add_nodes',
+        'configure_vcenter',
+        'deploy_cluster',
+        'create_instances',
+        'health_check_sanity_smoke_ha',
+        'manage_nodes_power',
+        'vcenter_ostf',
+        'manage_nodes_power',
+        'vcenter_ostf',
+        'manage_nodes_power',
+        'vcenter_ostf',
+        'manage_nodes_power',
+        'delete_instances',
+        'health_check_sanity_smoke_ha'
+    ]
+
+
+@testcase(groups=['system_test',
+                  'system_test.vcenter',
+                  'system_test.vcenter.vcenter_reboot_cindervmware'])
+class RebootCinderNodeWithVMware(ActionTest, BaseActions, VMwareActions):
+    """Restart CinderVMware node.
+
+    Scenario:
+        1. Upload plugin to the master node
+        2. Install plugin
+        3. Create cluster
+        4. Configure dvs settings (depends on yaml config)
+        5. Add nodes (depends on yaml config)
+        6. Configure vmware settings (depends on yaml config)
+        7. Deploy the cluster
+        8. Reboot CinderVMware node.
+        9. Check CinderVMware services.
+
+    Duration 3h 00min
+    Snapshot vcenter_reboot_cindervmware
+    """
+
+    plugin_name = "fuel-plugin-vmware-dvs"
+    plugin_path = DVS_PLUGIN_PATH
+    plugin_version = DVS_PLUGIN_VERSION
+
+    actions_order = [
+        'prepare_env_with_plugin',
+        'create_env',
+        'configure_dvs_plugin',
+        'add_nodes',
+        'configure_vcenter',
+        'deploy_cluster',
+        'manage_nodes_power',
+        'check_cinder_vmware_srv',
+        'health_check'
+    ]
+
+
+@testcase(groups=['system_test',
+                  'system_test.vcenter',
+                  'system_test.vcenter.vcenter_iname_glance_ds'])
+class DeployINameDSWithVMware(ActionTest, BaseActions, VMwareActions):
+    """Deploy with controller and incorrect name of vCenter Glance Datastore.
+
+    Scenario:
+        1. Upload plugin to the master node
+        2. Install plugin
+        3. Create cluster
+        4. Configure dvs settings (depends on yaml config)
+        5. Add nodes (depends on yaml config)
+        6. Configure vmware settings (depends on yaml config)
+        7. Deploy the cluster (Deploy should fail)
+
+    Duration 3h 00min
+    Snapshot vcenter_iname_glance_ds
+    """
+
+    plugin_name = "fuel-plugin-vmware-dvs"
+    plugin_path = DVS_PLUGIN_PATH
+    plugin_version = DVS_PLUGIN_VERSION
+
+    actions_order = [
+        'prepare_env_with_plugin',
+        'create_env',
+        'configure_dvs_plugin',
+        'add_nodes',
+        'configure_vcenter',
+        'config_ids_glance',
+        'fail_deploy_cluster'
+    ]
+
+
+@testcase(groups=['system_test',
+                  'system_test.vcenter',
+                  'system_test.vcenter.vcenter_iname_glance_dc'])
+class DeployINameDCWithVMware(ActionTest, BaseActions, VMwareActions):
+    """Deploy with controller and incorrect name of vCenter Glance Datacenter.
+
+    Scenario:
+        1. Upload plugin to the master node
+        2. Install plugin
+        3. Create cluster
+        4. Configure dvs settings (depends on yaml config)
+        5. Add nodes (depends on yaml config)
+        6. Configure vmware settings (depends on yaml config)
+        7. Deploy the cluster (Deploy should fail)
+
+    Duration 3h 00min
+    Snapshot vcenter_iname_glance_dc
+    """
+
+    plugin_name = "fuel-plugin-vmware-dvs"
+    plugin_path = DVS_PLUGIN_PATH
+    plugin_version = DVS_PLUGIN_VERSION
+
+    actions_order = [
+        'prepare_env_with_plugin',
+        'create_env',
+        'configure_dvs_plugin',
+        'add_nodes',
+        'configure_vcenter',
+        'config_idc_glance',
+        'fail_deploy_cluster'
+    ]
+
+
+@testcase(groups=['system_test',
+                  'system_test.vcenter',
+                  'system_test.vcenter.vcenter_idatastore'])
+class DeployIDSWithVMware(ActionTest, BaseActions, VMwareActions):
+    """Deploy with controller and not correct regex of vCenter Datastore.
+
+    Scenario:
+        1. Upload plugin to the master node
+        2. Install plugin
+        3. Create cluster
+        4. Configure dvs settings (depends on yaml config)
+        5. Add nodes (depends on yaml config)
+        6. Configure vmware settings (depends on yaml config)
+        7. Redefine vmware settings with incorrect ds
+        8. Deploy the cluster
+        9. Run OSTF tests (should fail)
+
+    Duration 3h 00min
+    Snapshot vcenter_idatastore
+    """
+
+    plugin_name = "fuel-plugin-vmware-dvs"
+    plugin_path = DVS_PLUGIN_PATH
+    plugin_version = DVS_PLUGIN_VERSION
+
+    actions_order = [
+        'prepare_env_with_plugin',
+        'create_env',
+        'configure_dvs_plugin',
+        'add_nodes',
+        'configure_vcenter',
+        'config_idatastore',
+        'deploy_cluster',
+        'fail_ostf'
+    ]

system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_dvs_failover.yaml

@@ -0,0 +1,37 @@
+---
+template:
+ name: 5 nodes on Neutron/VLAN with DVS plugin (3 controllers, 1 compute + cinder, 1 cinder-vmware)
+ slaves: 5
+ cluster_template:
+  name: vcenter_dvs_failover
+  release: ubuntu
+  network:
+    !include cluster_configs/networks/neutron_vlan.yaml
+  settings:
+    components:
+      !include cluster_configs/settings/components/wo_components.yaml
+    storages:
+      !include cluster_configs/settings/storages/cinder_only.yaml
+    vmware_vcenter:
+      settings:
+        !include cluster_configs/settings/vmware/vcenter_main.yaml
+      nova-compute:
+        !include cluster_configs/settings/vmware/nova_compute/2clusters_ctrl.yaml
+      glance:
+        !include cluster_configs/settings/vmware/vcenter_glance.yaml
+    vmware_dvs:
+      !include cluster_configs/settings/vmware/dvs/dvs_main.yaml
+  nodes:
+    - roles:
+      - controller
+      iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml
+      count: 3
+    - roles:
+      - compute
+      - cinder
+      iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml
+      count: 1
+    - roles:
+      - cinder-vmware
+      iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml
+      count: 1

system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_reboot_cvm.yaml

@@ -0,0 +1,39 @@
+---
+template:
+ name: 1 controller, 1 cinder-vmware on Neutron/VLAN with DVS plugin
+ slaves: 2
+ cluster_template:
+  name: vcenter_reboot_cindervmware
+  release: ubuntu
+  network:
+    !include cluster_configs/networks/neutron_vlan.yaml
+  settings:
+    components:
+      !include cluster_configs/settings/components/wo_components.yaml
+    storages:
+      !include cluster_configs/settings/storages/cinder_only.yaml
+    vmware_vcenter:
+      settings:
+        !include cluster_configs/settings/vmware/vcenter_main.yaml
+      nova-compute:
+        !include cluster_configs/settings/vmware/nova_compute/1cluster_ctrl.yaml
+      glance:
+        enable: false
+    vmware_dvs:
+      !include cluster_configs/settings/vmware/dvs/dvs_main.yaml
+  nodes:
+    - roles:
+      - controller
+      iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml
+      count: 1
+    - roles:
+      - cinder-vmware
+      iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml
+      count: 1
+
+
+  manage_nodes_power:
+  - - roles:
+      - cinder-vmware
+      node_number: 0
+      action: reboot_warm

system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_shutdown_cvm.yaml

@@ -0,0 +1,56 @@
+---
+template:
+ name: 1 controller, 1 compute + cinder, 2 cinder-vmware on Neutron/VLAN with DVS plugin
+ slaves: 4
+ cluster_template:
+  name: vcenter_shutdown_cindervmware
+  release: ubuntu
+  network:
+    !include cluster_configs/networks/neutron_vlan.yaml
+  settings:
+    components:
+      !include cluster_configs/settings/components/wo_components.yaml
+    storages:
+      !include cluster_configs/settings/storages/cinder_only.yaml
+    vmware_vcenter:
+      settings:
+        !include cluster_configs/settings/vmware/vcenter_main.yaml
+      nova-compute:
+        !include cluster_configs/settings/vmware/nova_compute/1cluster_ctrl.yaml
+      glance:
+        enable: false
+    vmware_dvs:
+      !include cluster_configs/settings/vmware/dvs/dvs_main.yaml
+  nodes:
+    - roles:
+      - controller
+      iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml
+      count: 1
+    - roles:
+      - compute
+      - cinder
+      iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml
+      count: 1
+    - roles:
+      - cinder-vmware
+      iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml
+      count: 2
+
+
+  manage_nodes_power:
+  - - roles:
+      - cinder-vmware
+      node_number: 0
+      action: power_off_warm
+  - - roles:
+      - cinder-vmware
+      node_number: 0
+      action: power_on_warm
+  - - roles:
+      - cinder-vmware
+      node_number: 1
+      action: power_off_warm
+  - - roles:
+      - cinder-vmware
+      node_number: 1
+      action: power_on_warm