Browse Source

Merge "Add failover test suite"

Jenkins 2 years ago
parent
commit
560d1f4524

+ 17
- 23
doc/test/source/test_suite_failover.rst View File

@@ -65,14 +65,14 @@ core
65 65
 Steps
66 66
 #####
67 67
 
68
-    1. Log in to the Fuel with preinstalled plugin and deployed enviroment with 3 controllers and 1 compute.
68
+    1. Log in to the Fuel with preinstalled plugin and deployed ha enviroment with 3 controllers, 1 compute and 1 compute-vmware nodes.
69 69
     2. Log in to Horizon.
70
-    3. Create vcenter VM and check connectivity to outside world from VM.
70
+    3. Launch two instances in different az (nova and vcenter) and check connectivity to outside world from VMs.
71 71
     4. Shutdown primary controller.
72 72
     5. Ensure that VIPs are moved to other controller.
73
-    6. Ensure taht there is a connectivity to outside world from created VM.
73
+    6. Ensure that there is a connectivity to outside world from created VMs.
74 74
     7. Create a new network and attach it to default router.
75
-    8. Create a vcenter VM with new network and check network connectivity via ICMP.
75
+    8. Launch two instances in different az (nova and vcenter) with new network and check network connectivity via ICMP.
76 76
 
77 77
 
78 78
 Expected result
@@ -81,20 +81,20 @@ Expected result
81 81
 Networking works correct after failure of primary controller.
82 82
 
83 83
 
84
-Check cluster functionality after reboot vcenter.
85
--------------------------------------------------
84
+Check cluster functionality after interrupt connection with NSX manager.
85
+------------------------------------------------------------------------
86 86
 
87 87
 
88 88
 ID
89 89
 ##
90 90
 
91
-nsxt_reboot_vcenter
91
+nsxt_interrupt_connection
92 92
 
93 93
 
94 94
 Description
95 95
 ###########
96 96
 
97
-Test verifies that system functionality is ok when vcenter has been rebooted.
97
+Test verifies that cluster will functional after interrupt connection with NSX manager.
98 98
 
99 99
 
100 100
 Complexity
@@ -107,23 +107,17 @@ Steps
107 107
 #####
108 108
 
109 109
     1. Log in to the Fuel with preinstalled plugin and deployed enviroment.
110
-    2. Log in to Horizon.
111
-    3. Launch vcenter instance VM_1 with image TestVM-VMDK and flavor m1.tiny.
112
-    4. Launch vcenter instance VM_2 with image TestVM-VMDK and flavor m1.tiny.
113
-    5. Check connection between VMs, send ping from VM_1 to VM_2 and vice verse.
114
-    6. Reboot vcenter::
115
-
116
-          vmrun -T ws-shared -h https://localhost:443/sdk -u vmware -p pass
117
-          reset "[standard] vcenter/vcenter.vmx"
118
-
119
-    7. Check that controller lost connection with vCenter.
120
-    8. Wait for vCenter is online.
121
-    9. Ensure that all instances from vCenter are displayed in dashboard.
122
-    10. Ensure there is connectivity between vcenter1's and vcenter2's VMs.
123
-    11. Run OSTF.
110
+    2. Launch instances in each az with default network.
111
+    3. Disrupt connection with NSX manager and check that controller lost connection with NSX.
112
+    4. Try to create new network.
113
+    5. Restore connection with NSX manager.
114
+    6. Try to create new network again.
115
+    7. Launch instance in created network.
116
+    8. Ensure that all instances have connectivity to external network.
117
+    9. Run OSTF.
124 118
 
125 119
 
126 120
 Expected result
127 121
 ###############
128 122
 
129
-Cluster should be deployed and all OSTF test cases should be passed. Ping should get response.
123
+After restore connection with NSX manager cluster should be fully functional. All created VMs should be operable. All OSTF test cases should be passed.

+ 1
- 0
plugin_test/run_tests.py View File

@@ -45,6 +45,7 @@ def import_tests():
45 45
     from tests import test_plugin_nsxt  # noqa
46 46
     from tests import test_plugin_integration  # noqa
47 47
     from tests import test_plugin_scale  # noqa
48
+    from tests import test_plugin_failover  # noqa
48 49
 
49 50
 
50 51
 def run_tests():

+ 40
- 0
plugin_test/tests/base_plugin_test.py View File

@@ -15,14 +15,17 @@ under the License.
15 15
 
16 16
 import os
17 17
 
18
+from devops.helpers.ssh_client import SSHAuth
18 19
 from proboscis.asserts import assert_true
19 20
 
20 21
 from fuelweb_test import logger
21 22
 from fuelweb_test.helpers import utils
22 23
 from fuelweb_test.helpers.utils import pretty_log
23 24
 from fuelweb_test.tests.base_test_case import TestBasic
25
+from fuelweb_test.settings import SSH_IMAGE_CREDENTIALS
24 26
 from helpers import settings
25 27
 
28
+cirros_auth = SSHAuth(**SSH_IMAGE_CREDENTIALS)
26 29
 
27 30
 class TestNSXtBase(TestBasic):
28 31
     """Base class for NSX-T plugin tests"""
@@ -30,6 +33,8 @@ class TestNSXtBase(TestBasic):
30 33
     def __init__(self):
31 34
         super(TestNSXtBase, self).__init__()
32 35
         self.default = settings
36
+        self.vcenter_az = 'vcenter'
37
+        self.vmware_image = 'TestVM-VMDK'
33 38
 
34 39
     def install_nsxt_plugin(self):
35 40
         """Download and install NSX-T plugin on master node.
@@ -90,3 +95,38 @@ class TestNSXtBase(TestBasic):
90 95
             expected=[1 if failover else 0],
91 96
             raise_on_err=not failover
92 97
         )
98
+
99
+    def _get_controller_with_vip(self):
100
+        """Return name of controller with VIPs."""
101
+        for node in self.env.d_env.nodes().slaves:
102
+            ng_node = self.fuel_web.get_nailgun_node_by_devops_node(node)
103
+            if ng_node['online'] and 'controller' in ng_node['roles']:
104
+                hosts_vip = self.fuel_web.get_pacemaker_resource_location(
105
+                    ng_node['devops_name'], 'vip__management')
106
+                logger.info('Now primary controller is '
107
+                            '{}'.format(hosts_vip[0].name))
108
+                return hosts_vip[0].name
109
+        return True
110
+
111
+    def ping_from_instance(self, src_floating_ip, dst_ip, primary,
112
+                           size=56, count=1):
113
+        """Verify ping between instances.
114
+
115
+        :param src_floating_ip: floating ip address of instance
116
+        :param dst_ip: destination ip address
117
+        :param primary: name of the primary controller
118
+        :param size: number of data bytes to be sent
119
+        :param count: number of packets to be sent
120
+        """
121
+
122
+        with self.fuel_web.get_ssh_for_node(primary) as ssh:
123
+            command = "ping -s {0} -c {1} {2}".format(size, count,
124
+                                                      dst_ip)
125
+            ping = ssh.execute_through_host(
126
+                hostname=src_floating_ip,
127
+                cmd=command,
128
+                auth=cirros_auth
129
+            )
130
+
131
+            logger.info("Ping result is {}".format(ping['stdout_str']))
132
+            return 0 == ping['exit_code']

+ 193
- 0
plugin_test/tests/test_plugin_failover.py View File

@@ -0,0 +1,193 @@
1
+"""Copyright 2016 Mirantis, Inc.
2
+
3
+Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+not use this file except in compliance with the License. You may obtain
5
+copy of the License at
6
+
7
+http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+Unless required by applicable law or agreed to in writing, software
10
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+License for the specific language governing permissions and limitations
13
+under the License.
14
+"""
15
+
16
+from proboscis import test
17
+from proboscis.asserts import assert_true
18
+from devops.helpers.helpers import tcp_ping
19
+from devops.helpers.helpers import wait
20
+
21
+from fuelweb_test.helpers.os_actions import OpenStackActions
22
+from fuelweb_test.helpers.decorators import log_snapshot_after_test
23
+from fuelweb_test.settings import DEPLOYMENT_MODE
24
+from fuelweb_test.settings import SERVTEST_PASSWORD
25
+from fuelweb_test.settings import SERVTEST_TENANT
26
+from fuelweb_test.settings import SERVTEST_USERNAME
27
+from fuelweb_test.tests.base_test_case import SetupEnvironment
28
+from system_test import logger
29
+from tests.base_plugin_test import TestNSXtBase
30
+from tests.test_plugin_nsxt import TestNSXtBVT
31
+
32
+
33
+@test(groups=['nsxt_plugin', 'nsxt_failover'])
34
+class TestNSXtFailover(TestNSXtBase):
35
+    """NSX-t failover automated tests"""
36
+
37
+    @test(depends_on=[SetupEnvironment.prepare_slaves_5],
38
+          groups=['nsxt_uninstall_negative'])
39
+    @log_snapshot_after_test
40
+    def nsxt_uninstall_negative(self):
41
+        """Check plugin can not be removed while it is enabled for environment.
42
+
43
+        Scenario:
44
+            1. Install NSX-T plugin on Fuel Master node with 5 slaves.
45
+            2. Create new environment with enabled NSX-T plugin.
46
+            3. Try to delete plugin via cli from master node.
47
+
48
+        Duration: 10 min
49
+        """
50
+        # Install NSX-T plugin on Fuel Master node with 5 slaves
51
+        self.show_step(1)
52
+        self.env.revert_snapshot('ready_with_5_slaves')
53
+        self.install_nsxt_plugin()
54
+
55
+        # Create new environment with enabled NSX-T plugin
56
+        self.show_step(2)
57
+        cluster_id = self.fuel_web.create_cluster(
58
+            name=self.__class__.__name__,
59
+            mode=DEPLOYMENT_MODE,
60
+            settings=self.default.cluster_settings,
61
+            configure_ssl=False)
62
+        self.enable_plugin(cluster_id)
63
+
64
+        # Try to delete plugin via cli from master node
65
+        self.show_step(3)
66
+        self.delete_nsxt_plugin(failover=True)
67
+
68
+    @test(depends_on=[TestNSXtBVT.nsxt_bvt],
69
+          groups=['nsxt_shutdown_controller'])
70
+    @log_snapshot_after_test
71
+    def nsxt_shutdown_controller(self):
72
+        """Check plugin functionality after shutdown primary controller.
73
+
74
+        Scenario:
75
+            1. Get access to OpenStack.
76
+            2. Create VMs and check connectivity to outside world
77
+               from VM.
78
+            3. Shutdown primary controller.
79
+            4. Ensure that VIPs are moved to another controller.
80
+            5. Ensure that there is a connectivity to outside world from
81
+               created VM.
82
+            6. Create new network and attach it to default router.
83
+            7. Create VMs with new network and check network
84
+               connectivity via ICMP.
85
+
86
+        Duration: 180 min
87
+        """
88
+
89
+        # Get access to OpenStack
90
+        self.show_step(1)
91
+        cluster_id = self.fuel_web.get_last_created_cluster()
92
+        os_conn = OpenStackActions(
93
+            self.fuel_web.get_public_vip(cluster_id),
94
+            SERVTEST_USERNAME,
95
+            SERVTEST_PASSWORD,
96
+            SERVTEST_TENANT)
97
+
98
+        # Create vcenter VM and check connectivity to outside world from VM
99
+        self.show_step(2)
100
+        image = os_conn.get_image(self.vmware_image)
101
+        net = os_conn.get_network(self.default.PRIVATE_NET)
102
+        sec_group = os_conn.create_sec_group_for_ssh()
103
+
104
+        vms=[]
105
+        vms.append(os_conn.create_server(
106
+            net_id=net['id'],
107
+            security_groups=[sec_group]))
108
+        vms.append(os_conn.create_server(
109
+            availability_zone=self.vcenter_az,
110
+            image=image,
111
+            net_id=net['id'],
112
+            security_groups=[sec_group]))
113
+
114
+        ips = []
115
+        for vm in vms:
116
+            floating = os_conn.assign_floating_ip(vm)
117
+            wait(lambda: tcp_ping(floating.ip, 22),
118
+                 timeout=180,
119
+                 timeout_msg="Node {ip} is not accessible by SSH.".format(
120
+                     ip=floating.ip))
121
+            ips.append(floating.ip)
122
+
123
+        vip_contr = self._get_controller_with_vip()
124
+        for ip in ips:
125
+            logger.info('Check connectivity from {0}'.format(ip))
126
+            assert_true(self.ping_from_instance(ip,
127
+                                                '8.8.8.8',
128
+                                                vip_contr),
129
+                        'Ping failed')
130
+
131
+        # Shutdown primary controller
132
+        self.show_step(3)
133
+        primary_ctrl_devops = self.fuel_web.get_nailgun_primary_node(
134
+            self.env.d_env.nodes().slaves[0])
135
+        self.fuel_web.warm_shutdown_nodes([primary_ctrl_devops])
136
+
137
+        # Ensure that VIPs are moved to another controller
138
+        self.show_step(4)
139
+        vip_contr_new = self._get_controller_with_vip()
140
+        assert_true(vip_contr_new and vip_contr_new != vip_contr,
141
+                    'VIPs have not been moved to another controller')
142
+        logger.info('VIPs have been moved to another controller')
143
+
144
+        # Ensure that there is a connectivity to outside world from created VM
145
+        self.show_step(5)
146
+        for ip in ips:
147
+            logger.info('Check connectivity from {0}'.format(ip))
148
+            assert_true(self.ping_from_instance(ip,
149
+                                                '8.8.8.8',
150
+                                                vip_contr_new),
151
+                        'Ping failed')
152
+
153
+        # Create new network and attach it to default router
154
+        self.show_step(6)
155
+        net_1 = os_conn.create_network(network_name='net_1')['network']
156
+        subnet_1 = os_conn.create_subnet(
157
+            subnet_name='subnet_1',
158
+            network_id=net_1['id'],
159
+            cidr='192.168.77.0/24')
160
+
161
+        default_router = os_conn.get_router(os_conn.get_network(
162
+            self.default.ADMIN_NET))
163
+        os_conn.add_router_interface(router_id=default_router['id'],
164
+                                     subnet_id=subnet_1['id'])
165
+
166
+        # Create vCenter VM with new network and check ICMP connectivity
167
+        self.show_step(7)
168
+
169
+        vms=[]
170
+        vms.append(os_conn.create_server(
171
+            net_id=net_1['id'],
172
+            security_groups=[sec_group]))
173
+        vms.append(os_conn.create_server(
174
+            availability_zone=self.vcenter_az,
175
+            image=image,
176
+            net_id=net_1['id'],
177
+            security_groups=[sec_group]))
178
+
179
+        ips = []
180
+        for vm in vms:
181
+            floating = os_conn.assign_floating_ip(vm)
182
+            wait(lambda: tcp_ping(floating.ip, 22),
183
+                 timeout=180,
184
+                 timeout_msg="Node {ip} is not accessible by SSH.".format(
185
+                     ip=floating.ip))
186
+            ips.append(floating.ip)
187
+
188
+        for ip in ips:
189
+            logger.info('Check connectivity from {0}'.format(ip))
190
+            assert_true(self.ping_from_instance(ip,
191
+                                                '8.8.8.8',
192
+                                                vip_contr_new),
193
+                        'Ping failed')

Loading…
Cancel
Save