Browse Source

Add galera tests

- Add tests on galera deployment, galera node reboot, shutdown,
poweroff, cold reboot, cluster shutdown
- Add test on scale up and down galera
- Galera checkers will be added after merging patch
https://review.openstack.org/#/c/399999/8/service/files/galera_checker.py

Change-Id: I31e1fe214e558497bcec7661f9e204ee645a56c1
asledzinskiy 2 years ago
parent
commit
8457ee9280

+ 50
- 0
fuel_ccp_tests/fixtures/os_fixtures.py View File

@@ -12,10 +12,14 @@
12 12
 #    License for the specific language governing permissions and limitations
13 13
 #    under the License.
14 14
 
15
+from copy import deepcopy
16
+import os
15 17
 import pytest
16 18
 
17 19
 from fuel_ccp_tests import logger
20
+from fuel_ccp_tests import settings
18 21
 from fuel_ccp_tests.helpers import ext
22
+from fuel_ccp_tests.helpers import post_os_deploy_checks
19 23
 from fuel_ccp_tests.managers.osmanager import OSManager
20 24
 
21 25
 LOG = logger.logger
@@ -43,3 +47,49 @@ def os_deployed(ccpcluster,
43 47
     else:
44 48
         LOG.info("Openstack allready installed and running...")
45 49
         osmanager.check_os_ready()
50
+
51
+
52
+@pytest.mark.revert_snapshot(ext.SNAPSHOT.os_galera_deployed)
53
+@pytest.fixture(scope='function')
54
+def galera_deployed(ccpcluster,
55
+                    hardware,
56
+                    underlay,
57
+                    revert_snapshot,
58
+                    config,
59
+                    k8s_actions):
60
+    """Deploy galera cluster
61
+    """
62
+    # If no snapshot was reverted, then try to revert the snapshot
63
+    # that belongs to the fixture.
64
+    # Note: keep fixtures in strict dependences from each other!
65
+    if not config.os.running:
66
+        general_config = deepcopy(settings.CCP_CONF)
67
+        if settings.BUILD_IMAGES:
68
+            k8s_actions.create_registry()
69
+            ccpcluster.build()
70
+        topology_path = \
71
+            os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
72
+                          '3galera_1comp.yaml'
73
+        remote = underlay.remote(host=config.k8s.kube_host)
74
+        remote.upload(topology_path, '/tmp')
75
+        ccpcluster.put_yaml_config('./config_1.yaml', general_config)
76
+        ccpcluster.add_includes('./config_1.yaml', [
77
+            settings.CCP_DEPLOY_CONFIG,
78
+            settings.CCP_SOURCES_CONFIG,
79
+            '/tmp/3galera_1comp.yaml'])
80
+
81
+        underlay.sudo_check_call("pip install python-openstackclient",
82
+                                 host=config.k8s.kube_host)
83
+        ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
84
+                          use_cli_params=True)
85
+        post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
86
+        post_os_deploy_checks.check_pods_status(k8s_actions.api)
87
+        # todo: add invocation of galera checker script
88
+        remote.check_call(
89
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
90
+            " create".format(
91
+                settings.CCP_CONF["kubernetes"]["namespace"]),
92
+            timeout=600)
93
+
94
+        config.os.running = True
95
+        hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed)

+ 2
- 1
fuel_ccp_tests/helpers/ext.py View File

@@ -42,7 +42,8 @@ SNAPSHOT = enum(
42 42
     'k8s_deployed',
43 43
     'ccp_deployed',
44 44
     'os_deployed',
45
-    'os_deployed_stacklight'
45
+    'os_deployed_stacklight',
46
+    'os_galera_deployed',
46 47
 )
47 48
 
48 49
 LOG_LEVELS = enum(

+ 2
- 2
fuel_ccp_tests/managers/ccpmanager.py View File

@@ -198,10 +198,10 @@ class CCPManager(object):
198 198
                      components=components,
199 199
                      params=params, suppress_output=suppress_output)
200 200
 
201
-    def deploy(self, components=None, params=None):
201
+    def deploy(self, components=None, params=None, use_cli_params=False):
202 202
         self.run('deploy',
203 203
                  components=components,
204
-                 params=params)
204
+                 params=params, use_cli_params=use_cli_params)
205 205
 
206 206
     def dry_deploy(self, export_dir, components=None, params=None):
207 207
         self.run('deploy --dry-run --export-dir={export_dir}'.format(

+ 44
- 0
fuel_ccp_tests/managers/envmanager_devops.py View File

@@ -470,3 +470,47 @@ class EnvironmentManager(object):
470 470
             self.__config.underlay.nameservers = [self.nameserver]
471 471
         if not self.__config.underlay.upstream_dns_servers:
472 472
             self.__config.underlay.upstream_dns_servers = [self.nameserver]
473
+
474
+    def get_node_by_ip(self, node_ip):
475
+        nodes = self._env.get_nodes()
476
+        node = [node for node in nodes if self.node_ip(node) == node_ip]
477
+        assert len(node) == 1, "Node with {} ip isn't found".format(node_ip)
478
+        return node[0]
479
+
480
+    def shutdown_node_by_ip(self, node_ip):
481
+        """Shutdown hardware node by ip address
482
+
483
+        """
484
+        node = self.get_node_by_ip(node_ip)
485
+        node.shutdown()
486
+
487
+    def start_node_by_ip(self, node_ip):
488
+        """Start hardware node by ip address
489
+
490
+        """
491
+        node = self.get_node_by_ip(node_ip)
492
+        node.start()
493
+
494
+    def wait_node_is_offline(self, node_ip, timeout):
495
+        """Wait node is shutdown and doesn't respond
496
+
497
+        """
498
+        helpers.wait(
499
+            lambda: not helpers.tcp_ping(node_ip, 22),
500
+            timeout=timeout,
501
+            timeout_msg="Node '{}' didn't go offline after {} sec".format(
502
+                node_ip, timeout
503
+            )
504
+        )
505
+
506
+    def wait_node_is_online(self, node_ip, timeout):
507
+        """Wait node is online after starting
508
+
509
+        """
510
+        helpers.wait(
511
+            lambda: helpers.tcp_ping(node_ip, 22),
512
+            timeout=timeout,
513
+            timeout_msg="Node '{}' didn't become online after {} sec".format(
514
+                node_ip, timeout
515
+            )
516
+        )

+ 5
- 0
fuel_ccp_tests/managers/k8smanager.py View File

@@ -343,3 +343,8 @@ class K8SManager(object):
343 343
                 self.__config.underlay.upstream_dns_servers
344 344
             LOG.info('Added custom upstream DNS servers (dnsmasq) to the '
345 345
                      'settings: {0}'.format(k8s_settings['nameservers']))
346
+
347
+    def get_pods_number(self, pod_name, namespace=None):
348
+        pods = [pod for pod in self.api.pods.list(namespace=namespace)
349
+                if pod_name in pod.name]
350
+        return len(pods)

+ 41
- 0
fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml View File

@@ -0,0 +1,41 @@
1
+nodes:
2
+  node1:
3
+    roles:
4
+      - openvswitch
5
+      - controller-net-host
6
+      - controller-net-bridge
7
+  node[2-3]:
8
+    roles:
9
+      - openvswitch
10
+      - controller-net-bridge
11
+      - compute
12
+  node[1-3]:
13
+    roles:
14
+      - galera
15
+roles:
16
+  controller-net-host:
17
+    - neutron-dhcp-agent
18
+    - neutron-l3-agent
19
+    - neutron-metadata-agent
20
+  controller-net-bridge:
21
+    - etcd
22
+    - glance-api
23
+    - glance-registry
24
+    - keystone
25
+    - memcached
26
+    - neutron-server
27
+    - nova-api
28
+    - nova-conductor
29
+    - nova-consoleauth
30
+    - nova-novncproxy
31
+    - nova-scheduler
32
+    - rabbitmq
33
+  compute:
34
+    - nova-compute
35
+    - nova-libvirt
36
+  openvswitch:
37
+    - neutron-openvswitch-agent
38
+    - openvswitch-db
39
+    - openvswitch-vswitchd
40
+  galera:
41
+    - galera

+ 261
- 0
fuel_ccp_tests/tests/system/test_galera.py View File

@@ -0,0 +1,261 @@
1
+#    Copyright 2016 Mirantis, Inc.
2
+#
3
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+#    not use this file except in compliance with the License. You may obtain
5
+#    a copy of the License at
6
+#
7
+#         http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+#    Unless required by applicable law or agreed to in writing, software
10
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+#    License for the specific language governing permissions and limitations
13
+#    under the License.
14
+
15
+import pytest
16
+
17
+import base_test
18
+from fuel_ccp_tests import logger
19
+from fuel_ccp_tests import settings
20
+from fuel_ccp_tests.helpers import post_os_deploy_checks
21
+
22
+LOG = logger.logger
23
+
24
+
25
+class TestGalera(base_test.SystemBaseTest):
26
+    """ Galera scale and destructive scenarios
27
+
28
+    """
29
+    @pytest.mark.fail_snapshot
30
+    @pytest.mark.galera_shutdown
31
+    @pytest.mark.galera
32
+    def test_galera_shutdown_node(self, hardware, underlay, config,
33
+                                  ccpcluster, k8s_actions, show_step,
34
+                                  galera_deployed):
35
+        """Shutdown galera node
36
+
37
+        Scenario:
38
+        1. Revert snapshot with deployed galera
39
+        2. Shutdown one galera node
40
+        3. Check galera state
41
+        4. Create 2 vms
42
+
43
+        Duration 30 min
44
+        """
45
+        show_step(2)
46
+        hardware.shutdown_node_by_ip(underlay.node_names()[1])
47
+        show_step(3)
48
+        # todo: add wait for galera to assemble when galera_checker is ready
49
+        remote = underlay.remote(host=config.k8s.kube_host)
50
+        show_step(4)
51
+        remote.check_call(
52
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
53
+            " create".format(
54
+                settings.CCP_CONF["kubernetes"]["namespace"]),
55
+            timeout=600)
56
+
57
+    @pytest.mark.fail_snapshot
58
+    @pytest.mark.galera_cold_restart
59
+    @pytest.mark.galera
60
+    def test_galera_cold_restart_node(self, hardware, underlay, config,
61
+                                      ccpcluster, k8s_actions, show_step,
62
+                                      galera_deployed):
63
+        """Cold restart galera node
64
+
65
+        Scenario:
66
+        1. Revert snapshot with deployed galera
67
+        2. Cold restart one galera node
68
+        3. Check galera state
69
+        4. Create 2 vms
70
+
71
+        Duration 30 min
72
+        """
73
+        show_step(2)
74
+        hardware.shutdown_node_by_ip(underlay.node_names()[1])
75
+        hardware.start_node_by_ip(underlay.host_by_node_name('slave-0'))
76
+        show_step(3)
77
+        # todo: add wait for galera to assemble when galera_checker is ready
78
+        remote = underlay.remote(host=config.k8s.kube_host)
79
+        show_step(4)
80
+        remote.check_call(
81
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
82
+            " create".format(
83
+                settings.CCP_CONF["kubernetes"]["namespace"]),
84
+            timeout=600)
85
+
86
+    @pytest.mark.fail_snapshot
87
+    @pytest.mark.galera_poweroff
88
+    @pytest.mark.galera
89
+    def test_galera_poweroff_node(self, hardware, underlay, config,
90
+                                  ccpcluster, k8s_actions, show_step,
91
+                                  galera_deployed):
92
+        """Poweroff galera node
93
+
94
+        Scenario:
95
+        1. Revert snapshot with deployed galera
96
+        2. Poweroff one galera node
97
+        3. Check galera state
98
+        4. Create 2 vms
99
+
100
+        Duration 30 min
101
+        """
102
+        galera_node = underlay.node_names()[1]
103
+        galera_node_ip = underlay.host_by_node_name(galera_node)
104
+        show_step(2)
105
+        underlay.sudo_check_call('shutdown +1', node_name=galera_node)
106
+        hardware.shutdown_node_by_ip(galera_node_ip)
107
+        hardware.wait_node_is_offline(galera_node_ip, 90)
108
+        show_step(3)
109
+        # todo: add wait for galera to assemble when galera_checker is ready
110
+        remote = underlay.remote(host=config.k8s.kube_host)
111
+        show_step(4)
112
+        remote.check_call(
113
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
114
+            " create".format(
115
+                settings.CCP_CONF["kubernetes"]["namespace"]),
116
+            timeout=600)
117
+
118
+    @pytest.mark.fail_snapshot
119
+    @pytest.mark.galera_soft_reboot
120
+    @pytest.mark.galera
121
+    def test_galera_soft_reboot_node(self, hardware, underlay, config,
122
+                                     ccpcluster, k8s_actions, show_step,
123
+                                     galera_deployed):
124
+        """Soft reboot galera node
125
+
126
+        Scenario:
127
+        1. Revert snapshot with deployed galera
128
+        2. Soft reboot one galera node
129
+        3. Check galera state
130
+        4. Create 2 vms
131
+
132
+        Duration 30 min
133
+        """
134
+        galera_node = underlay.node_names()[1]
135
+        galera_node_ip = underlay.host_by_node_name(galera_node)
136
+        show_step(2)
137
+        underlay.sudo_check_call('shutdown +1', node_name=galera_node)
138
+        hardware.shutdown_node_by_ip(galera_node_ip)
139
+        hardware.wait_node_is_offline(galera_node_ip, 90)
140
+        hardware.start_node_by_ip(galera_node_ip)
141
+        hardware.wait_node_is_online(galera_node_ip, 180)
142
+        show_step(3)
143
+        # todo: add wait for galera to assemble when galera_checker is ready
144
+        remote = underlay.remote(host=config.k8s.kube_host)
145
+        show_step(4)
146
+        remote.check_call(
147
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
148
+            " create".format(
149
+                settings.CCP_CONF["kubernetes"]["namespace"]),
150
+            timeout=600)
151
+
152
+    @pytest.mark.fail_snapshot
153
+    @pytest.mark.galera_cluster_shutdown
154
+    @pytest.mark.galera
155
+    def test_galera_cluster_shutdown(self, hardware, underlay, config,
156
+                                     ccpcluster, k8s_actions, show_step,
157
+                                     galera_deployed):
158
+        """Galera cluster shutdown
159
+
160
+        Scenario:
161
+        1. Revert snapshot with deployed galera
162
+        2. Shutdown all galera nodes and start them one by one
163
+        3. Check galera state
164
+        4. Create 2 vms
165
+
166
+        Duration 30 min
167
+        """
168
+        galera_nodes = underlay.node_names()[:3]
169
+        galera_node_ips = []
170
+        show_step(2)
171
+        for galera_node in galera_nodes:
172
+            galera_node_ip = underlay.host_by_node_name(galera_node)
173
+            galera_node_ips.append(galera_node_ip)
174
+            hardware.shutdown_node_by_ip(galera_node_ip)
175
+            hardware.wait_node_is_offline(galera_node_ip, 90)
176
+        for galera_ip in galera_node_ips:
177
+            hardware.start_node_by_ip(galera_ip)
178
+            hardware.wait_node_is_online(galera_ip, 180)
179
+        show_step(3)
180
+        # todo: add wait for galera to assemble when galera_checker is ready
181
+        remote = underlay.remote(host=config.k8s.kube_host)
182
+        show_step(4)
183
+        remote.check_call(
184
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
185
+            " create".format(
186
+                settings.CCP_CONF["kubernetes"]["namespace"]),
187
+            timeout=600)
188
+
189
+    @pytest.mark.fail_snapshot
190
+    @pytest.mark.galera_scale_up_down
191
+    @pytest.mark.galera
192
+    def test_galera_scale(self, hardware, underlay, config,
193
+                          ccpcluster, k8s_actions, show_step,
194
+                          galera_deployed):
195
+        """Galera cluster scale
196
+
197
+        Scenario:
198
+        1. Revert snapshot with deployed galera
199
+        2. Scale up galera to 5 replicas
200
+        3. Check galera state
201
+        4. Check number of galera pods
202
+        5. Create 2 vms
203
+        6. Scale down galera to 3 replicas
204
+        7. Check galera state
205
+        8. Check number of galera pods
206
+        9. Create 2 vms
207
+
208
+        Duration 30 min
209
+        """
210
+        show_step(2)
211
+        with underlay.yaml_editor('/tmp/3galera_1comp.yaml',
212
+                                  host=config.k8s.kube_host) as editor:
213
+            del editor.content['nodes']['node[1-3]']
214
+            editor.content['nodes']['node[1-5]'] = {'roles': ['galera']}
215
+
216
+        ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
217
+                          use_cli_params=True)
218
+        post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
219
+        post_os_deploy_checks.check_pods_status(k8s_actions.api)
220
+        show_step(3)
221
+        # todo: add invocation of galera checker script
222
+        show_step(4)
223
+        galera_pods = \
224
+            k8s_actions.get_pods_number('galera',
225
+                                        settings.
226
+                                        CCP_CONF['kubernetes']['namespace'])
227
+        assert galera_pods == 5,\
228
+            "Expcted tp have 5 galera pods, got {}".format(galera_pods)
229
+
230
+        show_step(5)
231
+        remote = underlay.remote(host=config.k8s.kube_host)
232
+        remote.check_call(
233
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
234
+            " create".format(
235
+                settings.CCP_CONF["kubernetes"]["namespace"]),
236
+            timeout=600)
237
+        show_step(6)
238
+        with underlay.yaml_editor('/tmp/3galera_1comp.yaml',
239
+                                  host=config.k8s.kube_host) as editor:
240
+            del editor.content['nodes']['node[1-5]']
241
+            editor.content['nodes']['node[1-3]'] = {'roles': ['galera']}
242
+
243
+        ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
244
+                          use_cli_params=True)
245
+        post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
246
+        post_os_deploy_checks.check_pods_status(k8s_actions.api)
247
+        show_step(7)
248
+        # todo: add invocation of galera checker script
249
+        show_step(8)
250
+        galera_pods = \
251
+            k8s_actions.get_pods_number('galera',
252
+                                        settings.
253
+                                        CCP_CONF['kubernetes']['namespace'])
254
+        assert galera_pods == 3,\
255
+            "Expcted tp have 3 galera pods, got {}".format(galera_pods)
256
+        show_step(9)
257
+        remote.check_call(
258
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
259
+            " create".format(
260
+                settings.CCP_CONF["kubernetes"]["namespace"]),
261
+            timeout=600)

Loading…
Cancel
Save