Browse Source

Add tests to check rabbit cluster

- Add client to work with rabbit cluster: create queue,
message publishing, check cluster state
- Add failover tests for rabbit nodes

Change-Id: Ifde0aac45a43460d9af15442381bf0843bfe072e
asledzinskiy 2 years ago
parent
commit
ca1ad32f20

+ 3
- 4
README.rst View File

@@ -12,7 +12,7 @@ fuel_ccp_tests
12 12
 ├── managers
13 13
 │   └── k8s
14 14
 ├── templates
15
-│   ├── k8s_templates
15
+│   ├── ccp_deploy_topology
16 16
 │   ├── misc
17 17
 │   └── registry_templates
18 18
 └── tests
@@ -47,7 +47,7 @@ Contains .yaml templates with environment configuration(virtual machines, networ
47 47
 
48 48
 Tests Overview
49 49
 ================
50
-The fuel-ccp-test are performed to verify that the completed software (ccp) functions according to the expectations defined by the requirements. 
50
+The fuel-ccp-test are performed to verify that the completed software (ccp) functions according to the expectations defined by the requirements.
51 51
 
52 52
 The tests depended on purpose are divided on several categories.
53 53
 
@@ -67,7 +67,7 @@ Consists from 2 categories **precommit** and **system**.  The purpose of the sys
67 67
 
68 68
 Test execution
69 69
 --------------
70
-To execute tests necessary to add value to several variables via *export* or in the test command. Variables: 
70
+To execute tests necessary to add value to several variables via *export* or in the test command. Variables:
71 71
 
72 72
 - ENV_NAME - prefix name for the env and VMs
73 73
 - IMAGE_PATH  - qcow2 image path
@@ -78,4 +78,3 @@ To execute tests necessary to add value to several variables via *export* or in
78 78
 After exporting execute the command:
79 79
 
80 80
 py.test -vvv -s -k <test_name> or py.test -vvv -s -m <test_mark>
81
-

+ 1
- 1
fuel_ccp_tests/fixtures/ccp_fixtures.py View File

@@ -76,7 +76,7 @@ def ccpcluster(revert_snapshot, config, hardware,
76 76
             path=settings.CCP_FETCH_CONFIG,
77 77
             config=settings.CCP_FETCH_PARAMS)
78 78
 
79
-        with open(settings.TOPOLOGY_PATH, 'r') as f:
79
+        with open(config.ccp_deploy.topology_path, 'r') as f:
80 80
             ccp_actions.put_raw_config(
81 81
                 path=settings.CCP_DEPLOY_TOPOLOGY,
82 82
                 content=f.read())

+ 18
- 7
fuel_ccp_tests/fixtures/os_fixtures.py View File

@@ -15,11 +15,13 @@
15 15
 from copy import deepcopy
16 16
 import os
17 17
 import pytest
18
+from devops.helpers import helpers
18 19
 
19 20
 from fuel_ccp_tests import logger
20 21
 from fuel_ccp_tests import settings
21 22
 from fuel_ccp_tests.helpers import ext
22 23
 from fuel_ccp_tests.helpers import post_os_deploy_checks
24
+from fuel_ccp_tests.helpers import rabbit
23 25
 from fuel_ccp_tests.managers.osmanager import OSManager
24 26
 
25 27
 LOG = logger.logger
@@ -37,12 +39,7 @@ def os_deployed(ccpcluster,
37 39
     """
38 40
     osmanager = OSManager(config, underlay, k8s_actions, ccpcluster)
39 41
     if not config.os.running:
40
-        LOG.info("Preparing openstack log collector fixture...")
41
-        topology = None
42
-        if config.os_deploy.stacklight_enable:
43
-            topology = ('/fuel_ccp_tests/templates/k8s_templates/'
44
-                        'stacklight_topology.yaml')
45
-        osmanager.install_os(topology=topology)
42
+        osmanager.install_os()
46 43
         hardware.create_snapshot(ext.SNAPSHOT.os_deployed)
47 44
     else:
48 45
         LOG.info("Openstack allready installed and running...")
@@ -68,7 +65,7 @@ def galera_deployed(ccpcluster,
68 65
             k8s_actions.create_registry()
69 66
             ccpcluster.build()
70 67
         topology_path = \
71
-            os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
68
+            os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
72 69
                           '3galera_1comp.yaml'
73 70
         remote = underlay.remote(host=config.k8s.kube_host)
74 71
         remote.upload(topology_path, '/tmp')
@@ -93,3 +90,17 @@ def galera_deployed(ccpcluster,
93 90
 
94 91
         config.os.running = True
95 92
         hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed)
93
+
94
+
95
+@pytest.fixture(scope='function')
96
+def rabbit_client(underlay, config, os_deployed):
97
+    """Deploy openstack
98
+    """
99
+    host = config.k8s.kube_host
100
+    remote = underlay.remote(host=host)
101
+    rabbit_port = ''.join(remote.execute(
102
+        "kubectl get service --namespace ccp rabbitmq -o yaml |"
103
+        " awk '/nodePort: / {print $NF}'")['stdout'])
104
+    client = helpers.wait_pass(lambda: rabbit.RabbitClient(host, rabbit_port),
105
+                               interval=60, timeout=360)
106
+    return client

+ 56
- 0
fuel_ccp_tests/helpers/rabbit.py View File

@@ -0,0 +1,56 @@
1
+import kombu
2
+
3
+from fuel_ccp_tests import logger
4
+from fuel_ccp_tests.helpers import utils
5
+
6
+LOG = logger.logger
7
+
8
+
9
+class RabbitClient(object):
10
+
11
+    def __init__(self, ip, port, user='rabbitmq', password='password'):
12
+        c = kombu.Connection("amqp://{0}:{1}@{2}:{3}//".format(user, password,
13
+                                                               ip, port))
14
+        c.connect()
15
+        self.ch = c.channel()
16
+
17
+    def list_nodes(self, remote, pod, namespace):
18
+        output = ''.join(
19
+            remote.execute("kubectl exec -i {} --namespace={}"
20
+                           " -- rabbitmqctl"
21
+                           " cluster_status".format(pod,
22
+                                                    namespace))['stdout'])
23
+        substring_ind = output.find('{running_nodes')
24
+        sub_end_ind = output.find('cluster_name')
25
+        result_str = output[substring_ind: sub_end_ind]
26
+        num_node = result_str.count("rabbit@")
27
+        return num_node
28
+
29
+    def check_queue_replicated(self, queue, remote, pod, namespace):
30
+        remote.check_call("kubectl exec -i {} --namespace={}"
31
+                          " -- rabbitmqctl list_queues |"
32
+                          " grep {}".format(pod, namespace,
33
+                                            queue))
34
+
35
+    def create_queue(self):
36
+        test_queue = 'test-rabbit-{}'.format(utils.rand_name())
37
+        q = kombu.Queue(test_queue, channel=self.ch, durable=False,
38
+                        queue_arguments={"x-expires": 15 * 60 * 1000})
39
+        q.declare()
40
+        return test_queue
41
+
42
+    def publish_message_to_queue(self, queue):
43
+        uid = utils.generate_uuid()
44
+        producer = kombu.Producer(channel=self.ch, routing_key=queue)
45
+        producer.publish(uid)
46
+        return {'queue': queue, 'id': uid}
47
+
48
+    def check_queue_message(self, message):
49
+        q = kombu.Queue(message['queue'], channel=self.ch)
50
+        msg = q.get(True)
51
+        assert msg.body in message['id'],\
52
+            "Message body is {}, expected {}".format(msg.body, message['id'])
53
+
54
+    def delete_queue(self, queue):
55
+        q = kombu.Queue(queue, channel=self.ch)
56
+        q.delete()

+ 10
- 0
fuel_ccp_tests/helpers/utils.py View File

@@ -14,10 +14,12 @@
14 14
 
15 15
 import copy
16 16
 import os
17
+import random
17 18
 import shutil
18 19
 import tempfile
19 20
 import time
20 21
 import traceback
22
+import uuid
21 23
 
22 24
 import paramiko
23 25
 import yaml
@@ -483,3 +485,11 @@ def get_top_fixtures_marks(request, mark_name):
483 485
               .format(top_fixtures_marks))
484 486
 
485 487
     return top_fixtures_marks
488
+
489
+
490
+def rand_name():
491
+    return str(random.randint(1, 0x7fffffff))
492
+
493
+
494
+def generate_uuid():
495
+    return uuid.uuid4().hex

+ 6
- 2
fuel_ccp_tests/managers/k8smanager.py View File

@@ -344,7 +344,11 @@ class K8SManager(object):
344 344
             LOG.info('Added custom upstream DNS servers (dnsmasq) to the '
345 345
                      'settings: {0}'.format(k8s_settings['nameservers']))
346 346
 
347
-    def get_pods_number(self, pod_name, namespace=None):
347
+    def get_running_pods(self, pod_name, namespace=None):
348 348
         pods = [pod for pod in self.api.pods.list(namespace=namespace)
349
-                if pod_name in pod.name]
349
+                if pod_name in pod.name and pod.status.phase == 'Running']
350
+        return pods
351
+
352
+    def get_pods_number(self, pod_name, namespace=None):
353
+        pods = self.get_running_pods(pod_name, namespace)
350 354
         return len(pods)

+ 4
- 16
fuel_ccp_tests/managers/osmanager.py View File

@@ -11,7 +11,6 @@
11 11
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 12
 #    License for the specific language governing permissions and limitations
13 13
 #    under the License.
14
-import os
15 14
 
16 15
 from fuel_ccp_tests import logger
17 16
 from fuel_ccp_tests import settings
@@ -32,8 +31,7 @@ class OSManager(object):
32 31
         self.__k8s_actions = k8s_actions
33 32
         self.__ccpcluster = ccpcluster
34 33
 
35
-    def install_os(self, topology=None,
36
-                   check_os_ready=True):
34
+    def install_os(self, check_os_ready=True):
37 35
         """Action to deploy openstack by ccp tool
38 36
 
39 37
         Additional steps:
@@ -47,7 +45,7 @@ class OSManager(object):
47 45
         LOG.info("Trying to install k8s")
48 46
 
49 47
         """
50
-        Deploy openstack with stacklight topology
48
+        Deploy openstack with provided topology
51 49
         """
52 50
         LOG.info("Preparing openstack log collector fixture...")
53 51
         if settings.REGISTRY == "127.0.0.1:31500":
@@ -55,16 +53,6 @@ class OSManager(object):
55 53
             self.__k8s_actions.create_registry()
56 54
             LOG.info("Building images...")
57 55
             self.__ccpcluster.build()
58
-        if topology:
59
-            LOG.info("Pushing topology yaml...")
60
-            LOG.warn(
61
-                "Patched topology used, workaround until kube 1.4 released")
62
-            topology_path = \
63
-                os.getcwd() + topology
64
-            self.__underlay.remote(
65
-                host=self.__config.k8s.kube_host).upload(
66
-                topology_path,
67
-                settings.DEPLOY_CONFIG)
68 56
         LOG.info("Deploy openstack")
69 57
         self.__ccpcluster.deploy()
70 58
         if check_os_ready:
@@ -77,8 +65,8 @@ class OSManager(object):
77 65
         if check_jobs_ready:
78 66
             LOG.info("Checking openstack jobs statuses...")
79 67
             post_os_deploy_checks.check_jobs_status(self.__k8s_actions.api,
80
-                                                    timeout=3600)
68
+                                                    timeout=4500)
81 69
         if check_pods_ready:
82 70
             LOG.info("Checking openstack pods statuses...")
83 71
             post_os_deploy_checks.check_pods_status(self.__k8s_actions.api,
84
-                                                    timeout=3600)
72
+                                                    timeout=4500)

+ 1
- 0
fuel_ccp_tests/requirements.txt View File

@@ -11,3 +11,4 @@ psycopg2
11 11
 python-k8sclient==0.3.0
12 12
 junit-xml
13 13
 elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0
14
+kombu>=3.0.25  # BSD

+ 2
- 2
fuel_ccp_tests/settings.py View File

@@ -123,8 +123,8 @@ CCP_DEPLOY_CONFIG = '~/.ccp.deploy-config.yaml'
123 123
 CCP_DEPLOY_TOPOLOGY = '~/.ccp.deploy-topology.yaml'
124 124
 TOPOLOGY_PATH = os.environ.get('TOPOLOGY_PATH',
125 125
                                os.getcwd() + '/fuel_ccp_tests/templates/'
126
-                               'k8s_templates/k8s_topology.yaml')
127
-
126
+                               'ccp_deploy_topology/'
127
+                               'default_deploy_topology.yaml')
128 128
 
129 129
 FUEL_CCP_KEYSTONE_LOCAL_REPO = os.environ.get('FUEL_CCP_KEYSTONE_LOCAL_REPO',
130 130
                                               None)

+ 6
- 1
fuel_ccp_tests/settings_oslo.py View File

@@ -26,6 +26,9 @@ from fuel_ccp_tests import settings
26 26
 _default_conf = pkg_resources.resource_filename(
27 27
     __name__, 'templates/default.yaml')
28 28
 
29
+_default_topology = pkg_resources.resource_filename(
30
+    __name__, 'templates/ccp_deploy_topology/default_deploy_topology.yaml')
31
+
29 32
 
30 33
 hardware_opts = [
31 34
     ct.Cfg('manager', ct.String(),
@@ -103,7 +106,9 @@ ccp_deploy_opts = [
103 106
     ct.Cfg('ccp_globals', ct.JSONDict(),
104 107
            help="", default=None),
105 108
     ct.Cfg('ccp_params', ct.JSONDict(),
106
-           help="", default=None)
109
+           help="", default=None),
110
+    ct.Cfg('topology_path', ct.String(),
111
+           help="", default=_default_topology),
107 112
 ]
108 113
 
109 114
 # Access credentials to a ready CCP

fuel_ccp_tests/templates/k8s_templates/1ctrl.yaml → fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl.yaml View File


fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp.yaml → fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp.yaml View File


fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp_diff_ctrl.yaml → fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp_diff_ctrl.yaml View File


fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp_same_ctrl.yaml → fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp_same_ctrl.yaml View File


fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml → fuel_ccp_tests/templates/ccp_deploy_topology/3galera_1comp.yaml View File


fuel_ccp_tests/templates/k8s_templates/k8s_topology.yaml → fuel_ccp_tests/templates/ccp_deploy_topology/default_deploy_topology.yaml View File

@@ -7,6 +7,9 @@ nodes:
7 7
     roles:
8 8
       - compute
9 9
       - openvswitch
10
+  node[1-3]:
11
+    roles:
12
+      - rabbitmq
10 13
 roles:
11 14
   controller:
12 15
     - etcd
@@ -27,7 +30,6 @@ roles:
27 30
     - nova-consoleauth
28 31
     - nova-novncproxy
29 32
     - nova-scheduler
30
-    - rabbitmq
31 33
   compute:
32 34
     - nova-compute
33 35
     - nova-libvirt
@@ -35,3 +37,5 @@ roles:
35 37
     - neutron-openvswitch-agent
36 38
     - openvswitch-db
37 39
     - openvswitch-vswitchd
40
+  rabbitmq:
41
+    - rabbitmq

fuel_ccp_tests/templates/k8s_templates/stacklight_topology.yaml → fuel_ccp_tests/templates/ccp_deploy_topology/stacklight_topology.yaml View File


+ 0
- 17
fuel_ccp_tests/templates/k8s_templates/build-deploy_cluster.yaml View File

@@ -1,17 +0,0 @@
1
-ccp-microservices-options:
2
-- images-base-distro: debian
3
-- images-maintainer: mos-microservices@mirantis.com
4
-- repositories-protocol: https
5
-- repositories-port: 443
6
-- builder-push
7
-- registry-address: {registry_address}
8
-- logfile: /var/log/microservices.log
9
-- verbose
10
-- debug
11
-- builder-workers: 1
12
-- registry-insecure
13
-- images-tag: {images_tag}
14
-- deploy-config: {deploy_config}
15
-- images-namespace: {images_namespace}
16
-dry_run_options:
17
-  export_dir: {export_dir}

+ 2
- 1
fuel_ccp_tests/tests/system/pre_commit/test_glance_api.py View File

@@ -51,7 +51,8 @@ class TestServiceGlance(object):
51 51
             ccpcluster.build()
52 52
 
53 53
         topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
54
-                                      'k8s_templates/k8s_topology.yaml'
54
+                                      'ccp_deploy_topology/' \
55
+                                      'default_deploy_topology.yaml'
55 56
         remote.upload(topology_path, './')
56 57
         with remote.get_sudo(remote):
57 58
             ccpcluster.deploy()

+ 2
- 1
fuel_ccp_tests/tests/system/pre_commit/test_horizon.py View File

@@ -63,7 +63,8 @@ class TestServiceHorizon(object):
63 63
             ccpcluster.build()
64 64
 
65 65
         topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
66
-                                      'k8s_templates/k8s_topology.yaml'
66
+                                      'ccp_deploy_topology/' \
67
+                                      'default_deploy_topology.yaml'
67 68
         remote.upload(topology_path, settings.CCP_CLI_PARAMS['deploy-config'])
68 69
         ccpcluster.deploy()
69 70
         post_os_deploy_checks.check_jobs_status(k8sclient, timeout=1500,

+ 2
- 1
fuel_ccp_tests/tests/system/pre_commit/test_stacklight.py View File

@@ -64,7 +64,8 @@ class TestPreStackLight(object):
64 64
             ccpcluster.build(suppress_output=False)
65 65
 
66 66
         topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
67
-                                      'k8s_templates/stacklight_topology.yaml'
67
+                                      'ccp_deploy_topology/' \
68
+                                      'stacklight_topology.yaml'
68 69
 
69 70
         remote.upload(topology_path, settings.CCP_CLI_PARAMS['deploy-config'])
70 71
         ccpcluster.deploy()

+ 5
- 5
fuel_ccp_tests/tests/system/test_few_os.py View File

@@ -49,7 +49,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
49 49
             k8s_actions.create_registry()
50 50
             ccpcluster.build()
51 51
         topology_path = \
52
-            os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
52
+            os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
53 53
                           '1ctrl_1comp.yaml'
54 54
         remote = underlay.remote(host=config.k8s.kube_host)
55 55
         remote.upload(topology_path, '/tmp')
@@ -71,7 +71,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
71 71
             timeout=600)
72 72
 
73 73
         topology_path = \
74
-            os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
74
+            os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
75 75
                           '1ctrl_1comp_diff.yaml'
76 76
         remote.upload(topology_path, '/tmp')
77 77
         conf = copy.deepcopy(settings.CCP_CONF)
@@ -117,7 +117,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
117 117
             k8s_actions.create_registry()
118 118
             ccpcluster.build()
119 119
         topology_path = \
120
-            os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
120
+            os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
121 121
                           '1ctrl_1comp.yaml'
122 122
         remote = underlay.remote(host=config.k8s.kube_host)
123 123
         remote.upload(topology_path, '/tmp')
@@ -139,7 +139,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
139 139
             timeout=600)
140 140
 
141 141
         topology_path = \
142
-            os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
142
+            os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
143 143
                           '1ctrl_1comp_same.yaml'
144 144
         remote.upload(topology_path, '/tmp')
145 145
         conf = copy.deepcopy(settings.CCP_CONF)
@@ -186,7 +186,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
186 186
             k8s_actions.create_registry()
187 187
             ccpcluster.build()
188 188
         topology_path = \
189
-            os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
189
+            os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
190 190
                           '1ctrl.yaml'
191 191
         remote = underlay.remote(host=config.k8s.kube_host)
192 192
         remote.upload(topology_path, '/tmp')

+ 487
- 0
fuel_ccp_tests/tests/system/test_rabbitmq.py View File

@@ -0,0 +1,487 @@
1
+#    Copyright 2016 Mirantis, Inc.
2
+#
3
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+#    not use this file except in compliance with the License. You may obtain
5
+#    a copy of the License at
6
+#
7
+#         http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+#    Unless required by applicable law or agreed to in writing, software
10
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+#    License for the specific language governing permissions and limitations
13
+#    under the License.
14
+
15
+import pytest
16
+import time
17
+from devops.helpers import helpers
18
+
19
+import base_test
20
+from fuel_ccp_tests import logger
21
+from fuel_ccp_tests import settings
22
+from fuel_ccp_tests.helpers import post_os_deploy_checks
23
+from fuel_ccp_tests.helpers import rabbit
24
+
25
+LOG = logger.logger
26
+
27
+
28
+class TestRabbitmq(base_test.SystemBaseTest):
29
+    """ Galera scale and destructive scenarios
30
+
31
+    """
32
+    def get_pods(self, k8s):
33
+        return \
34
+            k8s.get_running_pods('rabbit',
35
+                                 settings.CCP_CONF['kubernetes']['namespace'])
36
+
37
+    @pytest.mark.fail_snapshot
38
+    @pytest.mark.rabbitmq_deploy
39
+    @pytest.mark.rabbitmq
40
+    def test_rabbitmq(self, underlay, config,
41
+                      k8s_actions, show_step,
42
+                      os_deployed, rabbit_client):
43
+        """Deploy rabbit cluster
44
+
45
+        Scenario:
46
+        1. Revert snapshot with deployed rabbit
47
+        2. Check rabbit cluster state
48
+        3. Check queue, messages replication
49
+
50
+        Duration 30 min
51
+        """
52
+        show_step(2)
53
+        show_step(3)
54
+        namespace = settings.CCP_CONF["kubernetes"]["namespace"]
55
+        remote = underlay.remote(host=config.k8s.kube_host)
56
+        rabbit_pods = self.get_pods(k8s_actions)
57
+        queue = rabbit_client.create_queue()
58
+        message = rabbit_client.publish_message_to_queue(queue)
59
+        rabbit_client.check_queue_message(message)
60
+        for pod in rabbit_pods:
61
+            rabbit_cluster_nodes = rabbit_client.list_nodes(
62
+                remote, pod.name, namespace)
63
+            assert rabbit_cluster_nodes == len(rabbit_pods),\
64
+                "Expected to have {} nodes in cluster," \
65
+                " got {}".format(len(rabbit_pods), rabbit_cluster_nodes)
66
+            rabbit_client.check_queue_replicated(queue, remote,
67
+                                                 pod.name, namespace)
68
+        rabbit_client.delete_queue(queue)
69
+
70
+    @pytest.mark.fail_snapshot
71
+    @pytest.mark.rabbitmq_shutdown
72
+    @pytest.mark.rabbitmq
73
+    def test_rabbitmq_shutdown_node(self, hardware, underlay, config,
74
+                                    ccpcluster, k8s_actions, show_step,
75
+                                    os_deployed, rabbit_client):
76
+        """Shutdown rabbitmq node
77
+
78
+        Scenario:
79
+        1. Revert snapshot with deployed rabbit
80
+        2. Shutdown one rabbit node
81
+        3. Check rabbit cluster state
82
+        4. Check queue, messages replication
83
+        5. Create 2 vms
84
+
85
+        Duration 30 min
86
+        """
87
+        rabbit_node = underlay.node_names()[1]
88
+        rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
89
+        namespace = settings.CCP_CONF["kubernetes"]["namespace"]
90
+        rabbit_pods = self.get_pods(k8s_actions)
91
+        show_step(2)
92
+        hardware.shutdown_node_by_ip(rabbit_node_ip)
93
+        show_step(3)
94
+
95
+        helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
96
+                              len(rabbit_pods) - 1),
97
+                     timeout=600,
98
+                     timeout_msg='Timeout waiting for rabbit pod'
99
+                                 ' to be terminated')
100
+        pods_after_shutdown = self.get_pods(k8s_actions)
101
+        remote = underlay.remote(host=config.k8s.kube_host)
102
+        show_step(4)
103
+        queue = rabbit_client.create_queue()
104
+        message = rabbit_client.publish_message_to_queue(queue)
105
+        rabbit_client.check_queue_message(message)
106
+        for pod in pods_after_shutdown:
107
+            rabbit_cluster_nodes = rabbit_client.list_nodes(
108
+                remote, pod.name, namespace)
109
+            assert rabbit_cluster_nodes == len(pods_after_shutdown),\
110
+                "Expected to have {} nodes in cluster," \
111
+                " got {}".format(len(pods_after_shutdown),
112
+                                 rabbit_cluster_nodes)
113
+            rabbit_client.check_queue_replicated(queue, remote,
114
+                                                 pod.name, namespace)
115
+        rabbit_client.delete_queue(queue)
116
+        show_step(5)
117
+        remote.check_call(
118
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
119
+            " create".format(namespace), timeout=600)
120
+
121
+    @pytest.mark.fail_snapshot
122
+    @pytest.mark.rabbitmq_cold_restart
123
+    @pytest.mark.rabbitmq
124
+    def test_rabbitmq_cold_restart_node(self, hardware, underlay, config,
125
+                                        ccpcluster, k8s_actions, show_step,
126
+                                        os_deployed, rabbit_client):
127
+        """Cold restart rabbitmq node
128
+
129
+        Scenario:
130
+        1. Revert snapshot with deployed rabbit
131
+        2. Cold restart one rabbit node
132
+        3. Check rabbit cluster state
133
+        4. Check queue, messages replication
134
+        5. Create 2 vms
135
+
136
+        Duration 30 min
137
+        """
138
+        rabbit_node = underlay.node_names()[1]
139
+        rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
140
+        namespace = settings.CCP_CONF["kubernetes"]["namespace"]
141
+        rabbit_pods = self.get_pods(k8s_actions)
142
+
143
+        show_step(2)
144
+        hardware.shutdown_node_by_ip(rabbit_node_ip)
145
+        hardware.wait_node_is_offline(rabbit_node_ip, 90)
146
+        time.sleep(15)
147
+        hardware.start_node_by_ip(rabbit_node_ip)
148
+        hardware.wait_node_is_online(rabbit_node_ip, 180)
149
+        show_step(3)
150
+
151
+        helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
152
+                              len(rabbit_pods) - 1),
153
+                     timeout=1200,
154
+                     timeout_msg='Expected to have one pod destroyed'
155
+                                 ' after reboot')
156
+
157
+        helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
158
+                              len(rabbit_pods)),
159
+                     timeout=1200,
160
+                     timeout_msg='Expected pod to come back after reboot')
161
+
162
+        pods_after_reboot = self.get_pods(k8s_actions)
163
+        remote = underlay.remote(host=config.k8s.kube_host)
164
+        show_step(4)
165
+        queue = rabbit_client.create_queue()
166
+        message = rabbit_client.publish_message_to_queue(queue)
167
+        rabbit_client.check_queue_message(message)
168
+        for pod in pods_after_reboot:
169
+            rabbit_cluster_nodes = rabbit_client.list_nodes(
170
+                remote, pod.name, namespace)
171
+            assert rabbit_cluster_nodes == len(pods_after_reboot),\
172
+                "Expected to have {} nodes in cluster," \
173
+                " got {}".format(len(pods_after_reboot), rabbit_cluster_nodes)
174
+            rabbit_client.check_queue_replicated(queue, remote,
175
+                                                 pod.name, namespace)
176
+        rabbit_client.delete_queue(queue)
177
+        show_step(5)
178
+        remote.check_call(
179
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
180
+            " create".format(namespace), timeout=600)
181
+
182
+    @pytest.mark.fail_snapshot
183
+    @pytest.mark.rabbitmq_poweroff
184
+    @pytest.mark.rabbitmq
185
+    def test_rabbitmq_poweroff_node(self, hardware, underlay, config,
186
+                                    ccpcluster, k8s_actions, show_step,
187
+                                    os_deployed, rabbit_client):
188
+        """Poweroff rabbit node
189
+
190
+        Scenario:
191
+        1. Revert snapshot with deployed rabbit
192
+        2. Poweroff one rabbit node
193
+        3. Check rabbit cluster state
194
+        4. Check queue, messages replication
195
+        5. Create 2 vms
196
+
197
+        Duration 30 min
198
+        """
199
+        rabbit_node = underlay.node_names()[1]
200
+        rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
201
+        namespace = settings.CCP_CONF["kubernetes"]["namespace"]
202
+        rabbit_pods = self.get_pods(k8s_actions)
203
+        show_step(2)
204
+        underlay.sudo_check_call('shutdown +1', node_name=rabbit_node)
205
+        hardware.shutdown_node_by_ip(rabbit_node_ip)
206
+        hardware.wait_node_is_offline(rabbit_node_ip, 90)
207
+        show_step(3)
208
+
209
+        helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
210
+                              len(rabbit_pods) - 1),
211
+                     timeout=600,
212
+                     timeout_msg='Timeout waiting for rabbit pod'
213
+                                 ' to be terminated')
214
+        pods_after_reboot = self.get_pods(k8s_actions)
215
+        remote = underlay.remote(host=config.k8s.kube_host)
216
+        show_step(4)
217
+        queue = rabbit_client.create_queue()
218
+        message = rabbit_client.publish_message_to_queue(queue)
219
+        rabbit_client.check_queue_message(message)
220
+        for pod in pods_after_reboot:
221
+            rabbit_cluster_nodes = rabbit_client.list_nodes(
222
+                remote, pod.name, namespace)
223
+            assert rabbit_cluster_nodes == len(pods_after_reboot),\
224
+                "Expected to have {} nodes in cluster," \
225
+                " got {}".format(len(pods_after_reboot), rabbit_cluster_nodes)
226
+            rabbit_client.check_queue_replicated(queue, remote,
227
+                                                 pod.name, namespace)
228
+        rabbit_client.delete_queue(queue)
229
+        show_step(5)
230
+        remote.check_call(
231
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
232
+            " create".format(namespace), timeout=600)
233
+
234
+    @pytest.mark.fail_snapshot
235
+    @pytest.mark.rabbitmq_soft_reboot
236
+    @pytest.mark.rabbitmq
237
+    def test_rabbitmq_soft_reboot_node(self, hardware, underlay, config,
238
+                                       ccpcluster, k8s_actions, show_step,
239
+                                       os_deployed, rabbit_client):
240
+        """Soft reboot rabbitmq node
241
+
242
+        Scenario:
243
+        1. Revert snapshot with deployed rabbit
244
+        2. Reboot one rabbit node
245
+        3. Check rabbit cluster state
246
+        4. Check queue, messages replication
247
+        5. Create 2 vms
248
+
249
+        Duration 30 min
250
+        """
251
+        rabbit_node = underlay.node_names()[1]
252
+        rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
253
+        namespace = settings.CCP_CONF["kubernetes"]["namespace"]
254
+        rabbit_pods = self.get_pods(k8s_actions)
255
+        show_step(2)
256
+        underlay.sudo_check_call('shutdown +1', node_name=rabbit_node)
257
+        hardware.shutdown_node_by_ip(rabbit_node_ip)
258
+        hardware.wait_node_is_offline(rabbit_node_ip, 90)
259
+        time.sleep(15)
260
+        hardware.start_node_by_ip(rabbit_node_ip)
261
+        hardware.wait_node_is_online(rabbit_node_ip, 180)
262
+        show_step(3)
263
+        helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
264
+                              len(rabbit_pods)),
265
+                     timeout=600,
266
+                     timeout_msg='Timeout waiting for rabbit pod'
267
+                                 ' to be terminated')
268
+        pods_after_reboot = self.get_pods(k8s_actions)
269
+        remote = underlay.remote(host=config.k8s.kube_host)
270
+        show_step(4)
271
+        queue = rabbit_client.create_queue()
272
+        message = rabbit_client.publish_message_to_queue(queue)
273
+        rabbit_client.check_queue_message(message)
274
+        for pod in pods_after_reboot:
275
+            rabbit_cluster_nodes = rabbit_client.list_nodes(
276
+                remote, pod.name, namespace)
277
+            assert rabbit_cluster_nodes == len(pods_after_reboot),\
278
+                "Expected to have {} nodes in cluster," \
279
+                " got {}".format(len(pods_after_reboot), rabbit_cluster_nodes)
280
+            rabbit_client.check_queue_replicated(queue, remote,
281
+                                                 pod.name, namespace)
282
+        rabbit_client.delete_queue(queue)
283
+        show_step(5)
284
+        remote.check_call(
285
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
286
+            " create".format(namespace), timeout=600)
287
+
288
+    @pytest.mark.fail_snapshot
289
+    @pytest.mark.rabbitmq_cluster_shutdown
290
+    @pytest.mark.rabbitmq
291
+    def test_rabbitmq_cluster_shutdown(self, hardware, underlay, config,
292
+                                       ccpcluster, k8s_actions, show_step,
293
+                                       os_deployed, rabbit_client):
294
+        """Rabbitmq cluster shutdown
295
+
296
+        Scenario:
297
+        1. Revert snapshot with deployed rabbit
298
+        2. Shutdown all rabbit nodes and start them one by one
299
+        3. Check rabbit cluster state
300
+        4. Check queue, messages replication
301
+        5. Create 2 vms
302
+
303
+        Duration 30 min
304
+        """
305
+        rabbit_nodes = underlay.node_names()[:3]
306
+        namespace = settings.CCP_CONF["kubernetes"]["namespace"]
307
+        rabbit_pods = self.get_pods(k8s_actions)
308
+        rabbit_node_ips = []
309
+        show_step(2)
310
+        for rabbit_node in rabbit_nodes:
311
+            rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
312
+            rabbit_node_ips.append(rabbit_node_ip)
313
+            hardware.shutdown_node_by_ip(rabbit_node_ip)
314
+            hardware.wait_node_is_offline(rabbit_node_ip, 90)
315
+        for rabbit_ip in rabbit_node_ips:
316
+            hardware.start_node_by_ip(rabbit_ip)
317
+            hardware.wait_node_is_online(rabbit_ip, 180)
318
+        show_step(3)
319
+        post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
320
+        post_os_deploy_checks.check_pods_status(k8s_actions.api)
321
+        pods_after_shutdown = self.get_pods(k8s_actions)
322
+        assert len(rabbit_pods) == len(pods_after_shutdown),\
323
+            "Different number of pods after shutdown, was {}," \
324
+            " now {}".format(len(rabbit_pods), len(pods_after_shutdown))
325
+        remote = underlay.remote(host=config.k8s.kube_host)
326
+        show_step(4)
327
+        host = config.k8s.kube_host
328
+        rabbit_port = ''.join(remote.execute(
329
+            "kubectl get service --namespace ccp rabbitmq -o yaml |"
330
+            " awk '/nodePort: / {print $NF}'")['stdout'])
331
+        rabbit_client = rabbit.RabbitClient(host, rabbit_port)
332
+        queue = rabbit_client.create_queue()
333
+        message = rabbit_client.publish_message_to_queue(queue)
334
+        rabbit_client.check_queue_message(message)
335
+        for pod in pods_after_shutdown:
336
+            rabbit_cluster_nodes = rabbit_client.list_nodes(
337
+                remote, pod.name, namespace)
338
+            assert rabbit_cluster_nodes == len(pods_after_shutdown),\
339
+                "Expected to have {} nodes in cluster," \
340
+                " got {}".format(len(pods_after_shutdown),
341
+                                 rabbit_cluster_nodes)
342
+            rabbit_client.check_queue_replicated(queue, remote,
343
+                                                 pod.name, namespace)
344
+        rabbit_client.delete_queue(queue)
345
+        remote.check_call(
346
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
347
+            " create".format(namespace), timeout=600)
348
+
349
+    @pytest.mark.fail_snapshot
350
+    @pytest.mark.rabbitmq_scale_up_down
351
+    @pytest.mark.rabbitmq
352
+    def test_rabbitmq_scale(self, hardware, underlay, config,
353
+                            ccpcluster, k8s_actions, show_step,
354
+                            os_deployed, rabbit_client):
355
+        """Rabbit cluster scale
356
+
357
+        Scenario:
358
+        1. Revert snapshot with deployed rabbit
359
+        2. Scale up rabbit to 5 replicas
360
+        3. Check rabbit state
361
+        4. Check number of rabbit pods
362
+        5. Create 2 vms
363
+        6. Scale down rabbit to 3 replicas
364
+        7. Check rabbit state
365
+        8. Check number of rabbit pods
366
+        9. Create 2 vms
367
+
368
+        Duration 30 min
369
+        """
370
+        show_step(2)
371
+        with underlay.yaml_editor(settings.CCP_DEPLOY_TOPOLOGY,
372
+                                  host=config.k8s.kube_host) as editor:
373
+            del editor.content['nodes']['node[1-3]']
374
+            editor.content['nodes']['node[1-5]'] = {'roles': ['rabbitmq']}
375
+
376
+        ccpcluster.deploy()
377
+        post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
378
+        post_os_deploy_checks.check_pods_status(k8s_actions.api)
379
+        namespace = settings.CCP_CONF["kubernetes"]["namespace"]
380
+        remote = underlay.remote(host=config.k8s.kube_host)
381
+        rabbit_pods = self.get_pods(k8s_actions)
382
+        queue = rabbit_client.create_queue()
383
+        message = rabbit_client.publish_message_to_queue(queue)
384
+        rabbit_client.check_queue_message(message)
385
+        for pod in rabbit_pods:
386
+            rabbit_cluster_nodes = rabbit_client.list_nodes(
387
+                remote, pod.name, namespace)
388
+            assert rabbit_cluster_nodes == len(rabbit_pods),\
389
+                "Expected to have {} nodes in cluster," \
390
+                " got {}".format(len(rabbit_pods), rabbit_cluster_nodes)
391
+            rabbit_client.check_queue_replicated(queue, remote,
392
+                                                 pod.name, namespace)
393
+        rabbit_client.delete_queue(queue)
394
+
395
+        show_step(4)
396
+        rabbit_pods = \
397
+            k8s_actions.get_pods_number('rabbit', namespace)
398
+        assert rabbit_pods == 5,\
399
+            "Expcted to have 5 rabbit pods, got {}".format(rabbit_pods)
400
+
401
+        show_step(5)
402
+        remote.check_call(
403
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
404
+            " create".format(namespace), timeout=600)
405
+        show_step(6)
406
+        with underlay.yaml_editor(settings.CCP_DEPLOY_TOPOLOGY,
407
+                                  host=config.k8s.kube_host) as editor:
408
+            del editor.content['nodes']['node[1-5]']
409
+            editor.content['nodes']['node[1-3]'] = {'roles': ['rabbitmq']}
410
+
411
+        ccpcluster.deploy()
412
+        post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
413
+        post_os_deploy_checks.check_pods_status(k8s_actions.api)
414
+        show_step(7)
415
+        show_step(8)
416
+        rabbit_pods = \
417
+            k8s_actions.get_pods_number('rabbit', namespace)
418
+        assert rabbit_pods == 3,\
419
+            "Expcted to have 3 rabbit pods, got {}".format(rabbit_pods)
420
+        show_step(9)
421
+        remote.check_call(
422
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
423
+            " create".format(namespace), timeout=600)
424
+
425
+    @pytest.mark.fail_snapshot
426
+    @pytest.mark.rabbitmq_node_replacement
427
+    @pytest.mark.rabbitmq
428
+    def test_rabbitmq_node_replacement(self, hardware, underlay, config,
429
+                                       ccpcluster, k8s_actions, show_step,
430
+                                       os_deployed, rabbit_client):
431
+        """Rabbitmq node replacement
432
+
433
+        Scenario:
434
+        1. Revert snapshot with deployed rabbit
435
+        2. Shutdown one rabbit node
436
+        3. Add new rabbit node to config
437
+        4. Re-deploy cluster
438
+        5. Check rabbit cluster state
439
+        6. Check queue, messages replication
440
+        7. Create 2 vms
441
+
442
+        Duration 30 min
443
+        """
444
+        rabbit_node = underlay.node_names()[1]
445
+        rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
446
+        namespace = settings.CCP_CONF["kubernetes"]["namespace"]
447
+        rabbit_pods = self.get_pods(k8s_actions)
448
+        show_step(2)
449
+        hardware.shutdown_node_by_ip(rabbit_node_ip)
450
+
451
+        helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
452
+                              len(rabbit_pods) - 1),
453
+                     timeout=600,
454
+                     timeout_msg='Timeout waiting for rabbit pod'
455
+                                 ' to be terminated')
456
+        show_step(3)
457
+        with underlay.yaml_editor(settings.CCP_DEPLOY_TOPOLOGY,
458
+                                  host=config.k8s.kube_host) as editor:
459
+            del editor.content['nodes']['node[1-3]']
460
+            editor.content['nodes']['node[1-2]'] = {'roles': ['rabbitmq']}
461
+            editor.content['nodes']['node4'] = {'roles': ['rabbitmq', 'etcd']}
462
+        show_step(4)
463
+        ccpcluster.deploy()
464
+        post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
465
+        post_os_deploy_checks.check_pods_status(k8s_actions.api)
466
+
467
+        pods_after_replacement = self.get_pods(k8s_actions)
468
+        remote = underlay.remote(host=config.k8s.kube_host)
469
+        show_step(6)
470
+        queue = rabbit_client.create_queue()
471
+        message = rabbit_client.publish_message_to_queue(queue)
472
+        rabbit_client.check_queue_message(message)
473
+        show_step(5)
474
+        for pod in pods_after_replacement:
475
+            rabbit_cluster_nodes = rabbit_client.list_nodes(
476
+                remote, pod.name, namespace)
477
+            assert rabbit_cluster_nodes == len(pods_after_replacement),\
478
+                "Expected to have {} nodes in cluster," \
479
+                " got {}".format(len(pods_after_replacement),
480
+                                 rabbit_cluster_nodes)
481
+            rabbit_client.check_queue_replicated(queue, remote,
482
+                                                 pod.name, namespace)
483
+        rabbit_client.delete_queue(queue)
484
+        show_step(7)
485
+        remote.check_call(
486
+            "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
487
+            " create".format(namespace), timeout=600)

Loading…
Cancel
Save