Browse Source

Adds Hyper-V Cluster scenario

Hyper-V VMs can be clustered, making them highly available.

We can force a VM failover through WinRM, causing the VM to
restart on another host. For this, the Hyper-V hosts must have
WinRM enabled.

A VM must have network connectivity after the failover,
and operations (resize, migrate, etc.) must still succeed after failover.

Adds the following config options:
- cluster_enabled (default = False)
- username
- password
- failover_timeout (default = 120 seconds)
- failover_sleep_interval (default = 5 seconds)

Adds HyperVClusterTest.
Claudiu Belu 1 year ago
parent
commit
763348b367

+ 0
- 0
oswin_tempest_plugin/clients/__init__.py View File


+ 60
- 0
oswin_tempest_plugin/clients/wsman.py View File

@@ -0,0 +1,60 @@
1
+# Copyright 2013 Cloudbase Solutions Srl
2
+# All Rights Reserved.
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+from oslo_log import log as logging
17
+from winrm import protocol
18
+
19
+from oswin_tempest_plugin import exceptions
20
+
21
+LOG = logging.getLogger(__name__)
22
+
23
+protocol.Protocol.DEFAULT_TIMEOUT = "PT3600S"
24
+
25
+
26
+def run_wsman_cmd(host, username, password, cmd, fail_on_error=False):
27
+    url = 'https://%s:5986/wsman' % host
28
+    LOG.debug('Connecting to: %s', host)
29
+    p = protocol.Protocol(endpoint=url,
30
+                          transport='plaintext',
31
+                          server_cert_validation='ignore',
32
+                          username=username,
33
+                          password=password)
34
+
35
+    shell_id = p.open_shell()
36
+    LOG.debug('Running command on host %(host)s: %(cmd)s',
37
+              {'host': host, 'cmd': cmd})
38
+    command_id = p.run_command(shell_id, cmd)
39
+    std_out, std_err, return_code = p.get_command_output(shell_id, command_id)
40
+
41
+    p.cleanup_command(shell_id, command_id)
42
+    p.close_shell(shell_id)
43
+
44
+    LOG.debug('Results from %(host)s: return_code: %(return_code)s, std_out: '
45
+              '%(std_out)s, std_err: %(std_err)s',
46
+              {'host': host, 'return_code': return_code, 'std_out': std_out,
47
+               'std_err': std_err})
48
+
49
+    if fail_on_error and return_code:
50
+        raise exceptions.WSManException(
51
+            cmd=cmd, host=host, return_code=return_code,
52
+            std_out=std_out, std_err=std_err)
53
+
54
+    return (std_out, std_err, return_code)
55
+
56
+
57
+def run_wsman_ps(host, username, password, cmd, fail_on_error=False):
58
+    cmd = ("powershell -NonInteractive -ExecutionPolicy RemoteSigned "
59
+           "-Command \"%s\"" % cmd)
60
+    return run_wsman_cmd(host, username, password, cmd, fail_on_error)

+ 15
- 0
oswin_tempest_plugin/config.py View File

@@ -36,6 +36,21 @@ HyperVGroup = [
36 36
     cfg.StrOpt('gen2_image_ref',
37 37
                help="Valid Generation 2 VM VHDX image reference to be used "
38 38
                     "in tests."),
39
+    cfg.BoolOpt('cluster_enabled',
40
+                default=False,
41
+                help="The compute nodes are joined into a Hyper-V Cluster."),
42
+    cfg.StrOpt('username',
43
+               help="The username of the Hyper-V hosts."),
44
+    cfg.StrOpt('password',
45
+               secret=True,
46
+               help='The password of the Hyper-V hosts.'),
47
+    cfg.IntOpt('failover_timeout',
48
+               default=120,
49
+               help='The maximum amount of time to wait for a failover to '
50
+                    'occur.'),
51
+    cfg.IntOpt('failover_sleep_interval',
52
+               default=5,
53
+               help='The amount of time to wait between failover checks.'),
39 54
 ]
40 55
 
41 56
 

+ 10
- 0
oswin_tempest_plugin/exceptions.py View File

@@ -19,3 +19,13 @@ from tempest.lib import exceptions
19 19
 class ResizeException(exceptions.TempestException):
20 20
     message = ("Server %(server_id)s failed to resize to the given "
21 21
                "flavor %(flavor)s")
22
+
23
+
24
+class NotFoundException(exceptions.TempestException):
25
+    message = "Resource %(resource)s (%(res_type)s) was not found."
26
+
27
+
28
+class WSManException(exceptions.TempestException):
29
+    message = ('Command "%(cmd)s" failed on host %(host)s failed with the '
30
+               'return code %(return_code)s. std_out: %(std_out)s, '
31
+               'std_err: %(std_err)s')

+ 158
- 0
oswin_tempest_plugin/tests/scenario/test_cluster.py View File

@@ -0,0 +1,158 @@
1
+# Copyright 2017 Cloudbase Solutions SRL
2
+# All Rights Reserved.
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+import time
17
+
18
+from oslo_log import log as logging
19
+from tempest.lib import exceptions as lib_exc
20
+
21
+from oswin_tempest_plugin.clients import wsman
22
+from oswin_tempest_plugin import config
23
+from oswin_tempest_plugin import exceptions
24
+from oswin_tempest_plugin.tests import test_base
25
+from oswin_tempest_plugin.tests._mixins import migrate
26
+from oswin_tempest_plugin.tests._mixins import resize
27
+
28
+CONF = config.CONF
29
+LOG = logging.getLogger(__name__)
30
+
31
+
32
+class HyperVClusterTest(test_base.TestBase,
33
+                        migrate._MigrateMixin,
34
+                        resize._ResizeMixin):
35
+
36
+    """The test suite for the Hyper-V Cluster.
37
+
38
+    This test suite will test the functionality of the Hyper-V Cluster Driver
39
+    in OpenStack. The tests will force a failover on its newly created
40
+    instance, and asserts the following:
41
+
42
+    * the instance moves to another host.
43
+    * the nova instance's host is properly updated.
44
+    * the instance's network connection still works.
45
+    * different nova operations can be performed properly.
46
+
47
+    This test suite relies on the fact that there are at least 2 compute nodes
48
+    available, that they are clustered, and have WSMan configured.
49
+
50
+    The test suite contains the following tests:
51
+
52
+    * test_check_clustered_vm
53
+    * test_check_migration
54
+    * test_check_resize
55
+    * test_check_resize_negative
56
+    """
57
+
58
+    _BIGGER_FLAVOR = {'disk': 1}
59
+    _BAD_FLAVOR = {'disk': -1}
60
+
61
+    @classmethod
62
+    def skip_checks(cls):
63
+        super(HyperVClusterTest, cls).skip_checks()
64
+
65
+        # check if the cluster Tests can be run.
66
+        conf_opts = ['cluster_enabled', 'username', 'password']
67
+        for conf_opt in conf_opts:
68
+            if not getattr(CONF.hyperv, conf_opt):
69
+                msg = ('The config option "hyperv.%s" has not been set. '
70
+                       'Skipping.' % conf_opt)
71
+                raise cls.skipException(msg)
72
+
73
+        if not CONF.compute.min_compute_nodes >= 2:
74
+            msg = 'Expected at least 2 compute nodes.'
75
+            raise cls.skipException(msg)
76
+
77
+    def _failover_server(self, server_name, host_ip):
78
+        """Triggers the failover for the given server on the given host."""
79
+
80
+        resource_name = "Virtual Machine %s" % server_name
81
+        cmd = "Test-ClusterResourceFailure -Name '%s'" % resource_name
82
+        # NOTE(claudiub): we issue the failover command twice, because on
83
+        # the first failure, the Hyper-V Cluster will prefer the current
84
+        # node, and will try to reactivate the VM on the it, and it will
85
+        # succeed. On the 2nd failure, the VM will failover to another
86
+        # node. Also, there needs to be a delay between commands, so the
87
+        # original failover has time to finish.
88
+        wsman.run_wsman_ps(host_ip, CONF.hyperv.username,
89
+                           CONF.hyperv.password, cmd, True)
90
+        time.sleep(CONF.hyperv.failover_sleep_interval)
91
+        wsman.run_wsman_ps(host_ip, CONF.hyperv.username,
92
+                           CONF.hyperv.password, cmd, True)
93
+
94
+    def _wait_for_failover(self, server, original_host):
95
+        """Waits for the given server to failover to another host.
96
+
97
+        :raises TimeoutException: if the given server did not failover to
98
+            another host within the configured "CONF.hyperv.failover_timeout"
99
+            interval.
100
+        """
101
+        LOG.debug('Waiting for server %(server)s to failover from '
102
+                  'compute node %(host)s',
103
+                  dict(server=server['id'], host=original_host))
104
+
105
+        start_time = int(time.time())
106
+        timeout = CONF.hyperv.failover_timeout
107
+        while True:
108
+            elapsed_time = int(time.time()) - start_time
109
+            admin_server = self._get_server_as_admin(server)
110
+            current_host = admin_server['OS-EXT-SRV-ATTR:host']
111
+            if current_host != original_host:
112
+                LOG.debug('Server %(server)s failovered from compute node '
113
+                          '%(host)s in %(seconds)s seconds.',
114
+                          dict(server=server['id'], host=original_host,
115
+                               seconds=elapsed_time))
116
+                return
117
+
118
+            if elapsed_time >= timeout:
119
+                msg = ('Server %(server)s did not failover in the given '
120
+                       'amount of time (%(timeout)s s).')
121
+                raise lib_exc.TimeoutException(
122
+                    msg % dict(server=server['id'], timeout=timeout))
123
+
124
+            time.sleep(CONF.hyperv.failover_sleep_interval)
125
+
126
+    def _get_hypervisor(self, hostname):
127
+        hypervisors = self.admin_hypervisor_client.list_hypervisors(
128
+            detail=True)['hypervisors']
129
+        hypervisor = [h for h in hypervisors if
130
+                      h['hypervisor_hostname'] == hostname]
131
+
132
+        if not hypervisor:
133
+            raise exceptions.NotFoundException(resource=hostname,
134
+                                               res_type='hypervisor')
135
+        return hypervisor[0]
136
+
137
+    def _get_server_as_admin(self, server):
138
+        # only admins have access to certain instance properties.
139
+        return self.admin_servers_client.show_server(
140
+            server['id'])['server']
141
+
142
+    def _create_server(self):
143
+        server_tuple = super(HyperVClusterTest, self)._create_server()
144
+        server = server_tuple.server
145
+        admin_server = self._get_server_as_admin(server)
146
+
147
+        server_name = admin_server['OS-EXT-SRV-ATTR:instance_name']
148
+        hostname = admin_server['OS-EXT-SRV-ATTR:host']
149
+        host_ip = self._get_hypervisor(hostname)['host_ip']
150
+
151
+        self._failover_server(server_name, host_ip)
152
+        self._wait_for_failover(server, hostname)
153
+
154
+        return server_tuple
155
+
156
+    def test_clustered_vm(self):
157
+        server_tuple = self._create_server()
158
+        self._check_server_connectivity(server_tuple)

+ 1
- 0
oswin_tempest_plugin/tests/test_base.py View File

@@ -76,6 +76,7 @@ class TestBase(tempest.test.BaseTestCase):
76 76
         cls.admin_servers_client = cls.os_admin.servers_client
77 77
         cls.admin_flavors_client = cls.os_admin.flavors_client
78 78
         cls.admin_migrations_client = cls.os_admin.migrations_client
79
+        cls.admin_hypervisor_client = cls.os_admin.hypervisor_client
79 80
 
80 81
         # Neutron network client
81 82
         cls.security_groups_client = (

+ 1
- 0
requirements.txt View File

@@ -3,3 +3,4 @@
3 3
 # process, which may cause wedges in the gate later.
4 4
 
5 5
 pbr>=2.0 # Apache-2.0
6
+pywinrm>=0.2.2 # MIT

Loading…
Cancel
Save