310 lines
12 KiB
Python
310 lines
12 KiB
Python
"""Copyright 2016 Mirantis, Inc.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
not use this file except in compliance with the License. You may obtain
|
|
copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
License for the specific language governing permissions and limitations
|
|
under the License.
|
|
"""
|
|
|
|
import os
|
|
import time
|
|
import libvirt
|
|
|
|
from proboscis import test
|
|
from proboscis.asserts import assert_true
|
|
from proboscis.asserts import assert_equal
|
|
from fuelweb_test import logger
|
|
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
|
from fuelweb_test.settings import CONTRAIL_PLUGIN_PACK_UB_PATH
|
|
from fuelweb_test.tests import base_test_case
|
|
from helpers import vsrx
|
|
from helpers import plugin
|
|
from helpers import openstack
|
|
from helpers.settings import CONTRAIL_PLUGIN_VERSION
|
|
from helpers.settings import OSTF_RUN_TIMEOUT
|
|
from tests.test_contrail_check import TestContrailCheck
|
|
|
|
|
|
@test(groups=["plugins"])
|
|
class FailoverTests(base_test_case.TestBasic):
|
|
"""FailoverTests."""
|
|
|
|
pack_copy_path = '/var/www/nailgun/plugins/contrail-5.1'
|
|
add_package = '/var/www/nailgun/plugins/contrail-5.1/'\
|
|
'repositories/ubuntu/contrail-setup*'
|
|
ostf_msg = 'OSTF tests passed successfully.'
|
|
cluster_id = ''
|
|
pack_path = CONTRAIL_PLUGIN_PACK_UB_PATH
|
|
CONTRAIL_DISTRIBUTION = os.environ.get('CONTRAIL_DISTRIBUTION')
|
|
|
|
@test(depends_on=[base_test_case.SetupEnvironment.prepare_slaves_3],
|
|
groups=["contrail_uninstall", "contrail_failover"])
|
|
@log_snapshot_after_test
|
|
def contrail_uninstall(self):
|
|
"""Check that plugin can be removed.
|
|
|
|
Scenario:
|
|
1. Install plugin and create cluster with activated plugin.
|
|
2. Try to remove plugin and ensure that alert presents in cli:
|
|
'400 Client Error: Bad Request (Can not delete plugin which
|
|
is enabled for some environment.)'
|
|
3. Remove environment.
|
|
4. Remove plugin.
|
|
5. Check that it was removed successfully.
|
|
|
|
|
|
Duration: 5 min
|
|
|
|
"""
|
|
# constants
|
|
plugin_name = 'contrail'
|
|
message = "400 Client Error: Bad Request for url: " + \
|
|
"http://10.109.0.2:8000/api/v1/plugins/1 " + \
|
|
"(Can't delete plugin which is enabled for some environment.)"
|
|
|
|
self.show_step(1)
|
|
plugin.prepare_contrail_plugin(self, slaves=3)
|
|
plugin.activate_plugin(self)
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
self.show_step(2)
|
|
cmd = 'fuel plugins --remove {0}=={1}'.format(
|
|
plugin_name, CONTRAIL_PLUGIN_VERSION)
|
|
|
|
result = self.env.d_env.get_admin_remote().execute(cmd)
|
|
assert_true(
|
|
result['exit_code'] == 1,
|
|
'Plugin is removed.')
|
|
|
|
assert_true(
|
|
result['stderr'].pop().splitlines().pop() == message,
|
|
'Error message was not displayed.')
|
|
|
|
self.show_step(3)
|
|
self.fuel_web.delete_env_wait(cluster_id)
|
|
|
|
self.show_step(4)
|
|
result = self.env.d_env.get_admin_remote().execute(cmd)
|
|
assert_true(
|
|
result['exit_code'] == 0,
|
|
'Plugin was not removed.')
|
|
|
|
self.show_step(5)
|
|
cmd = 'fuel plugins list'
|
|
output = list(self.env.d_env.get_admin_remote().execute(
|
|
cmd)['stdout']).pop().split(' ')
|
|
|
|
assert_true(
|
|
plugin_name not in output,
|
|
"Plugin is not removed {}".format(plugin_name)
|
|
)
|
|
|
|
@test(depends_on=[base_test_case.SetupEnvironment.prepare_slaves_9],
|
|
groups=["contrail_ha_with_node_problems", "contrail_failover"])
|
|
@log_snapshot_after_test
|
|
def contrail_ha_with_node_problems(self):
|
|
"""Check Contrail HA using node problems
|
|
|
|
Scenario:
|
|
1. Deploy openstack with HA (at lest 3 controllers
|
|
and 3 nodes with contrail`s roles) and Ceph
|
|
2. Run OSTF tests
|
|
3. Run contrail health check tests
|
|
4. Disable first contrail node via libvirt
|
|
5. Run OSTF tests
|
|
6. Run contrail health check tests
|
|
7. Enable first contrail node and wait 6 minutes
|
|
8. Disable second cotrail node
|
|
9. Run OSTF test
|
|
10. Run contrail health check tests
|
|
11. Enable second contrail node and wait 6 minutes
|
|
12. Disable third cotrail node
|
|
13. Run OSTF tests
|
|
14. Run contrail health check test
|
|
15. Enable third node
|
|
16. Run OSTF tests
|
|
17. Run contrail health check test
|
|
"""
|
|
|
|
self.show_step(1)
|
|
plugin.prepare_contrail_plugin(self, slaves=9,
|
|
options={'images_ceph': True,
|
|
'volumes_ceph': True,
|
|
'ephemeral_ceph': True,
|
|
'objects_ceph': True,
|
|
'volumes_lvm': False})
|
|
|
|
plugin.activate_plugin(self)
|
|
# activate vSRX image
|
|
vsrx_setup_result = vsrx.activate()
|
|
|
|
conf_env = {
|
|
'slave-01': ['controller', 'ceph-osd'],
|
|
'slave-02': ['controller', 'ceph-osd'],
|
|
'slave-03': ['controller', 'ceph-osd'],
|
|
'slave-04': ['contrail-controller'],
|
|
'slave-05': ['contrail-controller'],
|
|
'slave-06': ['contrail-controller'],
|
|
'slave-07': ['contrail-analytics', 'contrail-analytics-db'],
|
|
'slave-08': ['contrail-analytics', 'contrail-analytics-db'],
|
|
'slave-09': ['contrail-analytics', 'contrail-analytics-db'],
|
|
}
|
|
|
|
self.show_step(2)
|
|
openstack.update_deploy_check(self, conf_env,
|
|
is_vsrx=vsrx_setup_result)
|
|
|
|
self.show_step(3)
|
|
TestContrailCheck(self).cloud_check(['contrail'])
|
|
|
|
#open connection to qemu
|
|
conn = libvirt.open('qemu:///system')
|
|
if conn == None:
|
|
raise ValueError("Failed to open connection to qemu:///system")
|
|
else:
|
|
logger.info("open connection to qemu:///system")
|
|
#get list of domains
|
|
node_list =conn.listAllDomains(0)
|
|
|
|
for i in range(0,3):
|
|
m = 7
|
|
n = str(m + i)
|
|
steps = [4, 5, 6, 7]
|
|
|
|
for index, step in enumerate(steps):
|
|
steps[index] = step + (4 * i)
|
|
|
|
for node in node_list:
|
|
#find node with contrail
|
|
if n in node.name():
|
|
|
|
self.show_step(steps[0])
|
|
node.destroy() # stop node
|
|
logger.info("Node {0} was stopped".format(n))
|
|
|
|
self.show_step(steps[1])
|
|
self.fuel_web.run_ostf(cluster_id=self.cluster_id,
|
|
test_sets=['smoke', 'sanity', 'ha'])
|
|
self.show_step(steps[2])
|
|
TestContrailCheck(self).cloud_check(['contrail'],
|
|
['test_contrail_node_status'])
|
|
|
|
self.show_step(steps[3])
|
|
node.create() # start node
|
|
logger.info("Node {0} was run".format(n))
|
|
break
|
|
logger.info("Waiting for node {0}".format(n))
|
|
time.sleep(6 * 60)
|
|
conn.close()
|
|
|
|
self.show_step(16)
|
|
if vsrx_setup_result:
|
|
self.fuel_web.run_ostf(cluster_id=self.cluster_id,
|
|
test_sets=['smoke', 'sanity', 'ha'],
|
|
timeout=OSTF_RUN_TIMEOUT)
|
|
|
|
self.show_step(17)
|
|
TestContrailCheck(self).cloud_check(['contrail'])
|
|
|
|
|
|
@test(depends_on=[SetupEnvironment.prepare_slaves_9],
|
|
groups=["contrail_ha_with_network_problems", "contrail_failover"])
|
|
@log_snapshot_after_test
|
|
def contrail_ha_with_network_problems(self):
|
|
"""Check Contrail HA using network problems
|
|
|
|
Scenario:
|
|
1. Deploy openstack with HA (at lest 3 controllers
|
|
and 3 nodes with contrail`s roles)
|
|
2. Run OSTF tests
|
|
3. Run contrail health check tests
|
|
4. Connect to a contrail controller host, stop the network
|
|
interfaces connected to private and management networks
|
|
5. Run OSTF tests
|
|
6. Run contrail health check tests
|
|
"""
|
|
|
|
self.show_step(1)
|
|
plugin.prepare_contrail_plugin(self, slaves=9)
|
|
|
|
plugin.activate_plugin(self)
|
|
# activate vSRX image
|
|
vsrx_setup_result = vsrx.activate()
|
|
|
|
conf_env = {
|
|
'slave-01': ['controller'],
|
|
'slave-02': ['controller'],
|
|
'slave-03': ['controller'],
|
|
'slave-04': ['contrail-controller'],
|
|
'slave-05': ['contrail-controller'],
|
|
'slave-06': ['contrail-controller'],
|
|
'slave-07': ['contrail-analytics', 'contrail-analytics-db'],
|
|
'slave-08': ['contrail-analytics', 'contrail-analytics-db'],
|
|
'slave-09': ['contrail-analytics', 'contrail-analytics-db'],
|
|
}
|
|
|
|
self.show_step(2)
|
|
openstack.update_deploy_check(self, conf_env,
|
|
is_vsrx=vsrx_setup_result)
|
|
|
|
self.show_step(3)
|
|
TestContrailCheck(self).cloud_check(['contrail'])
|
|
|
|
self.show_step(4)
|
|
|
|
def get_interface_name4role(cluster_id, node_name, role):
|
|
logger.info('[get_interface_name4role] cluster_id: {0}, '
|
|
'node: {1}, '
|
|
'role:{2}'.format(cluster_id, node_name, role))
|
|
nailgun_node = None
|
|
for node in self.fuel_web.client.list_cluster_nodes(cluster_id):
|
|
logger.info('Check node {0} for name: {1}'.format(node['name'],
|
|
node_name))
|
|
if node_name in node['name']:
|
|
nailgun_node = node
|
|
break
|
|
else:
|
|
assert 'slave-01 node not found in cluster: {0}'.format(
|
|
self.cluster_id)
|
|
|
|
interfaces = self.fuel_web.client.get_node_interfaces(
|
|
nailgun_node['id'])
|
|
for iface in interfaces:
|
|
for net in iface['assigned_networks']:
|
|
logger.info('Check iface {0} for role {1}'.format(
|
|
iface['name'], role))
|
|
if role in net['name']: # check role name for interface
|
|
logger.info('Found interface: {0}'.format(
|
|
iface['name']))
|
|
return iface['name'] # return interface name
|
|
return ''
|
|
|
|
cluster_id = self.cluster_id
|
|
ifaces = [
|
|
get_interface_name4role(cluster_id, 'slave-01', 'private'),
|
|
get_interface_name4role(cluster_id, 'slave-01', 'management')
|
|
]
|
|
for iface in ifaces:
|
|
cmd = 'sudo ifconfig {0} down'.format(iface)
|
|
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
|
|
res_pgrep = remote.execute(cmd)
|
|
assert_equal(0, res_pgrep['exit_code'],
|
|
'Failed with error code:{0}, {1}, out:{2}'.format(
|
|
res_pgrep['exit_code'],
|
|
res_pgrep['stderr'],
|
|
res_pgrep['stdout']))
|
|
self.show_step(5)
|
|
if vsrx_setup_result:
|
|
self.fuel_web.run_ostf(cluster_id=self.cluster_id,
|
|
test_sets=['smoke', 'sanity', 'ha'],
|
|
timeout=OSTF_RUN_TIMEOUT)
|
|
self.show_step(6)
|
|
TestContrailCheck(self).cloud_check(['contrail'])
|