Tests for separate services feature
- Tests to cover separate deployment of db, keystone, rabbitmq, horizon were added - Failover tests for each service were added Change-Id: Id2e644404509ddfe9c412c48919e503824876c2a Implements: blueprint separate-controller-services-system-tests
This commit is contained in:
asledzinskiy
@@ -52,6 +52,11 @@ def import_tests():
|
||||
from tests import test_services # noqa
|
||||
from tests import test_ha_one_controller # noqa
|
||||
from tests import test_vcenter # noqa
|
||||
from tests.tests_separate_services import test_separate_db # noqa
|
||||
from tests.tests_separate_services import test_separate_horizon # noqa
|
||||
from tests.tests_separate_services import test_separate_keystone # noqa
|
||||
from tests.tests_separate_services import test_separate_multiroles # noqa
|
||||
from tests.tests_separate_services import test_separate_rabbitmq # noqa
|
||||
from tests.tests_strength import test_failover # noqa
|
||||
from tests.tests_strength import test_failover_with_ceph # noqa
|
||||
from tests.tests_strength import test_master_node_failover # noqa
|
||||
|
||||
@@ -349,6 +349,14 @@ LMA_COLLECTOR_PLUGIN_PATH = os.environ.get('LMA_COLLECTOR_PLUGIN_PATH')
|
||||
ELASTICSEARCH_KIBANA_PLUGIN_PATH = os.environ.get(
|
||||
'ELASTICSEARCH_KIBANA_PLUGIN_PATH')
|
||||
INFLUXDB_GRAFANA_PLUGIN_PATH = os.environ.get('INFLUXDB_GRAFANA_PLUGIN_PATH')
|
||||
SEPARATE_SERVICE_DB_PLUGIN_PATH = os.environ.get(
|
||||
'SEPARATE_SERVICE_DB_PLUGIN_PATH')
|
||||
SEPARATE_SERVICE_RABBIT_PLUGIN_PATH = os.environ.get(
|
||||
'SEPARATE_SERVICE_RABBIT_PLUGIN_PATH')
|
||||
SEPARATE_SERVICE_KEYSTONE_PLUGIN_PATH = os.environ.get(
|
||||
'SEPARATE_SERVICE_KEYSTONE_PLUGIN_PATH')
|
||||
SEPARATE_SERVICE_HORIZON_PLUGIN_PATH = os.environ.get(
|
||||
'SEPARATE_SERVICE_HORIZON_PLUGIN_PATH')
|
||||
|
||||
FUEL_STATS_CHECK = get_var_as_bool('FUEL_STATS_CHECK', False)
|
||||
FUEL_STATS_ENABLED = get_var_as_bool('FUEL_STATS_ENABLED', True)
|
||||
|
||||
244
fuelweb_test/tests/tests_separate_services/test_separate_db.py
Normal file
244
fuelweb_test/tests/tests_separate_services/test_separate_db.py
Normal file
@@ -0,0 +1,244 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
from proboscis.asserts import assert_true
|
||||
from proboscis import test
|
||||
from devops.helpers.helpers import wait
|
||||
|
||||
from fuelweb_test.helpers import checkers
|
||||
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
||||
from fuelweb_test import settings
|
||||
from fuelweb_test import logger
|
||||
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
||||
from fuelweb_test.tests.base_test_case import TestBasic
|
||||
|
||||
|
||||
@test(groups=["thread_separate_services", "thread_1_separate_services"])
|
||||
class SeparateDb(TestBasic):
|
||||
"""SeparateDb""" # TODO documentation
|
||||
|
||||
@test(depends_on=[SetupEnvironment.prepare_slaves_9],
|
||||
groups=["separate_db_service"])
|
||||
@log_snapshot_after_test
|
||||
def separate_db_service(self):
|
||||
"""Deploy cluster with 3 separate database roles
|
||||
|
||||
Scenario:
|
||||
1. Create cluster
|
||||
2. Add 3 nodes with controller role
|
||||
3. Add 3 nodes with database role
|
||||
4. Add 1 compute and cinder
|
||||
5. Verify networks
|
||||
6. Deploy the cluster
|
||||
7. Verify networks
|
||||
8. Run OSTF
|
||||
|
||||
Duration 120m
|
||||
Snapshot separate_db_service
|
||||
"""
|
||||
self.check_run("separate_db_service")
|
||||
self.env.revert_snapshot("ready_with_9_slaves")
|
||||
|
||||
# copy plugins to the master node
|
||||
|
||||
checkers.upload_tarball(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
settings.SEPARATE_SERVICE_DB_PLUGIN_PATH, "/var")
|
||||
|
||||
# install plugins
|
||||
|
||||
checkers.install_plugin_check_code(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
plugin=os.path.basename(settings.SEPARATE_SERVICE_DB_PLUGIN_PATH))
|
||||
|
||||
data = {
|
||||
'tenant': 'separatedb',
|
||||
'user': 'separatedb',
|
||||
'password': 'separatedb',
|
||||
"net_provider": 'neutron',
|
||||
"net_segment_type": settings.NEUTRON_SEGMENT['vlan'],
|
||||
}
|
||||
|
||||
cluster_id = self.fuel_web.create_cluster(
|
||||
name=self.__class__.__name__,
|
||||
mode=settings.DEPLOYMENT_MODE,
|
||||
settings=data)
|
||||
|
||||
plugin_name = 'detach-database'
|
||||
msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
|
||||
assert_true(
|
||||
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
|
||||
msg)
|
||||
options = {'metadata/enabled': True}
|
||||
self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)
|
||||
|
||||
self.fuel_web.update_nodes(
|
||||
cluster_id,
|
||||
{
|
||||
'slave-01': ['controller'],
|
||||
'slave-02': ['controller'],
|
||||
'slave-03': ['controller'],
|
||||
'slave-04': ['standalone-database'],
|
||||
'slave-05': ['standalone-database'],
|
||||
'slave-06': ['standalone-database'],
|
||||
'slave-07': ['compute'],
|
||||
'slave-08': ['cinder']
|
||||
}
|
||||
)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
|
||||
# Cluster deploy
|
||||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
self.env.make_snapshot("separate_db_service", is_make=True)
|
||||
|
||||
|
||||
@test(groups=["thread_separate_services", "thread_1_separate_services"])
|
||||
class SeparateDbFailover(TestBasic):
|
||||
"""SeparateDbFailover""" # TODO documentation
|
||||
|
||||
@test(depends_on=[SeparateDb.separate_db_service],
|
||||
groups=["separate_db_service_shutdown"])
|
||||
@log_snapshot_after_test
|
||||
def separate_db_service_shutdown(self):
|
||||
"""Shutdown one database node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_db_service
|
||||
2. Destroy db node that is master
|
||||
3. Wait galera is up
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_db_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#destroy one db node
|
||||
db_node = self.env.d_env.nodes().slaves[3]
|
||||
db_node.destroy()
|
||||
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
db_node)['online'], timeout=60 * 5)
|
||||
# Wait until MySQL Galera is UP on some db node
|
||||
self.fuel_web.wait_mysql_galera_is_up(['slave-05'])
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
@test(depends_on=[SeparateDb.separate_db_service],
|
||||
groups=["separate_db_service_restart"])
|
||||
@log_snapshot_after_test
|
||||
def separate_db_service_restart(self):
|
||||
"""Restart one database node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_db_service
|
||||
2. Restart db node that is master
|
||||
3. Wait galera is up
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_db_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#restart one db node
|
||||
db_node = self.env.d_env.nodes().slaves[3]
|
||||
self.fuel_web.warm_restart_nodes([db_node])
|
||||
wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
db_node)['online'], timeout=60 * 5)
|
||||
# Wait until MySQL Galera is UP on some db node
|
||||
self.fuel_web.wait_mysql_galera_is_up(['slave-05'])
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
@test(depends_on=[SeparateDb.separate_db_service],
|
||||
groups=["separate_db_service_controller_shutdown"])
|
||||
@log_snapshot_after_test
|
||||
def separate_db_service_controller_shutdown(self):
|
||||
"""Shutdown primary controller node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_db_service
|
||||
2. Shutdown primary controller node
|
||||
3. Wait rabbit and db are operational
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_db_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#shutdown primary controller
|
||||
controller = self.fuel_web.get_nailgun_primary_node(
|
||||
self.env.d_env.nodes().slaves[0])
|
||||
logger.debug(
|
||||
"controller with primary role is {}".format(controller.name))
|
||||
controller.destroy()
|
||||
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
controller)['online'], timeout=60 * 5)
|
||||
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id, should_fail=1)
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id, should_fail=1)
|
||||
|
||||
@test(depends_on=[SeparateDb.separate_db_service],
|
||||
groups=["separate_db_service_add_delete_node"])
|
||||
@log_snapshot_after_test
|
||||
def separate_db_service_add_delete_node(self):
|
||||
"""Add and delete database node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_db_service
|
||||
2. Add one databse node and re-deploy cluster
|
||||
3. Run network verification
|
||||
4. Run OSTF
|
||||
5. Delete one database node
|
||||
6. Run network verification
|
||||
7. Run ostf
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_db_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
|
||||
node = {'slave-09': ['standalone-database']}
|
||||
self.fuel_web.update_nodes(
|
||||
cluster_id, node, True, False)
|
||||
|
||||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self.fuel_web.run_ostf(cluster_id=cluster_id,
|
||||
test_sets=['sanity', 'smoke', 'ha'])
|
||||
|
||||
nailgun_node = self.fuel_web.update_nodes(cluster_id, node,
|
||||
False, True)
|
||||
nodes = filter(lambda x: x["pending_deletion"] is True, nailgun_node)
|
||||
self.fuel_web.deploy_cluster(cluster_id)
|
||||
wait(lambda: self.fuel_web.is_node_discovered(nodes[0]),
|
||||
timeout=6 * 60)
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self.fuel_web.run_ostf(cluster_id=cluster_id,
|
||||
test_sets=['sanity', 'smoke', 'ha'])
|
||||
@@ -0,0 +1,245 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
from proboscis import test
|
||||
from proboscis.asserts import assert_true
|
||||
from devops.helpers.helpers import wait
|
||||
|
||||
from fuelweb_test.helpers import checkers
|
||||
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
||||
from fuelweb_test import settings
|
||||
from fuelweb_test import logger
|
||||
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
||||
from fuelweb_test.tests.base_test_case import TestBasic
|
||||
|
||||
|
||||
@test(groups=["thread_separate_services"])
|
||||
class SeparateHorizon(TestBasic):
|
||||
"""SeparateHorizon""" # TODO documentation
|
||||
|
||||
@test(depends_on=[SetupEnvironment.prepare_slaves_9],
|
||||
groups=["separate_horizon_service"])
|
||||
@log_snapshot_after_test
|
||||
def separate_horizon_service(self):
|
||||
"""Deploy cluster with 3 separate horizon roles
|
||||
|
||||
Scenario:
|
||||
1. Create cluster
|
||||
2. Add 3 nodes with controller role
|
||||
3. Add 3 nodes with horizon role
|
||||
4. Add 1 compute and cinder
|
||||
5. Verify networks
|
||||
6. Deploy the cluster
|
||||
7. Verify networks
|
||||
8. Run OSTF
|
||||
|
||||
Duration 120m
|
||||
Snapshot separate_horizon_service
|
||||
"""
|
||||
self.check_run("separate_horizon_service")
|
||||
self.env.revert_snapshot("ready_with_9_slaves")
|
||||
|
||||
# copy plugins to the master node
|
||||
|
||||
checkers.upload_tarball(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
settings.SEPARATE_SERVICE_HORIZON_PLUGIN_PATHPLUGIN_PATH, "/var")
|
||||
|
||||
# install plugins
|
||||
|
||||
checkers.install_plugin_check_code(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
plugin=os.path.basename(
|
||||
settings.SEPARATE_SERVICE_HORIZON_PLUGIN_PATH))
|
||||
|
||||
data = {
|
||||
'volumes_lvm': False,
|
||||
'volumes_ceph': True,
|
||||
'images_ceph': True,
|
||||
'objects_ceph': True,
|
||||
'tenant': 'separatehorizon',
|
||||
'user': 'separatehorizon',
|
||||
'password': 'separatehorizon',
|
||||
"net_provider": 'neutron',
|
||||
"net_segment_type": settings.NEUTRON_SEGMENT['vlan'],
|
||||
}
|
||||
|
||||
cluster_id = self.fuel_web.create_cluster(
|
||||
name=self.__class__.__name__,
|
||||
mode=settings.DEPLOYMENT_MODE,
|
||||
settings=data)
|
||||
|
||||
plugin_name = 'detach-horizon'
|
||||
msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
|
||||
assert_true(
|
||||
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
|
||||
msg)
|
||||
options = {'metadata/enabled': True}
|
||||
self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)
|
||||
|
||||
self.fuel_web.update_nodes(
|
||||
cluster_id,
|
||||
{
|
||||
'slave-01': ['controller'],
|
||||
'slave-02': ['controller'],
|
||||
'slave-03': ['controller'],
|
||||
'slave-04': ['standalone-horizon'],
|
||||
'slave-05': ['stabdalone-horizon'],
|
||||
'slave-06': ['standalone-horizon'],
|
||||
'slave-07': ['compute'],
|
||||
'slave-08': ['ceph-osd'],
|
||||
'slave-09': ['ceph-osd']
|
||||
}
|
||||
)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
|
||||
# Cluster deploy
|
||||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
self.env.make_snapshot("separate_horizon_service", is_make=True)
|
||||
|
||||
|
||||
@test(groups=["thread_separate_services"])
|
||||
class SeparateHorizonFailover(TestBasic):
|
||||
"""SeparateHorizonFailover""" # TODO documentation
|
||||
|
||||
@test(depends_on=[SeparateHorizon.separate_horizon_service],
|
||||
groups=["separate_horizon_service_shutdown"])
|
||||
@log_snapshot_after_test
|
||||
def separate_horizon_service_shutdown(self):
|
||||
"""Shutdown one horizon node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_horizon_service
|
||||
2. Destroy horizon node
|
||||
3. Wait OS services are working
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_horizon_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#destroy one horizon node
|
||||
horizon_node = self.env.d_env.nodes().slaves[3]
|
||||
horizon_node.destroy()
|
||||
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
horizon_node)['online'], timeout=60 * 5)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
@test(depends_on=[SeparateHorizon.separate_horizon_service],
|
||||
groups=["separate_horizon_service_restart"])
|
||||
@log_snapshot_after_test
|
||||
def separate_horizon_service_restart(self):
|
||||
"""Restart one horizon node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_horizon_service
|
||||
2. Restart horizon node
|
||||
3. Wait OS services are ready
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_horizon_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#restart one horizon node
|
||||
horizon_node = self.env.d_env.nodes().slaves[3]
|
||||
self.fuel_web.warm_restart_nodes([horizon_node])
|
||||
wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
horizon_node)['online'], timeout=60 * 5)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
@test(depends_on=[SeparateHorizon.separate_horizon_service],
|
||||
groups=["separate_horizon_service_controller_shutdown"])
|
||||
@log_snapshot_after_test
|
||||
def separate_horizon_service_controller_shutdown(self):
|
||||
"""Shutdown primary controller node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_horizon_service
|
||||
2. Shutdown primary controller node
|
||||
3. Wait HA is working
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_horizon_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#shutdown primary controller
|
||||
controller = self.fuel_web.get_nailgun_primary_node(
|
||||
self.env.d_env.nodes().slaves[0])
|
||||
logger.debug(
|
||||
"controller with primary role is {}".format(controller.name))
|
||||
controller.destroy()
|
||||
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
controller)['online'], timeout=60 * 5)
|
||||
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id, should_fail=1)
|
||||
|
||||
@test(depends_on=[SeparateHorizon.separate_horizon_service],
|
||||
groups=["separate_horizon_service_add_delete_node"])
|
||||
@log_snapshot_after_test
|
||||
def separate_horizon_service_add_delete_node(self):
|
||||
"""Add and delete horizon node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_horizon_service
|
||||
2. Add one horizon node and re-deploy cluster
|
||||
3. Run network verification
|
||||
4. Run OSTF
|
||||
5. Delete one horizon node
|
||||
6. Run network verification
|
||||
7. Run ostf
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_horizon_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
|
||||
node = {'slave-09': ['horizon']}
|
||||
self.fuel_web.update_nodes(
|
||||
cluster_id, node, True, False)
|
||||
|
||||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self.fuel_web.run_ostf(cluster_id=cluster_id,
|
||||
test_sets=['sanity', 'smoke', 'ha'])
|
||||
|
||||
nailgun_node = self.fuel_web.update_nodes(cluster_id, node,
|
||||
False, True)
|
||||
nodes = filter(lambda x: x["pending_deletion"] is True, nailgun_node)
|
||||
self.fuel_web.deploy_cluster(cluster_id)
|
||||
wait(lambda: self.fuel_web.is_node_discovered(nodes[0]),
|
||||
timeout=6 * 60)
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self.fuel_web.run_ostf(cluster_id=cluster_id,
|
||||
test_sets=['sanity', 'smoke', 'ha'])
|
||||
@@ -0,0 +1,252 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
from proboscis import test
|
||||
from proboscis.asserts import assert_true
|
||||
from devops.helpers.helpers import wait
|
||||
|
||||
from fuelweb_test.helpers import checkers
|
||||
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
||||
from fuelweb_test import settings
|
||||
from fuelweb_test import logger
|
||||
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
||||
from fuelweb_test.tests.base_test_case import TestBasic
|
||||
|
||||
|
||||
@test(groups=["thread_separate_services", "thread_1_separate_services"])
|
||||
class SeparateKeystone(TestBasic):
|
||||
"""SeparateKeystone""" # TODO documentation
|
||||
|
||||
@test(depends_on=[SetupEnvironment.prepare_slaves_9],
|
||||
groups=["separate_keystone_service"])
|
||||
@log_snapshot_after_test
|
||||
def separate_keystone_service(self):
|
||||
"""Deploy cluster with 3 separate keystone roles
|
||||
|
||||
Scenario:
|
||||
1. Create cluster
|
||||
2. Add 3 nodes with controller role
|
||||
3. Add 3 nodes with keystone role
|
||||
4. Add 1 compute and cinder
|
||||
5. Verify networks
|
||||
6. Deploy the cluster
|
||||
7. Verify networks
|
||||
8. Run OSTF
|
||||
|
||||
Duration 120m
|
||||
Snapshot separate_keystone_service
|
||||
"""
|
||||
self.check_run("separate_keystone_service")
|
||||
self.env.revert_snapshot("ready_with_9_slaves")
|
||||
|
||||
# copy plugins to the master node
|
||||
|
||||
checkers.upload_tarball(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
settings.SEPARATE_SERVICE_DB_PLUGIN_PATH, "/var")
|
||||
|
||||
checkers.upload_tarball(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
settings.SEPARATE_SERVICE_KEYSTONE_PLUGIN_PATH, "/var")
|
||||
|
||||
# install plugins
|
||||
|
||||
checkers.install_plugin_check_code(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
plugin=os.path.basename(settings.SEPARATE_SERVICE_DB_PLUGIN_PATH))
|
||||
|
||||
checkers.install_plugin_check_code(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
plugin=os.path.basename(
|
||||
settings.SEPARATE_SERVICE_KEYSTONE_PLUGIN_PATH))
|
||||
|
||||
data = {
|
||||
'tenant': 'separatekeystone',
|
||||
'user': 'separatekeystone',
|
||||
'password': 'separatekeystone',
|
||||
"net_provider": 'neutron',
|
||||
"net_segment_type": settings.NEUTRON_SEGMENT['vlan'],
|
||||
}
|
||||
|
||||
cluster_id = self.fuel_web.create_cluster(
|
||||
name=self.__class__.__name__,
|
||||
mode=settings.DEPLOYMENT_MODE,
|
||||
settings=data)
|
||||
|
||||
plugin_names = ['detach-database', 'detach-keystone']
|
||||
msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
|
||||
for plugin_name in plugin_names:
|
||||
assert_true(
|
||||
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
|
||||
msg)
|
||||
options = {'metadata/enabled': True}
|
||||
self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)
|
||||
|
||||
self.fuel_web.update_nodes(
|
||||
cluster_id,
|
||||
{
|
||||
'slave-01': ['controller'],
|
||||
'slave-02': ['controller'],
|
||||
'slave-03': ['controller'],
|
||||
'slave-04': ['standalone-database', 'standalone-keystone'],
|
||||
'slave-05': ['standalone-database', 'standalone-keystone'],
|
||||
'slave-06': ['standalone-database', 'standalone-keystone'],
|
||||
'slave-07': ['compute'],
|
||||
'slave-08': ['cinder']
|
||||
}
|
||||
)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
|
||||
# Cluster deploy
|
||||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
self.env.make_snapshot("separate_keystone_service", is_make=True)
|
||||
|
||||
|
||||
@test(groups=["thread_separate_services", "thread_1_separate_services"])
|
||||
class SeparateKeystoneFailover(TestBasic):
|
||||
"""SeparateKeystoneFailover""" # TODO documentation
|
||||
|
||||
@test(depends_on=[SeparateKeystone.separate_keystone_service],
|
||||
groups=["separate_keystone_service_shutdown"])
|
||||
@log_snapshot_after_test
|
||||
def separate_keystone_service_shutdown(self):
|
||||
"""Shutdown one keystone node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_keystone_service
|
||||
2. Destroy keystone node
|
||||
3. Wait HA is working
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_keystone_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#destroy one keystone node
|
||||
keystone_node = self.env.d_env.nodes().slaves[3]
|
||||
keystone_node.destroy()
|
||||
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
keystone_node)['online'], timeout=60 * 5)
|
||||
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
@test(depends_on=[SeparateKeystone.separate_keystone_service],
|
||||
groups=["separate_keystone_service_restart"])
|
||||
@log_snapshot_after_test
|
||||
def separate_keystone_service_restart(self):
|
||||
"""Restart one keystone node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_keystone_service
|
||||
2. Restart keystone
|
||||
3. Wait HA is working
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_keystone_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#restart one keystone node
|
||||
keystone_node = self.env.d_env.nodes().slaves[3]
|
||||
self.fuel_web.warm_restart_nodes([keystone_node])
|
||||
wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
keystone_node)['online'], timeout=60 * 5)
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
@test(depends_on=[SeparateKeystone.separate_keystone_service],
|
||||
groups=["separate_keystone_service_controller_shutdown"])
|
||||
@log_snapshot_after_test
|
||||
def separate_keystone_service_controller_shutdown(self):
|
||||
"""Shutdown primary controller node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_keystone_service
|
||||
2. Shutdown primary controller node
|
||||
3. Wait HA is working
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_keystone_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#shutdown primary controller
|
||||
controller = self.fuel_web.get_nailgun_primary_node(
|
||||
self.env.d_env.nodes().slaves[0])
|
||||
logger.debug(
|
||||
"controller with primary role is {}".format(controller.name))
|
||||
controller.destroy()
|
||||
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
controller)['online'], timeout=60 * 5)
|
||||
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id, should_fail=1)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id, should_fail=1)
|
||||
|
||||
@test(depends_on=[SeparateKeystone.separate_keystone_service],
|
||||
groups=["separate_keystone_service_add_delete_node"])
|
||||
@log_snapshot_after_test
|
||||
def separate_keystone_service_add_delete_node(self):
|
||||
"""Add and delete keystone node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_keystone_service
|
||||
2. Add one keystone node and re-deploy cluster
|
||||
3. Run network verification
|
||||
4. Run OSTF
|
||||
5. Delete one keystone node
|
||||
6. Run network verification
|
||||
7. Run ostf
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_keystone_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
|
||||
node = {'slave-09': ['keystone']}
|
||||
self.fuel_web.update_nodes(
|
||||
cluster_id, node, True, False)
|
||||
|
||||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self.fuel_web.run_ostf(cluster_id=cluster_id,
|
||||
test_sets=['sanity', 'smoke', 'ha'])
|
||||
|
||||
nailgun_node = self.fuel_web.update_nodes(cluster_id, node,
|
||||
False, True)
|
||||
nodes = filter(lambda x: x["pending_deletion"] is True, nailgun_node)
|
||||
self.fuel_web.deploy_cluster(cluster_id)
|
||||
wait(lambda: self.fuel_web.is_node_discovered(nodes[0]),
|
||||
timeout=6 * 60)
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self.fuel_web.run_ostf(cluster_id=cluster_id,
|
||||
test_sets=['sanity', 'smoke', 'ha'])
|
||||
@@ -0,0 +1,201 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
from proboscis import test
|
||||
from proboscis.asserts import assert_true
|
||||
from devops.helpers.helpers import wait
|
||||
|
||||
from fuelweb_test.helpers import checkers
|
||||
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
||||
from fuelweb_test import settings
|
||||
from fuelweb_test import logger
|
||||
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
||||
from fuelweb_test.tests.base_test_case import TestBasic
|
||||
|
||||
|
||||
@test(groups=["thread_separate_services", "thread_2_separate_services"])
|
||||
class SeparateAllServices(TestBasic):
|
||||
"""SeparateAllServices""" # TODO documentation
|
||||
|
||||
@test(depends_on=[SetupEnvironment.prepare_slaves_9],
|
||||
groups=["separate_all_service"])
|
||||
@log_snapshot_after_test
|
||||
def separate_all_service(self):
|
||||
"""Deploy cluster with 3 nodes with db, keystone, rabbit, horizon
|
||||
|
||||
Scenario:
|
||||
1. Create cluster
|
||||
2. Add 3 nodes with controller role
|
||||
3. Add 3 nodes with database, keystone, rabbit,
|
||||
horizon
|
||||
4. Add 1 compute and cinder
|
||||
5. Verify networks
|
||||
6. Deploy the cluster
|
||||
7. Verify networks
|
||||
8. Run OSTF
|
||||
|
||||
Duration 120m
|
||||
Snapshot separate_all_service
|
||||
"""
|
||||
self.check_run("separate_all_service")
|
||||
self.env.revert_snapshot("ready_with_9_slaves")
|
||||
|
||||
# copy plugins to the master node
|
||||
|
||||
checkers.upload_tarball(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
settings.SEPARATE_SERVICE_DB_PLUGIN_PATH, "/var")
|
||||
|
||||
checkers.upload_tarball(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
settings.SEPARATE_SERVICE_RABBIT_PLUGIN_PATH, "/var")
|
||||
|
||||
checkers.upload_tarball(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
settings.SEPARATE_SERVICE_KEYSTONE_PLUGIN_PATH, "/var")
|
||||
|
||||
# install plugins
|
||||
|
||||
checkers.install_plugin_check_code(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
plugin=os.path.basename(settings.SEPARATE_SERVICE_DB_PLUGIN_PATH))
|
||||
|
||||
checkers.install_plugin_check_code(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
plugin=os.path.basename(
|
||||
settings.SEPARATE_SERVICE_RABBIT_PLUGIN_PATH))
|
||||
|
||||
checkers.install_plugin_check_code(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
plugin=os.path.basename(
|
||||
settings.SEPARATE_SERVICE_KEYSTONE_PLUGIN_PATH))
|
||||
|
||||
data = {
|
||||
'tenant': 'separateall',
|
||||
'user': 'separateall',
|
||||
'password': 'separateall',
|
||||
"net_provider": 'neutron',
|
||||
"net_segment_type": settings.NEUTRON_SEGMENT['vlan'],
|
||||
}
|
||||
|
||||
cluster_id = self.fuel_web.create_cluster(
|
||||
name=self.__class__.__name__,
|
||||
mode=settings.DEPLOYMENT_MODE,
|
||||
settings=data)
|
||||
|
||||
plugin_names = ['detach-database', 'detach-keystone',
|
||||
'detach-rabbitmq']
|
||||
msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
|
||||
for plugin_name in plugin_names:
|
||||
assert_true(
|
||||
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
|
||||
msg)
|
||||
options = {'metadata/enabled': True}
|
||||
self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)
|
||||
|
||||
self.fuel_web.update_nodes(
|
||||
cluster_id,
|
||||
{
|
||||
'slave-01': ['controller'],
|
||||
'slave-02': ['controller'],
|
||||
'slave-03': ['controller'],
|
||||
'slave-04': ['standalone-database', 'standalone-rabbitmq',
|
||||
'standalone-keystone'],
|
||||
'slave-05': ['standalone-database', 'standalone-keystone',
|
||||
'standalone-rabbitmq'],
|
||||
'slave-06': ['standalone-database', 'standalone-keystone',
|
||||
'standalone-rabbitmq'],
|
||||
'slave-07': ['compute'],
|
||||
'slave-08': ['cinder']
|
||||
}
|
||||
)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
|
||||
# Cluster deploy
|
||||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
self.env.make_snapshot("separate_all_service", is_make=True)
|
||||
|
||||
|
||||
@test(groups=["thread_separate_services", "thread_2_separate_services"])
|
||||
class SeparateAllFailover(TestBasic):
|
||||
"""SeparateAllFailover""" # TODO documentation
|
||||
|
||||
@test(depends_on=[SeparateAllServices.separate_all_service],
|
||||
groups=["separate_all_service_shutdown"])
|
||||
@log_snapshot_after_test
|
||||
def separate_all_service_shutdown(self):
|
||||
"""Shutdown one multirole node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_all_service
|
||||
2. Destroy multirole node with rabbit master
|
||||
3. Wait HA is working
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
Snapshot None
|
||||
"""
|
||||
self.env.revert_snapshot("separate_all_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#destroy node with rabbit master
|
||||
all_node = self.fuel_web.get_rabbit_master_node(
|
||||
self.env.d_env.nodes().slaves[3].name)
|
||||
all_node.destroy()
|
||||
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
all_node)['online'], timeout=60 * 5)
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
@test(depends_on=[SeparateAllServices.separate_all_service],
|
||||
groups=["separate_all_service_controller_shutdown"])
|
||||
@log_snapshot_after_test
|
||||
def separate_all_service_controller_shutdown(self):
|
||||
"""Shutdown primary controller node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_all_service
|
||||
2. Shutdown primary controller node
|
||||
3. Wait HA is working
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
Snapshot None
|
||||
"""
|
||||
self.env.revert_snapshot("separate_all_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#shutdown primary controller
|
||||
controller = self.fuel_web.get_nailgun_primary_node(
|
||||
self.env.d_env.nodes().slaves[0])
|
||||
logger.debug(
|
||||
"controller with primary role is {}".format(controller.name))
|
||||
controller.destroy()
|
||||
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
controller)['online'], timeout=60 * 5)
|
||||
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id, should_fail=1)
|
||||
@@ -0,0 +1,245 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
from proboscis.asserts import assert_true
|
||||
from proboscis import test
|
||||
from devops.helpers.helpers import wait
|
||||
|
||||
from fuelweb_test.helpers import checkers
|
||||
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
||||
from fuelweb_test import settings
|
||||
from fuelweb_test import logger
|
||||
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
||||
from fuelweb_test.tests.base_test_case import TestBasic
|
||||
|
||||
|
||||
@test(groups=["thread_separate_services", "thread_2_separate_services"])
|
||||
class SeparateRabbit(TestBasic):
|
||||
"""SeparateRabbit""" # TODO documentation
|
||||
|
||||
@test(depends_on=[SetupEnvironment.prepare_slaves_9],
|
||||
groups=["separate_rabbit_service"])
|
||||
@log_snapshot_after_test
|
||||
def separate_rabbit_service(self):
|
||||
"""Deploy cluster with 3 separate rabbit roles
|
||||
|
||||
Scenario:
|
||||
1. Create cluster
|
||||
2. Add 3 nodes with controller role
|
||||
3. Add 3 nodes with rabbit role
|
||||
4. Add 1 compute and cinder
|
||||
5. Verify networks
|
||||
6. Deploy the cluster
|
||||
7. Verify networks
|
||||
8. Run OSTF
|
||||
|
||||
Duration 120m
|
||||
Snapshot separate_rabbit_service
|
||||
"""
|
||||
self.check_run("separate_rabbit_service")
|
||||
self.env.revert_snapshot("ready_with_9_slaves")
|
||||
|
||||
# copy plugins to the master node
|
||||
|
||||
checkers.upload_tarball(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
settings.SEPARATE_SERVICE_RABBIT_PLUGIN_PATH, "/var")
|
||||
|
||||
# install plugins
|
||||
|
||||
checkers.install_plugin_check_code(
|
||||
self.env.d_env.get_admin_remote(),
|
||||
plugin=os.path.basename(
|
||||
settings.SEPARATE_SERVICE_RABBIT_PLUGIN_PATH))
|
||||
|
||||
data = {
|
||||
'tenant': 'separaterabbit',
|
||||
'user': 'separaterabbit',
|
||||
'password': 'separaterabbit',
|
||||
"net_provider": 'neutron',
|
||||
"net_segment_type": settings.NEUTRON_SEGMENT['vlan'],
|
||||
}
|
||||
|
||||
cluster_id = self.fuel_web.create_cluster(
|
||||
name=self.__class__.__name__,
|
||||
mode=settings.DEPLOYMENT_MODE,
|
||||
settings=data)
|
||||
|
||||
plugin_name = 'detach-rabbitmq'
|
||||
msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
|
||||
assert_true(
|
||||
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
|
||||
msg)
|
||||
options = {'metadata/enabled': True}
|
||||
self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)
|
||||
|
||||
self.fuel_web.update_nodes(
|
||||
cluster_id,
|
||||
{
|
||||
'slave-01': ['controller'],
|
||||
'slave-02': ['controller'],
|
||||
'slave-03': ['controller'],
|
||||
'slave-04': ['standalone-rabbitmq'],
|
||||
'slave-05': ['standalone-rabbitmq'],
|
||||
'slave-06': ['standalone-rabbitmq'],
|
||||
'slave-07': ['compute'],
|
||||
'slave-08': ['cinder']
|
||||
}
|
||||
)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
|
||||
# Cluster deploy
|
||||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
self.env.make_snapshot("separate_rabbit_service", is_make=True)
|
||||
|
||||
|
||||
@test(groups=["thread_separate_services", "thread_2_separate_services"])
|
||||
class SeparateRabbitFailover(TestBasic):
|
||||
"""SeparateRabbitFailover""" # TODO documentation
|
||||
|
||||
@test(depends_on=[SeparateRabbit.separate_rabbit_service],
|
||||
groups=["separate_rabbit_service_shutdown"])
|
||||
@log_snapshot_after_test
|
||||
def separate_rabbit_service_shutdown(self):
|
||||
"""Shutdown one rabbit node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_rabbit_service
|
||||
2. Destroy rabbit node that is master
|
||||
3. Wait HA is working
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_rabbit_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#destroy master rabbit node
|
||||
rabbit_node = self.fuel_web.get_rabbit_master_node(
|
||||
self.env.d_env.nodes().slaves[3].name)
|
||||
rabbit_node.destroy()
|
||||
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
rabbit_node)['online'], timeout=60 * 5)
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
@test(depends_on=[SeparateRabbit.separate_rabbit_service],
|
||||
groups=["separate_rabbit_service_restart"])
|
||||
@log_snapshot_after_test
|
||||
def separate_rabbit_service_restart(self):
|
||||
"""Restart one rabbit node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_rabbit_service
|
||||
2. Restart rabbit node that is master
|
||||
3. Wait HA is working
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_rabbit_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#restart rabbit master node
|
||||
rabbit_node = self.fuel_web.get_rabbit_master_node(
|
||||
self.env.d_env.nodes().slaves[3].name)
|
||||
self.fuel_web.warm_restart_nodes([rabbit_node])
|
||||
wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
rabbit_node)['online'], timeout=60 * 5)
|
||||
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id)
|
||||
|
||||
@test(depends_on=[SeparateRabbit.separate_rabbit_service],
|
||||
groups=["separate_rabbit_service_controller_shutdown"])
|
||||
@log_snapshot_after_test
|
||||
def separate_rabbit_service_controller_shutdown(self):
|
||||
"""Shutdown primary controller node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_rabbit_service
|
||||
2. Shutdown primary controller node
|
||||
3. Wait HA is working
|
||||
4. Run OSTF
|
||||
|
||||
Duration 30m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_rabbit_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
#shutdown primary controller
|
||||
controller = self.fuel_web.get_nailgun_primary_node(
|
||||
self.env.d_env.nodes().slaves[0])
|
||||
logger.debug(
|
||||
"controller with primary role is {}".format(controller.name))
|
||||
controller.destroy()
|
||||
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
|
||||
controller)['online'], timeout=60 * 5)
|
||||
|
||||
self.fuel_web.assert_ha_services_ready(cluster_id)
|
||||
self.fuel_web.assert_os_services_ready(cluster_id, should_fail=1)
|
||||
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id, should_fail=1)
|
||||
|
||||
@test(depends_on=[SeparateRabbit.separate_rabbit_service],
|
||||
groups=["separate_rabbit_service_add_delete_node"])
|
||||
@log_snapshot_after_test
|
||||
def separate_rabbit_service_add_delete_node(self):
|
||||
"""Add and delete rabbit node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot separate_rabbit_service
|
||||
2. Add one rabbit node and re-deploy cluster
|
||||
3. Run network verification
|
||||
4. Run OSTF
|
||||
5. Delete one rabbit node
|
||||
6. Run network verification
|
||||
7. Run ostf
|
||||
|
||||
Duration 120m
|
||||
"""
|
||||
self.env.revert_snapshot("separate_rabbit_service")
|
||||
cluster_id = self.fuel_web.get_last_created_cluster()
|
||||
|
||||
node = {'slave-09': ['standalone-rabbitmq']}
|
||||
self.fuel_web.update_nodes(
|
||||
cluster_id, node, True, False)
|
||||
|
||||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self.fuel_web.run_ostf(cluster_id=cluster_id,
|
||||
test_sets=['sanity', 'smoke', 'ha'])
|
||||
|
||||
nailgun_node = self.fuel_web.update_nodes(cluster_id, node,
|
||||
False, True)
|
||||
nodes = filter(lambda x: x["pending_deletion"] is True, nailgun_node)
|
||||
self.fuel_web.deploy_cluster(cluster_id)
|
||||
wait(lambda: self.fuel_web.is_node_discovered(nodes[0]),
|
||||
timeout=6 * 60)
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self.fuel_web.run_ostf(cluster_id=cluster_id,
|
||||
test_sets=['sanity', 'smoke', 'ha'])
|
||||
Reference in New Issue
Block a user