Merge "Add galera tests"
This commit is contained in:
commit
2cc97d84ab
|
@ -12,10 +12,14 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
from copy import deepcopy
|
||||||
|
import os
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from fuel_ccp_tests import logger
|
from fuel_ccp_tests import logger
|
||||||
|
from fuel_ccp_tests import settings
|
||||||
from fuel_ccp_tests.helpers import ext
|
from fuel_ccp_tests.helpers import ext
|
||||||
|
from fuel_ccp_tests.helpers import post_os_deploy_checks
|
||||||
from fuel_ccp_tests.managers.osmanager import OSManager
|
from fuel_ccp_tests.managers.osmanager import OSManager
|
||||||
|
|
||||||
LOG = logger.logger
|
LOG = logger.logger
|
||||||
|
@ -43,3 +47,49 @@ def os_deployed(ccpcluster,
|
||||||
else:
|
else:
|
||||||
LOG.info("Openstack allready installed and running...")
|
LOG.info("Openstack allready installed and running...")
|
||||||
osmanager.check_os_ready()
|
osmanager.check_os_ready()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.revert_snapshot(ext.SNAPSHOT.os_galera_deployed)
|
||||||
|
@pytest.fixture(scope='function')
|
||||||
|
def galera_deployed(ccpcluster,
|
||||||
|
hardware,
|
||||||
|
underlay,
|
||||||
|
revert_snapshot,
|
||||||
|
config,
|
||||||
|
k8s_actions):
|
||||||
|
"""Deploy galera cluster
|
||||||
|
"""
|
||||||
|
# If no snapshot was reverted, then try to revert the snapshot
|
||||||
|
# that belongs to the fixture.
|
||||||
|
# Note: keep fixtures in strict dependences from each other!
|
||||||
|
if not config.os.running:
|
||||||
|
general_config = deepcopy(settings.CCP_CONF)
|
||||||
|
if settings.BUILD_IMAGES:
|
||||||
|
k8s_actions.create_registry()
|
||||||
|
ccpcluster.build()
|
||||||
|
topology_path = \
|
||||||
|
os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
|
||||||
|
'3galera_1comp.yaml'
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
remote.upload(topology_path, '/tmp')
|
||||||
|
ccpcluster.put_yaml_config('./config_1.yaml', general_config)
|
||||||
|
ccpcluster.add_includes('./config_1.yaml', [
|
||||||
|
settings.CCP_DEPLOY_CONFIG,
|
||||||
|
settings.CCP_SOURCES_CONFIG,
|
||||||
|
'/tmp/3galera_1comp.yaml'])
|
||||||
|
|
||||||
|
underlay.sudo_check_call("pip install python-openstackclient",
|
||||||
|
host=config.k8s.kube_host)
|
||||||
|
ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
|
||||||
|
use_cli_params=True)
|
||||||
|
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
|
||||||
|
post_os_deploy_checks.check_pods_status(k8s_actions.api)
|
||||||
|
# todo: add invocation of galera checker script
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(
|
||||||
|
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||||
|
timeout=600)
|
||||||
|
|
||||||
|
config.os.running = True
|
||||||
|
hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed)
|
||||||
|
|
|
@ -42,7 +42,8 @@ SNAPSHOT = enum(
|
||||||
'k8s_deployed',
|
'k8s_deployed',
|
||||||
'ccp_deployed',
|
'ccp_deployed',
|
||||||
'os_deployed',
|
'os_deployed',
|
||||||
'os_deployed_stacklight'
|
'os_deployed_stacklight',
|
||||||
|
'os_galera_deployed',
|
||||||
)
|
)
|
||||||
|
|
||||||
LOG_LEVELS = enum(
|
LOG_LEVELS = enum(
|
||||||
|
|
|
@ -198,10 +198,10 @@ class CCPManager(object):
|
||||||
components=components,
|
components=components,
|
||||||
params=params, suppress_output=suppress_output)
|
params=params, suppress_output=suppress_output)
|
||||||
|
|
||||||
def deploy(self, components=None, params=None):
|
def deploy(self, components=None, params=None, use_cli_params=False):
|
||||||
self.run('deploy',
|
self.run('deploy',
|
||||||
components=components,
|
components=components,
|
||||||
params=params)
|
params=params, use_cli_params=use_cli_params)
|
||||||
|
|
||||||
def dry_deploy(self, export_dir, components=None, params=None):
|
def dry_deploy(self, export_dir, components=None, params=None):
|
||||||
self.run('deploy --dry-run --export-dir={export_dir}'.format(
|
self.run('deploy --dry-run --export-dir={export_dir}'.format(
|
||||||
|
|
|
@ -470,3 +470,47 @@ class EnvironmentManager(object):
|
||||||
self.__config.underlay.nameservers = [self.nameserver]
|
self.__config.underlay.nameservers = [self.nameserver]
|
||||||
if not self.__config.underlay.upstream_dns_servers:
|
if not self.__config.underlay.upstream_dns_servers:
|
||||||
self.__config.underlay.upstream_dns_servers = [self.nameserver]
|
self.__config.underlay.upstream_dns_servers = [self.nameserver]
|
||||||
|
|
||||||
|
def get_node_by_ip(self, node_ip):
|
||||||
|
nodes = self._env.get_nodes()
|
||||||
|
node = [node for node in nodes if self.node_ip(node) == node_ip]
|
||||||
|
assert len(node) == 1, "Node with {} ip isn't found".format(node_ip)
|
||||||
|
return node[0]
|
||||||
|
|
||||||
|
def shutdown_node_by_ip(self, node_ip):
|
||||||
|
"""Shutdown hardware node by ip address
|
||||||
|
|
||||||
|
"""
|
||||||
|
node = self.get_node_by_ip(node_ip)
|
||||||
|
node.shutdown()
|
||||||
|
|
||||||
|
def start_node_by_ip(self, node_ip):
|
||||||
|
"""Start hardware node by ip address
|
||||||
|
|
||||||
|
"""
|
||||||
|
node = self.get_node_by_ip(node_ip)
|
||||||
|
node.start()
|
||||||
|
|
||||||
|
def wait_node_is_offline(self, node_ip, timeout):
|
||||||
|
"""Wait node is shutdown and doesn't respond
|
||||||
|
|
||||||
|
"""
|
||||||
|
helpers.wait(
|
||||||
|
lambda: not helpers.tcp_ping(node_ip, 22),
|
||||||
|
timeout=timeout,
|
||||||
|
timeout_msg="Node '{}' didn't go offline after {} sec".format(
|
||||||
|
node_ip, timeout
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def wait_node_is_online(self, node_ip, timeout):
|
||||||
|
"""Wait node is online after starting
|
||||||
|
|
||||||
|
"""
|
||||||
|
helpers.wait(
|
||||||
|
lambda: helpers.tcp_ping(node_ip, 22),
|
||||||
|
timeout=timeout,
|
||||||
|
timeout_msg="Node '{}' didn't become online after {} sec".format(
|
||||||
|
node_ip, timeout
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
|
@ -343,3 +343,8 @@ class K8SManager(object):
|
||||||
self.__config.underlay.upstream_dns_servers
|
self.__config.underlay.upstream_dns_servers
|
||||||
LOG.info('Added custom upstream DNS servers (dnsmasq) to the '
|
LOG.info('Added custom upstream DNS servers (dnsmasq) to the '
|
||||||
'settings: {0}'.format(k8s_settings['nameservers']))
|
'settings: {0}'.format(k8s_settings['nameservers']))
|
||||||
|
|
||||||
|
def get_pods_number(self, pod_name, namespace=None):
|
||||||
|
pods = [pod for pod in self.api.pods.list(namespace=namespace)
|
||||||
|
if pod_name in pod.name]
|
||||||
|
return len(pods)
|
||||||
|
|
|
@ -0,0 +1,41 @@
|
||||||
|
nodes:
|
||||||
|
node1:
|
||||||
|
roles:
|
||||||
|
- openvswitch
|
||||||
|
- controller-net-host
|
||||||
|
- controller-net-bridge
|
||||||
|
node[2-3]:
|
||||||
|
roles:
|
||||||
|
- openvswitch
|
||||||
|
- controller-net-bridge
|
||||||
|
- compute
|
||||||
|
node[1-3]:
|
||||||
|
roles:
|
||||||
|
- galera
|
||||||
|
roles:
|
||||||
|
controller-net-host:
|
||||||
|
- neutron-dhcp-agent
|
||||||
|
- neutron-l3-agent
|
||||||
|
- neutron-metadata-agent
|
||||||
|
controller-net-bridge:
|
||||||
|
- etcd
|
||||||
|
- glance-api
|
||||||
|
- glance-registry
|
||||||
|
- keystone
|
||||||
|
- memcached
|
||||||
|
- neutron-server
|
||||||
|
- nova-api
|
||||||
|
- nova-conductor
|
||||||
|
- nova-consoleauth
|
||||||
|
- nova-novncproxy
|
||||||
|
- nova-scheduler
|
||||||
|
- rabbitmq
|
||||||
|
compute:
|
||||||
|
- nova-compute
|
||||||
|
- nova-libvirt
|
||||||
|
openvswitch:
|
||||||
|
- neutron-openvswitch-agent
|
||||||
|
- openvswitch-db
|
||||||
|
- openvswitch-vswitchd
|
||||||
|
galera:
|
||||||
|
- galera
|
|
@ -0,0 +1,261 @@
|
||||||
|
# Copyright 2016 Mirantis, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import base_test
|
||||||
|
from fuel_ccp_tests import logger
|
||||||
|
from fuel_ccp_tests import settings
|
||||||
|
from fuel_ccp_tests.helpers import post_os_deploy_checks
|
||||||
|
|
||||||
|
LOG = logger.logger
|
||||||
|
|
||||||
|
|
||||||
|
class TestGalera(base_test.SystemBaseTest):
|
||||||
|
""" Galera scale and destructive scenarios
|
||||||
|
|
||||||
|
"""
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.galera_shutdown
|
||||||
|
@pytest.mark.galera
|
||||||
|
def test_galera_shutdown_node(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
galera_deployed):
|
||||||
|
"""Shutdown galera node
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed galera
|
||||||
|
2. Shutdown one galera node
|
||||||
|
3. Check galera state
|
||||||
|
4. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
show_step(2)
|
||||||
|
hardware.shutdown_node_by_ip(underlay.node_names()[1])
|
||||||
|
show_step(3)
|
||||||
|
# todo: add wait for galera to assemble when galera_checker is ready
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(4)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(
|
||||||
|
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||||
|
timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.galera_cold_restart
|
||||||
|
@pytest.mark.galera
|
||||||
|
def test_galera_cold_restart_node(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
galera_deployed):
|
||||||
|
"""Cold restart galera node
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed galera
|
||||||
|
2. Cold restart one galera node
|
||||||
|
3. Check galera state
|
||||||
|
4. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
show_step(2)
|
||||||
|
hardware.shutdown_node_by_ip(underlay.node_names()[1])
|
||||||
|
hardware.start_node_by_ip(underlay.host_by_node_name('slave-0'))
|
||||||
|
show_step(3)
|
||||||
|
# todo: add wait for galera to assemble when galera_checker is ready
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(4)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(
|
||||||
|
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||||
|
timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.galera_poweroff
|
||||||
|
@pytest.mark.galera
|
||||||
|
def test_galera_poweroff_node(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
galera_deployed):
|
||||||
|
"""Poweroff galera node
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed galera
|
||||||
|
2. Poweroff one galera node
|
||||||
|
3. Check galera state
|
||||||
|
4. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
galera_node = underlay.node_names()[1]
|
||||||
|
galera_node_ip = underlay.host_by_node_name(galera_node)
|
||||||
|
show_step(2)
|
||||||
|
underlay.sudo_check_call('shutdown +1', node_name=galera_node)
|
||||||
|
hardware.shutdown_node_by_ip(galera_node_ip)
|
||||||
|
hardware.wait_node_is_offline(galera_node_ip, 90)
|
||||||
|
show_step(3)
|
||||||
|
# todo: add wait for galera to assemble when galera_checker is ready
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(4)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(
|
||||||
|
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||||
|
timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.galera_soft_reboot
|
||||||
|
@pytest.mark.galera
|
||||||
|
def test_galera_soft_reboot_node(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
galera_deployed):
|
||||||
|
"""Soft reboot galera node
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed galera
|
||||||
|
2. Soft reboot one galera node
|
||||||
|
3. Check galera state
|
||||||
|
4. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
galera_node = underlay.node_names()[1]
|
||||||
|
galera_node_ip = underlay.host_by_node_name(galera_node)
|
||||||
|
show_step(2)
|
||||||
|
underlay.sudo_check_call('shutdown +1', node_name=galera_node)
|
||||||
|
hardware.shutdown_node_by_ip(galera_node_ip)
|
||||||
|
hardware.wait_node_is_offline(galera_node_ip, 90)
|
||||||
|
hardware.start_node_by_ip(galera_node_ip)
|
||||||
|
hardware.wait_node_is_online(galera_node_ip, 180)
|
||||||
|
show_step(3)
|
||||||
|
# todo: add wait for galera to assemble when galera_checker is ready
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(4)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(
|
||||||
|
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||||
|
timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.galera_cluster_shutdown
|
||||||
|
@pytest.mark.galera
|
||||||
|
def test_galera_cluster_shutdown(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
galera_deployed):
|
||||||
|
"""Galera cluster shutdown
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed galera
|
||||||
|
2. Shutdown all galera nodes and start them one by one
|
||||||
|
3. Check galera state
|
||||||
|
4. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
galera_nodes = underlay.node_names()[:3]
|
||||||
|
galera_node_ips = []
|
||||||
|
show_step(2)
|
||||||
|
for galera_node in galera_nodes:
|
||||||
|
galera_node_ip = underlay.host_by_node_name(galera_node)
|
||||||
|
galera_node_ips.append(galera_node_ip)
|
||||||
|
hardware.shutdown_node_by_ip(galera_node_ip)
|
||||||
|
hardware.wait_node_is_offline(galera_node_ip, 90)
|
||||||
|
for galera_ip in galera_node_ips:
|
||||||
|
hardware.start_node_by_ip(galera_ip)
|
||||||
|
hardware.wait_node_is_online(galera_ip, 180)
|
||||||
|
show_step(3)
|
||||||
|
# todo: add wait for galera to assemble when galera_checker is ready
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(4)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(
|
||||||
|
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||||
|
timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.galera_scale_up_down
|
||||||
|
@pytest.mark.galera
|
||||||
|
def test_galera_scale(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
galera_deployed):
|
||||||
|
"""Galera cluster scale
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed galera
|
||||||
|
2. Scale up galera to 5 replicas
|
||||||
|
3. Check galera state
|
||||||
|
4. Check number of galera pods
|
||||||
|
5. Create 2 vms
|
||||||
|
6. Scale down galera to 3 replicas
|
||||||
|
7. Check galera state
|
||||||
|
8. Check number of galera pods
|
||||||
|
9. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
show_step(2)
|
||||||
|
with underlay.yaml_editor('/tmp/3galera_1comp.yaml',
|
||||||
|
host=config.k8s.kube_host) as editor:
|
||||||
|
del editor.content['nodes']['node[1-3]']
|
||||||
|
editor.content['nodes']['node[1-5]'] = {'roles': ['galera']}
|
||||||
|
|
||||||
|
ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
|
||||||
|
use_cli_params=True)
|
||||||
|
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
|
||||||
|
post_os_deploy_checks.check_pods_status(k8s_actions.api)
|
||||||
|
show_step(3)
|
||||||
|
# todo: add invocation of galera checker script
|
||||||
|
show_step(4)
|
||||||
|
galera_pods = \
|
||||||
|
k8s_actions.get_pods_number('galera',
|
||||||
|
settings.
|
||||||
|
CCP_CONF['kubernetes']['namespace'])
|
||||||
|
assert galera_pods == 5,\
|
||||||
|
"Expcted tp have 5 galera pods, got {}".format(galera_pods)
|
||||||
|
|
||||||
|
show_step(5)
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(
|
||||||
|
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||||
|
timeout=600)
|
||||||
|
show_step(6)
|
||||||
|
with underlay.yaml_editor('/tmp/3galera_1comp.yaml',
|
||||||
|
host=config.k8s.kube_host) as editor:
|
||||||
|
del editor.content['nodes']['node[1-5]']
|
||||||
|
editor.content['nodes']['node[1-3]'] = {'roles': ['galera']}
|
||||||
|
|
||||||
|
ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
|
||||||
|
use_cli_params=True)
|
||||||
|
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
|
||||||
|
post_os_deploy_checks.check_pods_status(k8s_actions.api)
|
||||||
|
show_step(7)
|
||||||
|
# todo: add invocation of galera checker script
|
||||||
|
show_step(8)
|
||||||
|
galera_pods = \
|
||||||
|
k8s_actions.get_pods_number('galera',
|
||||||
|
settings.
|
||||||
|
CCP_CONF['kubernetes']['namespace'])
|
||||||
|
assert galera_pods == 3,\
|
||||||
|
"Expcted tp have 3 galera pods, got {}".format(galera_pods)
|
||||||
|
show_step(9)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(
|
||||||
|
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||||
|
timeout=600)
|
Loading…
Reference in New Issue