Add iptables-based active/standby scenario test
This patch adds a tempest scenario test for active/standby topology. This scenario takes a similar approach to scenario proposed in Change-Id Ibcd5552a67cea650edc72bfaa986357267ca2407 with the difference that it does not rely on amphora stats API. Instead, it uses iptables to log VIP traffic and make assertions based on logging. Co-Authored-By: Michael Johnson <johnsomor@gmail.com> Change-Id: I24a6fc3387166ec6cdbd57a5ca9f63743748ec68
This commit is contained in:
Carlos Goncalves
parent
c9414e8419
commit
c2e1216e8e
|
|
@ -84,6 +84,7 @@ FLAVOR_PROFILE_ID = 'flavor_profile_id'
|
|||
|
||||
# Other constants
|
||||
ACTIVE = 'ACTIVE'
|
||||
PENDING_UPDATE = 'PENDING_UPDATE'
|
||||
ADMIN_STATE_UP_TRUE = 'true'
|
||||
ASC = 'asc'
|
||||
DELETED = 'DELETED'
|
||||
|
|
|
|||
|
|
@ -151,8 +151,15 @@ OctaviaGroup = [
|
|||
cfg.StrOpt('member_2_ipv6_subnet_cidr',
|
||||
default='fd77:1457:4cf0:26a8::/64',
|
||||
help='CIDR format subnet to use for the member 1 ipv6 subnet.'),
|
||||
# Amphora specific options
|
||||
cfg.StrOpt('amphora_ssh_user',
|
||||
default='ubuntu',
|
||||
help='The amphora SSH user.'),
|
||||
cfg.StrOpt('amphora_ssh_key',
|
||||
default='/tmp/octavia_ssh_key',
|
||||
help='The amphora SSH key file.'),
|
||||
# Environment specific options
|
||||
# These are used to accomidate clouds with specific limitations
|
||||
# These are used to accomodate clouds with specific limitations
|
||||
cfg.IntOpt('random_server_name_length',
|
||||
default=0,
|
||||
help='If non-zero, generate a random name of the length '
|
||||
|
|
|
|||
|
|
@ -0,0 +1,303 @@
|
|||
# Copyright 2019 Rackspace US Inc. All rights reserved.
|
||||
# Copyright 2019 Red Hat Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import testtools
|
||||
|
||||
from oslo_log import log as logging
|
||||
from tempest import config
|
||||
from tempest.lib.common.utils import data_utils
|
||||
from tempest.lib.common.utils.linux import remote_client
|
||||
from tempest.lib import decorators
|
||||
from tempest.lib import exceptions
|
||||
|
||||
from octavia_tempest_plugin.common import constants as const
|
||||
from octavia_tempest_plugin.tests import test_base
|
||||
from octavia_tempest_plugin.tests import waiters
|
||||
|
||||
CONF = config.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@testtools.skipUnless(
|
||||
CONF.validation.run_validation,
|
||||
'Active-Standby tests will not work without run_validation enabled.')
|
||||
class ActiveStandbyIptablesScenarioTest(
|
||||
test_base.LoadBalancerBaseTestWithCompute):
|
||||
|
||||
@classmethod
|
||||
def skip_checks(cls):
|
||||
super(ActiveStandbyIptablesScenarioTest, cls).skip_checks()
|
||||
|
||||
if CONF.load_balancer.provider not in ['amphora', 'octavia']:
|
||||
raise cls.skipException("Amphora tests require provider 'amphora' "
|
||||
"or 'octavia' (alias to 'amphora', "
|
||||
"deprecated) set.")
|
||||
|
||||
@classmethod
|
||||
def resource_setup(cls):
|
||||
"""Setup resources needed by the tests."""
|
||||
super(ActiveStandbyIptablesScenarioTest, cls).resource_setup()
|
||||
|
||||
lb_name = data_utils.rand_name("lb_member_lb1_actstdby")
|
||||
lb_kwargs = {const.PROVIDER: CONF.load_balancer.provider,
|
||||
const.NAME: lb_name}
|
||||
|
||||
# TODO(rm_work): Make this work with ipv6 and split this test for both
|
||||
ip_version = 4
|
||||
cls._setup_lb_network_kwargs(lb_kwargs, ip_version)
|
||||
|
||||
lb = cls.mem_lb_client.create_loadbalancer(**lb_kwargs)
|
||||
cls.lb_id = lb[const.ID]
|
||||
cls.addClassResourceCleanup(
|
||||
cls.mem_lb_client.cleanup_loadbalancer,
|
||||
cls.lb_id)
|
||||
|
||||
if CONF.validation.connect_method == 'floating':
|
||||
port_id = lb[const.VIP_PORT_ID]
|
||||
result = cls.lb_mem_float_ip_client.create_floatingip(
|
||||
floating_network_id=CONF.network.public_network_id,
|
||||
port_id=port_id)
|
||||
floating_ip = result['floatingip']
|
||||
LOG.info('lb1_floating_ip: {}'.format(floating_ip))
|
||||
cls.addClassResourceCleanup(
|
||||
waiters.wait_for_not_found,
|
||||
cls.lb_mem_float_ip_client.delete_floatingip,
|
||||
cls.lb_mem_float_ip_client.show_floatingip,
|
||||
floatingip_id=floating_ip['id'])
|
||||
cls.lb_vip_address = floating_ip['floating_ip_address']
|
||||
else:
|
||||
cls.lb_vip_address = lb[const.VIP_ADDRESS]
|
||||
|
||||
waiters.wait_for_status(cls.mem_lb_client.show_loadbalancer,
|
||||
cls.lb_id, const.PROVISIONING_STATUS,
|
||||
const.ACTIVE,
|
||||
CONF.load_balancer.lb_build_interval,
|
||||
CONF.load_balancer.lb_build_timeout)
|
||||
|
||||
listener_name = data_utils.rand_name("lb_member_listener1_actstdby")
|
||||
listener_kwargs = {
|
||||
const.NAME: listener_name,
|
||||
const.PROTOCOL: const.HTTP,
|
||||
const.PROTOCOL_PORT: '80',
|
||||
const.LOADBALANCER_ID: cls.lb_id,
|
||||
}
|
||||
listener = cls.mem_listener_client.create_listener(**listener_kwargs)
|
||||
cls.listener_id = listener[const.ID]
|
||||
cls.addClassResourceCleanup(
|
||||
cls.mem_listener_client.cleanup_listener,
|
||||
cls.listener_id,
|
||||
lb_client=cls.mem_lb_client, lb_id=cls.lb_id)
|
||||
|
||||
waiters.wait_for_status(cls.mem_lb_client.show_loadbalancer,
|
||||
cls.lb_id, const.PROVISIONING_STATUS,
|
||||
const.ACTIVE,
|
||||
CONF.load_balancer.build_interval,
|
||||
CONF.load_balancer.build_timeout)
|
||||
|
||||
pool_name = data_utils.rand_name("lb_member_pool1_actstdby")
|
||||
pool_kwargs = {
|
||||
const.NAME: pool_name,
|
||||
const.PROTOCOL: const.HTTP,
|
||||
const.LB_ALGORITHM: const.LB_ALGORITHM_ROUND_ROBIN,
|
||||
const.LISTENER_ID: cls.listener_id,
|
||||
}
|
||||
pool = cls.mem_pool_client.create_pool(**pool_kwargs)
|
||||
cls.pool_id = pool[const.ID]
|
||||
cls.addClassResourceCleanup(
|
||||
cls.mem_pool_client.cleanup_pool,
|
||||
cls.pool_id,
|
||||
lb_client=cls.mem_lb_client, lb_id=cls.lb_id)
|
||||
|
||||
waiters.wait_for_status(cls.mem_lb_client.show_loadbalancer,
|
||||
cls.lb_id, const.PROVISIONING_STATUS,
|
||||
const.ACTIVE,
|
||||
CONF.load_balancer.build_interval,
|
||||
CONF.load_balancer.build_timeout)
|
||||
|
||||
# Set up Member 1 for Webserver 1
|
||||
member1_name = data_utils.rand_name("lb_member_member1_actstdby")
|
||||
member1_kwargs = {
|
||||
const.POOL_ID: cls.pool_id,
|
||||
const.NAME: member1_name,
|
||||
const.ADMIN_STATE_UP: True,
|
||||
const.ADDRESS: cls.webserver1_ip,
|
||||
const.PROTOCOL_PORT: 80,
|
||||
}
|
||||
if cls.lb_member_1_subnet:
|
||||
member1_kwargs[const.SUBNET_ID] = cls.lb_member_1_subnet[const.ID]
|
||||
|
||||
member1 = cls.mem_member_client.create_member(
|
||||
**member1_kwargs)
|
||||
cls.addClassResourceCleanup(
|
||||
cls.mem_member_client.cleanup_member,
|
||||
member1[const.ID], pool_id=cls.pool_id,
|
||||
lb_client=cls.mem_lb_client, lb_id=cls.lb_id)
|
||||
waiters.wait_for_status(
|
||||
cls.mem_lb_client.show_loadbalancer, cls.lb_id,
|
||||
const.PROVISIONING_STATUS, const.ACTIVE,
|
||||
CONF.load_balancer.check_interval,
|
||||
CONF.load_balancer.check_timeout)
|
||||
|
||||
# Set up Member 2 for Webserver 2
|
||||
member2_name = data_utils.rand_name("lb_member_member2_actstdby")
|
||||
member2_kwargs = {
|
||||
const.POOL_ID: cls.pool_id,
|
||||
const.NAME: member2_name,
|
||||
const.ADMIN_STATE_UP: True,
|
||||
const.ADDRESS: cls.webserver2_ip,
|
||||
const.PROTOCOL_PORT: 80,
|
||||
}
|
||||
if cls.lb_member_2_subnet:
|
||||
member2_kwargs[const.SUBNET_ID] = cls.lb_member_2_subnet[const.ID]
|
||||
|
||||
member2 = cls.mem_member_client.create_member(
|
||||
**member2_kwargs)
|
||||
cls.addClassResourceCleanup(
|
||||
cls.mem_member_client.cleanup_member,
|
||||
member2[const.ID], pool_id=cls.pool_id,
|
||||
lb_client=cls.mem_lb_client, lb_id=cls.lb_id)
|
||||
waiters.wait_for_status(
|
||||
cls.mem_lb_client.show_loadbalancer, cls.lb_id,
|
||||
const.PROVISIONING_STATUS, const.ACTIVE,
|
||||
CONF.load_balancer.check_interval,
|
||||
CONF.load_balancer.check_timeout)
|
||||
|
||||
@classmethod
|
||||
def _log_vip_traffic(cls, amp, log_prefix):
|
||||
ssh_key = cls._get_amphora_ssh_key()
|
||||
linux_client = remote_client.RemoteClient(
|
||||
amp['lb_network_ip'], CONF.load_balancer.amphora_ssh_user,
|
||||
pkey=ssh_key)
|
||||
linux_client.validate_authentication()
|
||||
|
||||
# Allow logging from non-init namespaces
|
||||
# https://lore.kernel.org/patchwork/patch/673714/
|
||||
linux_client.exec_command('echo 1 | sudo tee '
|
||||
'/proc/sys/net/netfilter/nf_log_all_netns')
|
||||
|
||||
linux_client.exec_command('sudo ip netns exec amphora-haproxy '
|
||||
'iptables -I INPUT 1 -d {0} -j LOG '
|
||||
'--log-prefix "{1}"'
|
||||
.format(amp['ha_ip'], log_prefix))
|
||||
|
||||
@classmethod
|
||||
def _has_vip_traffic(cls, ip_address, log_prefix):
|
||||
ssh_key = cls._get_amphora_ssh_key()
|
||||
linux_client = remote_client.RemoteClient(
|
||||
ip_address, CONF.load_balancer.amphora_ssh_user, pkey=ssh_key)
|
||||
linux_client.validate_authentication()
|
||||
|
||||
try:
|
||||
linux_client.exec_command('sudo journalctl -t kernel | grep {0}'
|
||||
.format(log_prefix))
|
||||
return True
|
||||
except exceptions.SSHExecCommandFailed:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _get_active_standby_amps(cls, amps, log_prefix):
|
||||
active = None
|
||||
stby = None
|
||||
for amp in amps:
|
||||
if cls._has_vip_traffic(amp['lb_network_ip'], log_prefix):
|
||||
if active:
|
||||
LOG.exception('Failed to determine single active amphora.')
|
||||
raise Exception('More than one amphora is forwarding VIP '
|
||||
'traffic.')
|
||||
active = amp
|
||||
else:
|
||||
stby = amp
|
||||
|
||||
return active, stby
|
||||
|
||||
@classmethod
|
||||
def _get_amphora_ssh_key(cls):
|
||||
key_file = CONF.load_balancer.amphora_ssh_key
|
||||
if not key_file:
|
||||
raise Exception("SSH key file not provided.")
|
||||
if not os.path.isfile(key_file):
|
||||
raise Exception("Could not find amphora ssh key file {1}."
|
||||
.format(key_file))
|
||||
with open(key_file, 'r') as f:
|
||||
return f.read()
|
||||
|
||||
@testtools.skipIf(CONF.load_balancer.test_with_noop,
|
||||
'Active/Standby tests will not work in noop mode.')
|
||||
@decorators.idempotent_id('deab2b3f-62c7-4a05-9e92-aa45a04773fd')
|
||||
def test_active_standby_vrrp_failover(self):
|
||||
"""Tests active/standby VRRP failover
|
||||
|
||||
* Test the load balancer to make sure it is functioning
|
||||
* Identifies the Master and Backup amphora
|
||||
* Deletes the Master amphora
|
||||
* Sends traffic through the load balancer
|
||||
* Validates that the Backup has assumed the Master role
|
||||
"""
|
||||
|
||||
# Send some traffic
|
||||
self.check_members_balanced(self.lb_vip_address)
|
||||
|
||||
# Check there are two amphorae associated to the load balancer
|
||||
amps = self.os_admin.amphora_client.list_amphorae(
|
||||
query_params='{loadbalancer_id}={lb_id}'.format(
|
||||
loadbalancer_id=const.LOADBALANCER_ID, lb_id=self.lb_id))
|
||||
self.assertEqual(2, len(amps))
|
||||
|
||||
# Log VIP traffic
|
||||
for amp in amps:
|
||||
self._log_vip_traffic(amp, 'ACTSTBY-1')
|
||||
|
||||
# Send some traffic
|
||||
self.check_members_balanced(self.lb_vip_address)
|
||||
|
||||
# Which amphora is the active?
|
||||
active = self._get_active_standby_amps(amps, 'ACTSTBY-1')[0]
|
||||
|
||||
# Delete active amphora
|
||||
self.os_admin_servers_client.delete_server(active[const.COMPUTE_ID])
|
||||
|
||||
# Send some traffic
|
||||
self.check_members_balanced(self.lb_vip_address)
|
||||
|
||||
# Wait for the amphora failover to start
|
||||
waiters.wait_for_status(
|
||||
self.mem_lb_client.show_loadbalancer,
|
||||
self.lb_id, const.PROVISIONING_STATUS,
|
||||
const.PENDING_UPDATE, CONF.load_balancer.check_interval,
|
||||
CONF.load_balancer.check_timeout)
|
||||
|
||||
# Wait for the load balancer to return to ACTIVE
|
||||
waiters.wait_for_status(
|
||||
self.mem_lb_client.show_loadbalancer,
|
||||
self.lb_id, const.PROVISIONING_STATUS,
|
||||
const.ACTIVE, CONF.load_balancer.lb_build_interval,
|
||||
CONF.load_balancer.lb_build_timeout)
|
||||
|
||||
# Check again there are two amphorae associated to the load balancer
|
||||
amps = self.os_admin.amphora_client.list_amphorae(
|
||||
query_params='{loadbalancer_id}={lb_id}'.format(
|
||||
loadbalancer_id=const.LOADBALANCER_ID, lb_id=self.lb_id))
|
||||
self.assertEqual(2, len(amps))
|
||||
|
||||
# Log VIP traffic
|
||||
for amp in amps:
|
||||
self._log_vip_traffic(amp, 'ACTSTBY-2')
|
||||
|
||||
# Send some traffic
|
||||
self.check_members_balanced(self.lb_vip_address)
|
||||
|
||||
# Ensure only one amphora is handling VIP traffic
|
||||
self._get_active_standby_amps(amps, 'ACTSTBY-2')
|
||||
|
|
@ -127,6 +127,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
|
|||
cls.lb_admin_flavor_client = cls.os_roles_lb_admin.flavor_client
|
||||
cls.mem_flavor_client = cls.os_roles_lb_member.flavor_client
|
||||
cls.mem_provider_client = cls.os_roles_lb_member.provider_client
|
||||
cls.os_admin_servers_client = cls.os_admin.servers_client
|
||||
|
||||
@classmethod
|
||||
def resource_setup(cls):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
# Tempest runs under an unpriviledged user, thus the user cannot read
|
||||
# /etc/octavia/.ssh/ created by the Octavia devstack plugin. As work-around,
|
||||
# a new SSH key is generated and given read access to all users.
|
||||
# See also OCTAVIA_USE_PREGENERATED_SSH_KEY and
|
||||
# OCTAVIA_PREGENERATED_SSH_KEY_PATH set in the
|
||||
# octavia-v2-act-stdby-dsvm-scenario job definition
|
||||
- hosts: all
|
||||
name: Octavia DSVM jobs pre-run playbook
|
||||
tasks:
|
||||
- name: Generate Octavia RSA key
|
||||
shell: ssh-keygen -t rsa -f /tmp/octavia_ssh_key -q -N ""
|
||||
- name: Allow read permissions to other users
|
||||
file:
|
||||
path: /tmp/octavia_ssh_key
|
||||
state: file
|
||||
mode: 0644
|
||||
|
|
@ -393,3 +393,66 @@
|
|||
- job:
|
||||
name: octavia-v2-dsvm-scenario-centos-7
|
||||
parent: octavia-v2-dsvm-py2-scenario-centos-7
|
||||
|
||||
- job:
|
||||
name: octavia-v2-act-stdby-dsvm-scenario
|
||||
parent: octavia-dsvm-live-base
|
||||
pre-run: playbooks/act_stby_iptables/pre.yaml
|
||||
vars:
|
||||
devstack_localrc:
|
||||
OCTAVIA_USE_PREGENERATED_SSH_KEY: True
|
||||
OCTAVIA_PREGENERATED_SSH_KEY_PATH: /tmp/octavia_ssh_key
|
||||
devstack_local_conf:
|
||||
post-config:
|
||||
$OCTAVIA_CONF:
|
||||
api_settings:
|
||||
api_v1_enabled: False
|
||||
controller_worker:
|
||||
loadbalancer_topology: ACTIVE_STANDBY
|
||||
task_flow:
|
||||
engine: parallel
|
||||
devstack_local_conf:
|
||||
test-config:
|
||||
"$TEMPEST_CONFIG":
|
||||
load_balancer:
|
||||
check_timeout: 180
|
||||
tempest_test_regex: ^octavia_tempest_plugin.tests.act_stdby_scenario.v2.test_active_standby_iptables
|
||||
tox_envlist: all
|
||||
|
||||
- job:
|
||||
name: octavia-v2-act-stdby-dsvm-py2-scenario
|
||||
parent: octavia-v2-act-stdby-dsvm-scenario
|
||||
vars:
|
||||
devstack_localrc:
|
||||
USE_PYTHON3: False
|
||||
|
||||
- job:
|
||||
name: octavia-v2-act-stdby-dsvm-py2-scenario-centos-7
|
||||
parent: octavia-v2-act-stdby-dsvm-py2-scenario
|
||||
nodeset: devstack-single-node-centos-7
|
||||
vars:
|
||||
devstack_localrc:
|
||||
USE_PYTHON3: False
|
||||
OCTAVIA_AMP_BASE_OS: centos
|
||||
OCTAVIA_AMP_DISTRIBUTION_RELEASE_ID: 7
|
||||
OCTAVIA_AMP_IMAGE_SIZE: 3
|
||||
devstack_local_conf:
|
||||
test-config:
|
||||
"$TEMPEST_CONFIG":
|
||||
load_balancer:
|
||||
amphora_ssh_user: centos
|
||||
|
||||
- job:
|
||||
name: octavia-v2-act-stdby-dsvm-scenario-stable-stein
|
||||
parent: octavia-v2-act-stdby-dsvm-scenario
|
||||
override-checkout: stable/stein
|
||||
|
||||
- job:
|
||||
name: octavia-v2-act-stdby-dsvm-py2-scenario-stable-rocky
|
||||
parent: octavia-v2-act-stdby-dsvm-py2-scenario
|
||||
override-checkout: stable/rocky
|
||||
|
||||
- job:
|
||||
name: octavia-v2-act-stdby-dsvm-py2-scenario-stable-queens
|
||||
parent: octavia-v2-act-stdby-dsvm-py2-scenario
|
||||
override-checkout: stable/queens
|
||||
|
|
|
|||
|
|
@ -23,6 +23,18 @@
|
|||
voting: false
|
||||
- octavia-v2-dsvm-py2-scenario-two-node:
|
||||
voting: false
|
||||
- octavia-v2-act-stdby-dsvm-scenario:
|
||||
voting: false
|
||||
- octavia-v2-act-stdby-dsvm-py2-scenario:
|
||||
voting: false
|
||||
- octavia-v2-act-stdby-dsvm-py2-scenario-centos-7:
|
||||
voting: false
|
||||
- octavia-v2-act-stdby-dsvm-scenario-stable-stein:
|
||||
voting: false
|
||||
- octavia-v2-act-stdby-dsvm-py2-scenario-stable-rocky:
|
||||
voting: false
|
||||
- octavia-v2-act-stdby-dsvm-py2-scenario-stable-queens:
|
||||
voting: false
|
||||
- octavia-v2-dsvm-tls-barbican:
|
||||
voting: false
|
||||
gate:
|
||||
|
|
|
|||
Loading…
Reference in New Issue