HA race condition test for DHCP scheduling
Add fullstack testcase for DHCP HA scheduling with fake scheduler that selects new agent with some extra sleep for every scheduling. Change-Id: I1045992ce9a18b37dd7dcdb46063698fad983932 Related-Bug: #1535554
This commit is contained in:
parent
19bc090b1b
commit
0041860e97
|
@ -23,6 +23,7 @@ import inspect
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import os.path
|
import os.path
|
||||||
|
import threading
|
||||||
|
|
||||||
import eventlet.timeout
|
import eventlet.timeout
|
||||||
import fixtures
|
import fixtures
|
||||||
|
@ -479,6 +480,49 @@ class BaseTestCase(DietTestCase):
|
||||||
self.config(group='AGENT',
|
self.config(group='AGENT',
|
||||||
root_helper_daemon=get_rootwrap_daemon_cmd())
|
root_helper_daemon=get_rootwrap_daemon_cmd())
|
||||||
|
|
||||||
|
def _simulate_concurrent_requests_process_and_raise(self, calls, args):
|
||||||
|
|
||||||
|
class SimpleThread(threading.Thread):
|
||||||
|
def __init__(self, q):
|
||||||
|
super(SimpleThread, self).__init__()
|
||||||
|
self.q = q
|
||||||
|
self.exception = None
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
try:
|
||||||
|
while not self.q.empty():
|
||||||
|
item = None
|
||||||
|
try:
|
||||||
|
item = self.q.get(False)
|
||||||
|
func, func_args = item[0], item[1]
|
||||||
|
func(*func_args)
|
||||||
|
except six.moves.queue.Empty:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
if item:
|
||||||
|
self.q.task_done()
|
||||||
|
except Exception as e:
|
||||||
|
self.exception = e
|
||||||
|
|
||||||
|
def get_exception(self):
|
||||||
|
return self.exception
|
||||||
|
|
||||||
|
q = six.moves.queue.Queue()
|
||||||
|
for func, func_args in zip(calls, args):
|
||||||
|
q.put_nowait((func, func_args))
|
||||||
|
|
||||||
|
threads = []
|
||||||
|
for z in range(len(calls)):
|
||||||
|
t = SimpleThread(q)
|
||||||
|
threads.append(t)
|
||||||
|
t.start()
|
||||||
|
q.join()
|
||||||
|
|
||||||
|
for t in threads:
|
||||||
|
e = t.get_exception()
|
||||||
|
if e:
|
||||||
|
raise e
|
||||||
|
|
||||||
|
|
||||||
class PluginFixture(fixtures.Fixture):
|
class PluginFixture(fixtures.Fixture):
|
||||||
|
|
||||||
|
|
|
@ -122,6 +122,10 @@ class NeutronConfigFixture(ConfigFixture):
|
||||||
env_desc.placement_port
|
env_desc.placement_port
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
if env_desc.dhcp_scheduler_class:
|
||||||
|
self.config['DEFAULT']['dhcp_agents_per_network'] = '1'
|
||||||
|
self.config['DEFAULT']['network_scheduler_driver'] = (
|
||||||
|
env_desc.dhcp_scheduler_class)
|
||||||
|
|
||||||
net_helpers.set_local_port_range(CLIENT_CONN_PORT_START,
|
net_helpers.set_local_port_range(CLIENT_CONN_PORT_START,
|
||||||
CLIENT_CONN_PORT_END)
|
CLIENT_CONN_PORT_END)
|
||||||
|
|
|
@ -39,7 +39,8 @@ class EnvironmentDescription(object):
|
||||||
agent_down_time=75, router_scheduler=None,
|
agent_down_time=75, router_scheduler=None,
|
||||||
global_mtu=constants.DEFAULT_NETWORK_MTU,
|
global_mtu=constants.DEFAULT_NETWORK_MTU,
|
||||||
debug_iptables=False, log=False, report_bandwidths=False,
|
debug_iptables=False, log=False, report_bandwidths=False,
|
||||||
has_placement=False, placement_port=None):
|
has_placement=False, placement_port=None,
|
||||||
|
dhcp_scheduler_class=None,):
|
||||||
self.network_type = network_type
|
self.network_type = network_type
|
||||||
self.l2_pop = l2_pop
|
self.l2_pop = l2_pop
|
||||||
self.qos = qos
|
self.qos = qos
|
||||||
|
@ -55,6 +56,7 @@ class EnvironmentDescription(object):
|
||||||
self.report_bandwidths = report_bandwidths
|
self.report_bandwidths = report_bandwidths
|
||||||
self.has_placement = has_placement
|
self.has_placement = has_placement
|
||||||
self.placement_port = placement_port
|
self.placement_port = placement_port
|
||||||
|
self.dhcp_scheduler_class = dhcp_scheduler_class
|
||||||
if self.qos:
|
if self.qos:
|
||||||
self.service_plugins += ',qos'
|
self.service_plugins += ',qos'
|
||||||
if self.log:
|
if self.log:
|
||||||
|
|
|
@ -0,0 +1,52 @@
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
from neutron.scheduler import base_scheduler
|
||||||
|
from neutron.scheduler import dhcp_agent_scheduler
|
||||||
|
|
||||||
|
|
||||||
|
class AlwaysTheOtherAgentScheduler(base_scheduler.BaseChanceScheduler,
|
||||||
|
dhcp_agent_scheduler.AutoScheduler):
|
||||||
|
"""Choose always different agent that the ones selected previously
|
||||||
|
|
||||||
|
This dhcp agent scheduler intended use is only in fullstack tests.
|
||||||
|
The goal is to ensure the concurrently running schedulings to select
|
||||||
|
different agents so the over-scheduling becomes visible in the number
|
||||||
|
of agents scheduled to the network.
|
||||||
|
To use this scheduler initialize your EnvironmentDescription with
|
||||||
|
dhcp_scheduler_class='neutron.tests.fullstack.test_dhcp_agent.'
|
||||||
|
'AlwaysTheOtherAgentScheduler'
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.last_selected_agent_ids = []
|
||||||
|
super(AlwaysTheOtherAgentScheduler, self).__init__(
|
||||||
|
dhcp_agent_scheduler.DhcpFilter())
|
||||||
|
|
||||||
|
def select(self, plugin, context, resource_hostable_agents,
|
||||||
|
resource_hosted_agents, num_agents_needed):
|
||||||
|
possible_agents = []
|
||||||
|
for agent in resource_hostable_agents:
|
||||||
|
if agent.id in self.last_selected_agent_ids:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
possible_agents.append(agent)
|
||||||
|
num_agents = min(len(possible_agents), num_agents_needed)
|
||||||
|
self.last_selected_agent_ids = [
|
||||||
|
ag.id for ag in possible_agents[0:num_agents]]
|
||||||
|
|
||||||
|
# Note(lajoskatona): To make the race window big enough let's delay
|
||||||
|
# the actual scheduling.
|
||||||
|
time.sleep(5)
|
||||||
|
return possible_agents[0:num_agents_needed]
|
|
@ -36,6 +36,8 @@ class BaseDhcpAgentTest(base.BaseFullStackTestCase):
|
||||||
(constants.AGENT_TYPE_LINUXBRIDGE,
|
(constants.AGENT_TYPE_LINUXBRIDGE,
|
||||||
{'l2_agent_type': constants.AGENT_TYPE_LINUXBRIDGE})
|
{'l2_agent_type': constants.AGENT_TYPE_LINUXBRIDGE})
|
||||||
]
|
]
|
||||||
|
boot_vm_for_test = True
|
||||||
|
dhcp_scheduler_class = None
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
host_descriptions = [
|
host_descriptions = [
|
||||||
|
@ -48,12 +50,15 @@ class BaseDhcpAgentTest(base.BaseFullStackTestCase):
|
||||||
environment.EnvironmentDescription(
|
environment.EnvironmentDescription(
|
||||||
l2_pop=False,
|
l2_pop=False,
|
||||||
arp_responder=False,
|
arp_responder=False,
|
||||||
agent_down_time=self.agent_down_time),
|
agent_down_time=self.agent_down_time,
|
||||||
|
dhcp_scheduler_class=self.dhcp_scheduler_class,
|
||||||
|
),
|
||||||
host_descriptions)
|
host_descriptions)
|
||||||
|
|
||||||
super(BaseDhcpAgentTest, self).setUp(env)
|
super(BaseDhcpAgentTest, self).setUp(env)
|
||||||
self.project_id = uuidutils.generate_uuid()
|
self.project_id = uuidutils.generate_uuid()
|
||||||
self._create_network_subnet_and_vm()
|
if self.boot_vm_for_test:
|
||||||
|
self._create_network_subnet_and_vm()
|
||||||
|
|
||||||
def _spawn_vm(self):
|
def _spawn_vm(self):
|
||||||
host = random.choice(self.environment.hosts)
|
host = random.choice(self.environment.hosts)
|
||||||
|
@ -193,3 +198,34 @@ class TestDhcpAgentHA(BaseDhcpAgentTest):
|
||||||
# check if new vm will get IP from DHCP agent which is still alive
|
# check if new vm will get IP from DHCP agent which is still alive
|
||||||
new_vm = self._spawn_vm()
|
new_vm = self._spawn_vm()
|
||||||
new_vm.block_until_dhcp_config_done()
|
new_vm.block_until_dhcp_config_done()
|
||||||
|
|
||||||
|
|
||||||
|
class TestDhcpAgentHARaceCondition(BaseDhcpAgentTest):
|
||||||
|
|
||||||
|
agent_down_time = 30
|
||||||
|
number_of_hosts = 2
|
||||||
|
boot_vm_for_test = False
|
||||||
|
dhcp_scheduler_class = ('neutron.tests.fullstack.schedulers.dhcp.'
|
||||||
|
'AlwaysTheOtherAgentScheduler')
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(TestDhcpAgentHARaceCondition, self).setUp()
|
||||||
|
self._create_network_with_multiple_subnets()
|
||||||
|
|
||||||
|
def _create_network_with_multiple_subnets(self):
|
||||||
|
self.network = self.safe_client.create_network(self.project_id)
|
||||||
|
|
||||||
|
funcs = []
|
||||||
|
args = []
|
||||||
|
for i in range(4):
|
||||||
|
funcs.append(self.safe_client.create_subnet)
|
||||||
|
args.append((
|
||||||
|
self.project_id, self.network['id'], '10.0.%s.0/24' % i,
|
||||||
|
'10.0.%s.1' % i, 'subnet-test-%s' % i, True
|
||||||
|
))
|
||||||
|
self._simulate_concurrent_requests_process_and_raise(funcs, args)
|
||||||
|
|
||||||
|
def test_dhcp_agent_ha_with_race_condition(self):
|
||||||
|
network_dhcp_agents = self.client.list_dhcp_agent_hosting_networks(
|
||||||
|
self.network['id'])['agents']
|
||||||
|
self.assertEqual(1, len(network_dhcp_agents))
|
||||||
|
|
|
@ -10,8 +10,6 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
import threading
|
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
from neutron_lib.api.definitions import floating_ip_port_forwarding as apidef
|
from neutron_lib.api.definitions import floating_ip_port_forwarding as apidef
|
||||||
from neutron_lib.callbacks import exceptions as c_exc
|
from neutron_lib.callbacks import exceptions as c_exc
|
||||||
|
@ -20,7 +18,6 @@ from neutron_lib.exceptions import l3 as lib_l3_exc
|
||||||
from neutron_lib.plugins import constants as plugin_constants
|
from neutron_lib.plugins import constants as plugin_constants
|
||||||
from neutron_lib.plugins import directory
|
from neutron_lib.plugins import directory
|
||||||
from oslo_utils import uuidutils
|
from oslo_utils import uuidutils
|
||||||
from six.moves import queue
|
|
||||||
|
|
||||||
from neutron.services.portforwarding.common import exceptions as pf_exc
|
from neutron.services.portforwarding.common import exceptions as pf_exc
|
||||||
from neutron.services.portforwarding import pf_plugin
|
from neutron.services.portforwarding import pf_plugin
|
||||||
|
@ -375,49 +372,6 @@ class PortForwardingTestCase(PortForwardingTestCaseBase):
|
||||||
self.pf_plugin.delete_floatingip_port_forwarding,
|
self.pf_plugin.delete_floatingip_port_forwarding,
|
||||||
self.context, res['id'], uuidutils.generate_uuid())
|
self.context, res['id'], uuidutils.generate_uuid())
|
||||||
|
|
||||||
def _simulate_concurrent_requests_process_and_raise(
|
|
||||||
self, funcs, args_list):
|
|
||||||
|
|
||||||
class SimpleThread(threading.Thread):
|
|
||||||
def __init__(self, q):
|
|
||||||
super(SimpleThread, self).__init__()
|
|
||||||
self.q = q
|
|
||||||
self.exception = None
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
try:
|
|
||||||
while not self.q.empty():
|
|
||||||
item = None
|
|
||||||
try:
|
|
||||||
item = self.q.get(False)
|
|
||||||
func, func_args = item[0], item[1]
|
|
||||||
func(*func_args)
|
|
||||||
except queue.Empty:
|
|
||||||
pass
|
|
||||||
finally:
|
|
||||||
if item:
|
|
||||||
self.q.task_done()
|
|
||||||
except Exception as e:
|
|
||||||
self.exception = e
|
|
||||||
|
|
||||||
def get_exception(self):
|
|
||||||
return self.exception
|
|
||||||
|
|
||||||
q = queue.Queue()
|
|
||||||
for func, func_args in zip(funcs, args_list):
|
|
||||||
q.put_nowait((func, func_args))
|
|
||||||
threads = []
|
|
||||||
for _ in range(len(funcs)):
|
|
||||||
t = SimpleThread(q)
|
|
||||||
threads.append(t)
|
|
||||||
t.start()
|
|
||||||
q.join()
|
|
||||||
|
|
||||||
for t in threads:
|
|
||||||
e = t.get_exception()
|
|
||||||
if e:
|
|
||||||
raise e
|
|
||||||
|
|
||||||
def test_concurrent_create_port_forwarding_delete_fip(self):
|
def test_concurrent_create_port_forwarding_delete_fip(self):
|
||||||
|
|
||||||
func1 = self.pf_plugin.create_floatingip_port_forwarding
|
func1 = self.pf_plugin.create_floatingip_port_forwarding
|
||||||
|
|
Loading…
Reference in New Issue