fullstack: Don't let dhcp agents failover

It turned out dhcp tests work only because agents are considered dead
after 10 seconds while they report to server every 60 seconds. This led
to calling network resync after agent revival and hiding the fact dhcp
agent is not capable of receiving any amqp messages.

This patch sets the report interval of agents to the half of
agent_down_time on server side and uses eventlet dhcp agent in order to
trigger eventlet monkey patching code.

Eventlet was behind the failure with messages not getting processed. As
[1] notes: "Note: If the “eventlet” executor is used, the threading and
time library need to be monkeypatched."

Because each port calls dhclient to obtain IP address and each dhclient
instance overwrites /etc/resolv.conf there was added a script that
generates fullstack-dhclient-script from an existing dhclient-script
before starting fulltstack tests. This generated script is passed to
each dhclient process running in fake fullstack machine using -sf
parameter.

[1] https://docs.openstack.org/developer/oslo.messaging/server.html

Related-bug: 1453350

Change-Id: I0336176b9c364fe3a95be5cef9e7a3af1ef9d7e9
This commit is contained in:
Jakub Libosvar 2017-04-11 12:27:46 -04:00
parent 991ea0b923
commit bc979efdb8
9 changed files with 49 additions and 5 deletions

View File

@ -26,6 +26,7 @@ from neutron.tests.unit import testlib_api
# This is the directory from which infra fetches log files for fullstack tests
DEFAULT_LOG_DIR = os.path.join(helpers.get_test_log_path(),
'dsvm-fullstack-logs')
ROOTDIR = os.path.dirname(__file__)
class BaseFullStackTestCase(testlib_api.MySQLTestCaseMixin,

View File

View File

@ -20,8 +20,8 @@ import sys
from oslo_config import cfg
from oslo_utils import uuidutils
from neutron.agent import dhcp_agent
from neutron.agent.linux import dhcp as linux_dhcp
from neutron.cmd.eventlet.agents import dhcp as dhcp_agent
OPTS = [

View File

@ -87,6 +87,9 @@ class NeutronConfigFixture(ConfigFixture):
'oslo_policy': {
'policy_file': self._generate_policy_json(),
},
'agent': {
'report_interval': env_desc.agent_down_time / 2.0
},
})
def _setUp(self):

View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from distutils import spawn
import itertools
import netaddr
@ -25,6 +26,8 @@ from neutron.common import utils
from neutron.tests.common import machine_fixtures
from neutron.tests.common import net_helpers
FULLSTACK_DHCLIENT_SCRIPT = 'fullstack-dhclient-script'
class FakeFullstackMachinesList(list):
"""A list of items implementing the FakeFullstackMachine interface."""
@ -42,6 +45,8 @@ class FakeFullstackMachinesList(list):
class FakeFullstackMachine(machine_fixtures.FakeMachineBase):
NO_RESOLV_CONF_DHCLIENT_SCRIPT_PATH = (
spawn.find_executable(FULLSTACK_DHCLIENT_SCRIPT))
def __init__(self, host, network_id, tenant_id, safe_client,
neutron_port=None, bridge_name=None, use_dhcp=False):
@ -129,7 +134,8 @@ class FakeFullstackMachine(machine_fixtures.FakeMachineBase):
self.addCleanup(self._stop_async_dhclient)
def _start_async_dhclient(self):
cmd = ["dhclient", '--no-pid', '-d', self.port.name]
cmd = ["dhclient", '-sf', self.NO_RESOLV_CONF_DHCLIENT_SCRIPT_PATH,
'--no-pid', '-d', self.port.name]
self.dhclient_async = async_process.AsyncProcess(
cmd, run_as_root=True, namespace=self.namespace)
self.dhclient_async.start()

View File

@ -301,8 +301,9 @@ class DhcpAgentFixture(fixtures.Fixture):
test_name=self.test_name,
process_name=self.NEUTRON_DHCP_AGENT,
exec_name=spawn.find_executable(
'fullstack_dhcp_agent.py',
path=os.path.join(base.ROOTDIR, 'common', 'agents')),
'dhcp_agent.py',
path=os.path.join(
fullstack_base.ROOTDIR, 'cmd')),
config_filenames=config_filenames,
namespace=self.namespace
)

View File

@ -46,7 +46,7 @@ class BaseDhcpAgentTest(base.BaseFullStackTestCase):
environment.EnvironmentDescription(
l2_pop=False,
arp_responder=False,
agent_down_time=10),
agent_down_time=self.agent_down_time),
host_descriptions)
super(BaseDhcpAgentTest, self).setUp(env)
@ -88,6 +88,7 @@ class BaseDhcpAgentTest(base.BaseFullStackTestCase):
class TestDhcpAgentNoHA(BaseDhcpAgentTest):
number_of_hosts = 1
agent_down_time = 60
def test_dhcp_assignment(self):
# First check if network was scheduled to one DHCP agent
@ -102,6 +103,7 @@ class TestDhcpAgentNoHA(BaseDhcpAgentTest):
class TestDhcpAgentHA(BaseDhcpAgentTest):
number_of_hosts = 2
agent_down_time = 10
def _wait_until_network_rescheduled(self, old_agent):
def _agent_rescheduled():

View File

@ -0,0 +1,28 @@
#!/bin/bash
MAKE_RESOLV_CONF_FUNCTION=make_resolv_conf
USAGE="$0 <path to virtual environment to place executable>
The script takes existing dhclient-script and makes $MAKE_RESOLV_CONF_FUNCTION function a noop function.
"
if [ $# -lt 1 ]; then
echo "Path to virtual environment directory is a required parameter."
echo $USAGE
exit 2
fi
VENV_DIR=$1
DHCLIENT_SCRIPT_NAME=dhclient-script
DHCLIENT_PATH=$(which $DHCLIENT_SCRIPT_NAME)
FULLSTACK_DHCLIENT_SCRIPT=$VENV_DIR/bin/fullstack-dhclient-script
if [ -n "$DHCLIENT_PATH" ]; then
# Return from make_resolv_conf function immediately. This will cause
# that /etc/resolv.conf will not be updated by fake fullstack machines.
sed "/^$MAKE_RESOLV_CONF_FUNCTION()/a\ return" $DHCLIENT_PATH > $FULLSTACK_DHCLIENT_SCRIPT
chmod +x $FULLSTACK_DHCLIENT_SCRIPT
else
echo "$DHCLIENT_SCRIPT_NAME not found."
exit 1
fi

View File

@ -76,6 +76,9 @@ setenv = {[testenv]setenv}
OS_TEST_PATH=./neutron/tests/fullstack
deps =
{[testenv:functional]deps}
commands =
{toxinidir}/tools/generate_dhclient_script_for_fullstack.sh {envdir}
{[testenv]commands}
[testenv:releasenotes]
commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html