From bc979efdb85b3ac2722d6607485d21e002718e02 Mon Sep 17 00:00:00 2001 From: Jakub Libosvar Date: Tue, 11 Apr 2017 12:27:46 -0400 Subject: [PATCH] fullstack: Don't let dhcp agents failover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It turned out dhcp tests work only because agents are considered dead after 10 seconds while they report to server every 60 seconds. This led to calling network resync after agent revival and hiding the fact dhcp agent is not capable of receiving any amqp messages. This patch sets the report interval of agents to the half of agent_down_time on server side and uses eventlet dhcp agent in order to trigger eventlet monkey patching code. Eventlet was behind the failure with messages not getting processed. As [1] notes: "Note: If the “eventlet” executor is used, the threading and time library need to be monkeypatched." Because each port calls dhclient to obtain IP address and each dhclient instance overwrites /etc/resolv.conf there was added a script that generates fullstack-dhclient-script from an existing dhclient-script before starting fulltstack tests. This generated script is passed to each dhclient process running in fake fullstack machine using -sf parameter. [1] https://docs.openstack.org/developer/oslo.messaging/server.html Related-bug: 1453350 Change-Id: I0336176b9c364fe3a95be5cef9e7a3af1ef9d7e9 --- neutron/tests/fullstack/base.py | 1 + neutron/tests/fullstack/cmd/__init__.py | 0 .../cmd/dhcp_agent.py} | 2 +- neutron/tests/fullstack/resources/config.py | 3 ++ neutron/tests/fullstack/resources/machine.py | 8 +++++- neutron/tests/fullstack/resources/process.py | 5 ++-- neutron/tests/fullstack/test_dhcp_agent.py | 4 ++- .../generate_dhclient_script_for_fullstack.sh | 28 +++++++++++++++++++ tox.ini | 3 ++ 9 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 neutron/tests/fullstack/cmd/__init__.py rename neutron/tests/{common/agents/fullstack_dhcp_agent.py => fullstack/cmd/dhcp_agent.py} (97%) create mode 100755 tools/generate_dhclient_script_for_fullstack.sh diff --git a/neutron/tests/fullstack/base.py b/neutron/tests/fullstack/base.py index 3d4516dff5d..d950491abae 100644 --- a/neutron/tests/fullstack/base.py +++ b/neutron/tests/fullstack/base.py @@ -26,6 +26,7 @@ from neutron.tests.unit import testlib_api # This is the directory from which infra fetches log files for fullstack tests DEFAULT_LOG_DIR = os.path.join(helpers.get_test_log_path(), 'dsvm-fullstack-logs') +ROOTDIR = os.path.dirname(__file__) class BaseFullStackTestCase(testlib_api.MySQLTestCaseMixin, diff --git a/neutron/tests/fullstack/cmd/__init__.py b/neutron/tests/fullstack/cmd/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/neutron/tests/common/agents/fullstack_dhcp_agent.py b/neutron/tests/fullstack/cmd/dhcp_agent.py similarity index 97% rename from neutron/tests/common/agents/fullstack_dhcp_agent.py rename to neutron/tests/fullstack/cmd/dhcp_agent.py index 3c7a06a7a43..416cd7874f1 100755 --- a/neutron/tests/common/agents/fullstack_dhcp_agent.py +++ b/neutron/tests/fullstack/cmd/dhcp_agent.py @@ -20,8 +20,8 @@ import sys from oslo_config import cfg from oslo_utils import uuidutils -from neutron.agent import dhcp_agent from neutron.agent.linux import dhcp as linux_dhcp +from neutron.cmd.eventlet.agents import dhcp as dhcp_agent OPTS = [ diff --git a/neutron/tests/fullstack/resources/config.py b/neutron/tests/fullstack/resources/config.py index f834b13116e..8329a41d0ef 100644 --- a/neutron/tests/fullstack/resources/config.py +++ b/neutron/tests/fullstack/resources/config.py @@ -87,6 +87,9 @@ class NeutronConfigFixture(ConfigFixture): 'oslo_policy': { 'policy_file': self._generate_policy_json(), }, + 'agent': { + 'report_interval': env_desc.agent_down_time / 2.0 + }, }) def _setUp(self): diff --git a/neutron/tests/fullstack/resources/machine.py b/neutron/tests/fullstack/resources/machine.py index ff5cb08b40e..35fd16953dd 100644 --- a/neutron/tests/fullstack/resources/machine.py +++ b/neutron/tests/fullstack/resources/machine.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from distutils import spawn import itertools import netaddr @@ -25,6 +26,8 @@ from neutron.common import utils from neutron.tests.common import machine_fixtures from neutron.tests.common import net_helpers +FULLSTACK_DHCLIENT_SCRIPT = 'fullstack-dhclient-script' + class FakeFullstackMachinesList(list): """A list of items implementing the FakeFullstackMachine interface.""" @@ -42,6 +45,8 @@ class FakeFullstackMachinesList(list): class FakeFullstackMachine(machine_fixtures.FakeMachineBase): + NO_RESOLV_CONF_DHCLIENT_SCRIPT_PATH = ( + spawn.find_executable(FULLSTACK_DHCLIENT_SCRIPT)) def __init__(self, host, network_id, tenant_id, safe_client, neutron_port=None, bridge_name=None, use_dhcp=False): @@ -129,7 +134,8 @@ class FakeFullstackMachine(machine_fixtures.FakeMachineBase): self.addCleanup(self._stop_async_dhclient) def _start_async_dhclient(self): - cmd = ["dhclient", '--no-pid', '-d', self.port.name] + cmd = ["dhclient", '-sf', self.NO_RESOLV_CONF_DHCLIENT_SCRIPT_PATH, + '--no-pid', '-d', self.port.name] self.dhclient_async = async_process.AsyncProcess( cmd, run_as_root=True, namespace=self.namespace) self.dhclient_async.start() diff --git a/neutron/tests/fullstack/resources/process.py b/neutron/tests/fullstack/resources/process.py index f6f74b80163..127c5d70d76 100644 --- a/neutron/tests/fullstack/resources/process.py +++ b/neutron/tests/fullstack/resources/process.py @@ -301,8 +301,9 @@ class DhcpAgentFixture(fixtures.Fixture): test_name=self.test_name, process_name=self.NEUTRON_DHCP_AGENT, exec_name=spawn.find_executable( - 'fullstack_dhcp_agent.py', - path=os.path.join(base.ROOTDIR, 'common', 'agents')), + 'dhcp_agent.py', + path=os.path.join( + fullstack_base.ROOTDIR, 'cmd')), config_filenames=config_filenames, namespace=self.namespace ) diff --git a/neutron/tests/fullstack/test_dhcp_agent.py b/neutron/tests/fullstack/test_dhcp_agent.py index 58ec7a0765a..f5546e8a2dd 100644 --- a/neutron/tests/fullstack/test_dhcp_agent.py +++ b/neutron/tests/fullstack/test_dhcp_agent.py @@ -46,7 +46,7 @@ class BaseDhcpAgentTest(base.BaseFullStackTestCase): environment.EnvironmentDescription( l2_pop=False, arp_responder=False, - agent_down_time=10), + agent_down_time=self.agent_down_time), host_descriptions) super(BaseDhcpAgentTest, self).setUp(env) @@ -88,6 +88,7 @@ class BaseDhcpAgentTest(base.BaseFullStackTestCase): class TestDhcpAgentNoHA(BaseDhcpAgentTest): number_of_hosts = 1 + agent_down_time = 60 def test_dhcp_assignment(self): # First check if network was scheduled to one DHCP agent @@ -102,6 +103,7 @@ class TestDhcpAgentNoHA(BaseDhcpAgentTest): class TestDhcpAgentHA(BaseDhcpAgentTest): number_of_hosts = 2 + agent_down_time = 10 def _wait_until_network_rescheduled(self, old_agent): def _agent_rescheduled(): diff --git a/tools/generate_dhclient_script_for_fullstack.sh b/tools/generate_dhclient_script_for_fullstack.sh new file mode 100755 index 00000000000..8f7a3c80b76 --- /dev/null +++ b/tools/generate_dhclient_script_for_fullstack.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +MAKE_RESOLV_CONF_FUNCTION=make_resolv_conf + +USAGE="$0 +The script takes existing dhclient-script and makes $MAKE_RESOLV_CONF_FUNCTION function a noop function. +" + +if [ $# -lt 1 ]; then + echo "Path to virtual environment directory is a required parameter." + echo $USAGE + exit 2 +fi + +VENV_DIR=$1 +DHCLIENT_SCRIPT_NAME=dhclient-script +DHCLIENT_PATH=$(which $DHCLIENT_SCRIPT_NAME) +FULLSTACK_DHCLIENT_SCRIPT=$VENV_DIR/bin/fullstack-dhclient-script + +if [ -n "$DHCLIENT_PATH" ]; then + # Return from make_resolv_conf function immediately. This will cause + # that /etc/resolv.conf will not be updated by fake fullstack machines. + sed "/^$MAKE_RESOLV_CONF_FUNCTION()/a\ return" $DHCLIENT_PATH > $FULLSTACK_DHCLIENT_SCRIPT + chmod +x $FULLSTACK_DHCLIENT_SCRIPT +else + echo "$DHCLIENT_SCRIPT_NAME not found." + exit 1 +fi diff --git a/tox.ini b/tox.ini index 61b234b2c0b..02da2763ecb 100644 --- a/tox.ini +++ b/tox.ini @@ -76,6 +76,9 @@ setenv = {[testenv]setenv} OS_TEST_PATH=./neutron/tests/fullstack deps = {[testenv:functional]deps} +commands = + {toxinidir}/tools/generate_dhclient_script_for_fullstack.sh {envdir} + {[testenv]commands} [testenv:releasenotes] commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html