diff --git a/neutron/tests/common/agents/fullstack_dhcp_agent.py b/neutron/tests/common/agents/fullstack_dhcp_agent.py new file mode 100755 index 00000000000..3c7a06a7a43 --- /dev/null +++ b/neutron/tests/common/agents/fullstack_dhcp_agent.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# Copyright 2016 OVH SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import os +import sys + +from oslo_config import cfg +from oslo_utils import uuidutils + +from neutron.agent import dhcp_agent +from neutron.agent.linux import dhcp as linux_dhcp + + +OPTS = [ + cfg.StrOpt('test_namespace_suffix', default='testprefix', + help=_("Suffix to append to all DHCP namespace names.")), +] + + +def NetModel_init(self, d): + super(linux_dhcp.NetModel, self).__init__(d) + + self._ns_name = "%s%s%s" % ( + linux_dhcp.NS_PREFIX, self.id, cfg.CONF.test_namespace_suffix) + + +@classmethod +def existing_dhcp_networks(cls, conf): + """Return a list of existing networks ids that we have configs for.""" + confs_dir = cls.get_confs_dir(conf) + networks = [] + try: + for c in os.listdir(confs_dir): + c = c.replace(cfg.CONF.test_namespace_suffix, "") + if uuidutils.is_uuid_like(c): + networks.append(c) + except OSError: + pass + return networks + + +def monkeypatch_dhcplocalprocess_init(): + original_init = linux_dhcp.DhcpLocalProcess.__init__ + + def new_init(self, conf, network, process_monitor, version=None, + plugin=None): + network_copy = copy.deepcopy(network) + network_copy.id = "%s%s" % (network.id, cfg.CONF.test_namespace_suffix) + original_init( + self, conf, network_copy, process_monitor, version, plugin) + self.network = network + + linux_dhcp.DhcpLocalProcess.__init__ = new_init + + +def monkeypatch_linux_dhcp(): + linux_dhcp.NetModel.__init__ = NetModel_init + linux_dhcp.Dnsmasq.existing_dhcp_networks = existing_dhcp_networks + monkeypatch_dhcplocalprocess_init() + + +def main(): + cfg.CONF.register_opts(OPTS) + monkeypatch_linux_dhcp() + dhcp_agent.main() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/neutron/tests/fullstack/resources/config.py b/neutron/tests/fullstack/resources/config.py index 84c67dea3f9..e2dba2bfeac 100644 --- a/neutron/tests/fullstack/resources/config.py +++ b/neutron/tests/fullstack/resources/config.py @@ -48,6 +48,9 @@ class ConfigFixture(fixtures.Fixture): self.useFixture(cfg_fixture) self.filename = cfg_fixture.filename + def _generate_namespace_suffix(self): + return utils.get_rand_name(prefix='test') + class NeutronConfigFixture(ConfigFixture): @@ -69,6 +72,7 @@ class NeutronConfigFixture(ConfigFixture): 'service_plugins': ','.join(service_plugins), 'auth_strategy': 'noauth', 'debug': 'True', + 'agent_down_time': env_desc.agent_down_time, 'transport_url': 'rabbit://%(user)s:%(password)s@%(host)s:5672/%(vhost)s' % {'user': rabbitmq_environment.user, @@ -292,9 +296,6 @@ class L3ConfigFixture(ConfigFixture): def get_external_bridge(self): return self.config.DEFAULT.external_network_bridge - def _generate_namespace_suffix(self): - return utils.get_rand_name(prefix='test') - class DhcpConfigFixture(ConfigFixture): @@ -309,6 +310,7 @@ class DhcpConfigFixture(ConfigFixture): self.config['DEFAULT'].update({ 'debug': 'True', 'dhcp_confs': self._generate_dhcp_path(), + 'test_namespace_suffix': self._generate_namespace_suffix() }) def _setUp(self): diff --git a/neutron/tests/fullstack/resources/environment.py b/neutron/tests/fullstack/resources/environment.py index ce171d1d3f8..614601df188 100644 --- a/neutron/tests/fullstack/resources/environment.py +++ b/neutron/tests/fullstack/resources/environment.py @@ -34,13 +34,15 @@ class EnvironmentDescription(object): Does the setup, as a whole, support tunneling? How about l2pop? """ def __init__(self, network_type='vxlan', l2_pop=True, qos=False, - mech_drivers='openvswitch,linuxbridge', arp_responder=False): + mech_drivers='openvswitch,linuxbridge', arp_responder=False, + agent_down_time=75): self.network_type = network_type self.l2_pop = l2_pop self.qos = qos self.network_range = None self.mech_drivers = mech_drivers self.arp_responder = arp_responder + self.agent_down_time = agent_down_time @property def tunneling_enabled(self): diff --git a/neutron/tests/fullstack/resources/process.py b/neutron/tests/fullstack/resources/process.py index ac12b2784b7..983ebbef0d8 100644 --- a/neutron/tests/fullstack/resources/process.py +++ b/neutron/tests/fullstack/resources/process.py @@ -15,6 +15,7 @@ import datetime from distutils import spawn import os +import re import signal import fixtures @@ -23,6 +24,7 @@ from neutronclient.v2_0 import client from oslo_utils import fileutils from neutron.agent.linux import async_process +from neutron.agent.linux import ip_lib from neutron.agent.linux import utils from neutron.common import utils as common_utils from neutron.tests import base @@ -66,7 +68,11 @@ class ProcessFixture(fixtures.Fixture): self.process.start(block=True) def stop(self): - self.process.stop(block=True, kill_signal=self.kill_signal) + try: + self.process.stop(block=True, kill_signal=self.kill_signal) + except async_process.AsyncProcessException as e: + if "Process is not running" not in str(e): + raise class RabbitmqEnvironmentFixture(fixtures.Fixture): @@ -259,8 +265,42 @@ class DhcpAgentFixture(fixtures.Fixture): ProcessFixture( test_name=self.test_name, process_name=self.NEUTRON_DHCP_AGENT, - exec_name=self.NEUTRON_DHCP_AGENT, + exec_name=spawn.find_executable( + 'fullstack_dhcp_agent.py', + path=os.path.join(base.ROOTDIR, 'common', 'agents')), config_filenames=config_filenames, namespace=self.namespace ) ) + self.dhcp_namespace_pattern = re.compile( + r"qdhcp-[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}%s" % + self.get_namespace_suffix()) + self.addCleanup(self.clean_dhcp_namespaces) + + def get_agent_hostname(self): + return self.neutron_cfg_fixture.config['DEFAULT']['host'] + + def get_namespace_suffix(self): + return self.plugin_config.DEFAULT.test_namespace_suffix + + def kill(self): + self.process_fixture.stop() + self.clean_dhcp_namespaces() + + def clean_dhcp_namespaces(self): + """Delete all DHCP namespaces created by DHCP agent. + + In some tests for DHCP agent HA agents are killed when handling DHCP + service for network(s). In such case DHCP namespace is not deleted by + DHCP agent and such namespaces are found and deleted using agent's + namespace suffix. + """ + + ip_wrapper = ip_lib.IPWrapper() + for namespace in ip_wrapper.get_namespaces(): + if self.dhcp_namespace_pattern.match(namespace): + try: + ip_wrapper.netns.delete(namespace) + except RuntimeError: + # Continue cleaning even if namespace deletions fails + pass diff --git a/neutron/tests/fullstack/test_dhcp_agent.py b/neutron/tests/fullstack/test_dhcp_agent.py index 9dd1fc3ce2c..58ec7a0765a 100644 --- a/neutron/tests/fullstack/test_dhcp_agent.py +++ b/neutron/tests/fullstack/test_dhcp_agent.py @@ -12,9 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. +import random + from neutron_lib import constants from oslo_utils import uuidutils +from neutron.common import utils as common_utils from neutron.tests.fullstack import base from neutron.tests.fullstack.resources import environment from neutron.tests.fullstack.resources import machine @@ -23,7 +26,7 @@ from neutron.tests.unit import testlib_api load_tests = testlib_api.module_load_tests -class TestDhcpAgent(base.BaseFullStackTestCase): +class BaseDhcpAgentTest(base.BaseFullStackTestCase): scenarios = [ (constants.AGENT_TYPE_OVS, @@ -36,18 +39,32 @@ class TestDhcpAgent(base.BaseFullStackTestCase): host_descriptions = [ environment.HostDescription( dhcp_agent=True, - l2_agent_type=self.l2_agent_type)] + l2_agent_type=self.l2_agent_type + ) for _ in range(self.number_of_hosts)] env = environment.Environment( environment.EnvironmentDescription( l2_pop=False, - arp_responder=False), + arp_responder=False, + agent_down_time=10), host_descriptions) - super(TestDhcpAgent, self).setUp(env) + super(BaseDhcpAgentTest, self).setUp(env) self.project_id = uuidutils.generate_uuid() self._create_network_subnet_and_vm() + def _spawn_vm(self): + host = random.choice(self.environment.hosts) + vm = self.useFixture( + machine.FakeFullstackMachine( + host, + self.network['id'], + self.project_id, + self.safe_client, + use_dhcp=True)) + vm.block_until_boot() + return vm + def _create_network_subnet_and_vm(self): self.network = self.safe_client.create_network(self.project_id) @@ -58,14 +75,19 @@ class TestDhcpAgent(base.BaseFullStackTestCase): name='subnet-test', enable_dhcp=True) - self.vm = self.useFixture( - machine.FakeFullstackMachine( - self.environment.hosts[0], - self.network['id'], - self.project_id, - self.safe_client, - use_dhcp=True)) - self.vm.block_until_boot() + self.vm = self._spawn_vm() + + def _wait_until_agent_down(self, agent_id): + def _agent_down(): + agent = self.client.show_agent(agent_id)['agent'] + return not agent.get('alive') + + common_utils.wait_until_true(_agent_down) + + +class TestDhcpAgentNoHA(BaseDhcpAgentTest): + + number_of_hosts = 1 def test_dhcp_assignment(self): # First check if network was scheduled to one DHCP agent @@ -75,3 +97,75 @@ class TestDhcpAgent(base.BaseFullStackTestCase): # And check if IP and gateway config is fine on FakeMachine self.vm.block_until_dhcp_config_done() + + +class TestDhcpAgentHA(BaseDhcpAgentTest): + + number_of_hosts = 2 + + def _wait_until_network_rescheduled(self, old_agent): + def _agent_rescheduled(): + network_agents = self.client.list_dhcp_agent_hosting_networks( + self.network['id'])['agents'] + if network_agents: + return network_agents[0]['id'] != old_agent['id'] + return False + + common_utils.wait_until_true(_agent_rescheduled) + + def _kill_dhcp_agent(self, agent): + for host in self.environment.hosts: + hostname = host.dhcp_agent.get_agent_hostname() + if hostname == agent['host']: + host.dhcp_agent.kill() + self._wait_until_agent_down(agent['id']) + break + + def _add_network_to_new_agent(self): + dhcp_agents = self.client.list_agents( + agent_type=constants.AGENT_TYPE_DHCP)['agents'] + dhcp_agents_ids = [agent['id'] for agent in dhcp_agents] + + current_agents = self.client.list_dhcp_agent_hosting_networks( + self.network['id'])['agents'] + current_agents_ids = [agent['id'] for agent in current_agents] + + new_agents_ids = list(set(dhcp_agents_ids) - set(current_agents_ids)) + if new_agents_ids: + new_agent_id = random.choice(new_agents_ids) + self.client.add_network_to_dhcp_agent( + new_agent_id, {'network_id': self.network['id']}) + + def test_reschedule_network_on_new_agent(self): + network_dhcp_agents = self.client.list_dhcp_agent_hosting_networks( + self.network['id'])['agents'] + self.assertEqual(1, len(network_dhcp_agents)) + + self._kill_dhcp_agent(network_dhcp_agents[0]) + self._wait_until_network_rescheduled(network_dhcp_agents[0]) + + # ensure that only one agent is handling DHCP for this network + new_network_dhcp_agents = self.client.list_dhcp_agent_hosting_networks( + self.network['id'])['agents'] + self.assertEqual(1, len(new_network_dhcp_agents)) + + # check if new vm will get IP from new DHCP agent + new_vm = self._spawn_vm() + new_vm.block_until_dhcp_config_done() + + def test_multiple_agents_for_network(self): + network_dhcp_agents = self.client.list_dhcp_agent_hosting_networks( + self.network['id'])['agents'] + self.assertEqual(1, len(network_dhcp_agents)) + + self._add_network_to_new_agent() + # ensure that two agents are handling DHCP for this network + network_dhcp_agents = self.client.list_dhcp_agent_hosting_networks( + self.network['id'])['agents'] + self.assertEqual(2, len(network_dhcp_agents)) + + self._kill_dhcp_agent(network_dhcp_agents[0]) + + # check if new vm will get IP from DHCP agent which is still alive + new_vm = self._spawn_vm() + new_vm.block_until_dhcp_config_done() diff --git a/tools/configure_for_func_testing.sh b/tools/configure_for_func_testing.sh index 228233fec88..7996552e8d7 100755 --- a/tools/configure_for_func_testing.sh +++ b/tools/configure_for_func_testing.sh @@ -202,7 +202,7 @@ function _install_rootwrap_sudoers { # # 1: https://bugs.launchpad.net/oslo.rootwrap/+bug/1417331 # -Defaults:$STACK_USER secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PROJECT_VENV/bin" +Defaults:$STACK_USER secure_path="$PROJECT_VENV/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" $STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_SUDOER_CMD $STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_DAEMON_SUDOER_CMD EOF