From 5a7c12f245fd665de5a0364059e4ad918def3e12 Mon Sep 17 00:00:00 2001 From: Swaminathan Vasudevan Date: Thu, 14 Jun 2018 13:49:23 -0700 Subject: [PATCH] DVR: Self recover from the loss of 'fg' ports in FIP Namespace Sometimes we have seen the 'fg' ports within the fip-namespace either goes down, not created in time or getting deleted due to some race conditions. When this happens, the code tries to recover itself after couple of exceptions when there is a router_update message. But after recovery we could see that the fip-namespace is recreated and the 'fg-' port is plugged in and active, but the 'fpr' and the 'rfp' ports are missing which leads to the FloatingIP failure. This patch will fix this issue by checking for the missing devices in all router_updates. Change-Id: I78c7ea9f3b6a1cf5b208286eb372da05dc1ba379 Closes-Bug: #1776984 --- neutron/agent/l3/dvr_local_router.py | 9 +++ .../functional/agent/l3/test_dvr_router.py | 58 +++++++++++++++++++ .../unit/agent/l3/test_dvr_local_router.py | 2 + 3 files changed, 69 insertions(+) diff --git a/neutron/agent/l3/dvr_local_router.py b/neutron/agent/l3/dvr_local_router.py index 9ac73b82545..4a8ede95694 100644 --- a/neutron/agent/l3/dvr_local_router.py +++ b/neutron/agent/l3/dvr_local_router.py @@ -601,7 +601,16 @@ class DvrLocalRouter(dvr_router_base.DvrRouterBase): self.connect_rtr_2_fip() super(DvrLocalRouter, self).process_external() + def _check_rtr_2_fip_connect(self): + """Checks if the rtr to fip connect exists, if not sets to false.""" + fip_ns_name = self.fip_ns.get_name() + if ip_lib.network_namespace_exists(fip_ns_name): + fip_2_rtr_name = self.fip_ns.get_int_device_name(self.router_id) + if not ip_lib.device_exists(fip_2_rtr_name, namespace=fip_ns_name): + self.rtr_fip_connect = False + def connect_rtr_2_fip(self): + self._check_rtr_2_fip_connect() if self.fip_ns.agent_gateway_port and not self.rtr_fip_connect: ex_gw_port = self.get_ex_gw_port() self.fip_ns.create_rtr_2_fip_link(self) diff --git a/neutron/tests/functional/agent/l3/test_dvr_router.py b/neutron/tests/functional/agent/l3/test_dvr_router.py index 312ce4875ce..0aa0e90a0e3 100644 --- a/neutron/tests/functional/agent/l3/test_dvr_router.py +++ b/neutron/tests/functional/agent/l3/test_dvr_router.py @@ -115,6 +115,64 @@ class TestDvrRouter(framework.L3AgentTestFramework): self.assertEqual(4, default_rules_list_count) self.assertEqual(2, interface_rules_list_count) + def test_dvr_update_gateway_port_no_fip_fg_port_recovers_itself_with_fpr( + self): + self.agent.conf.agent_mode = 'dvr' + # Create the router with external net + router_info = self.generate_dvr_router_info() + external_gw_port = router_info['gw_port'] + ext_net_id = router_info['_floatingips'][0]['floating_network_id'] + self.mock_plugin_api.get_external_network_id.return_value = ext_net_id + router = self.manage_router(self.agent, router_info) + fg_port = router.fip_ns.agent_gateway_port + fg_port_name = router.fip_ns.get_ext_device_name(fg_port['id']) + fg_device = ip_lib.IPDevice(fg_port_name, + namespace=router.fip_ns.name) + fip_2_rtr_name = router.fip_ns.get_int_device_name(router.router_id) + fpr_device = ip_lib.IPDevice(fip_2_rtr_name, + namespace=router.fip_ns.name) + # Now validate if the gateway is properly configured. + rtr_2_fip, fip_2_rtr = router.rtr_fip_subnet.get_pair() + tbl_index = router._get_snat_idx(fip_2_rtr) + tbl_filter = ['table', tbl_index] + self.assertIn('gateway', fg_device.route.get_gateway( + filters=tbl_filter)) + self._validate_fips_for_external_network( + router, router.fip_ns.get_name()) + # Now delete the fg- port that was created + ext_net_bridge = self.agent.conf.external_network_bridge + router.fip_ns.driver.unplug(fg_port_name, + bridge=ext_net_bridge, + namespace=router.fip_ns.name, + prefix=dvr_fip_ns.FIP_EXT_DEV_PREFIX) + # Now check if the fg- port is missing. + self.assertFalse(fg_device.exists()) + fpr_device.link.set_down() + # Now change the gateway ip for the router and do an update. + router.ex_gw_port = copy.deepcopy(router.ex_gw_port) + new_fg_port = copy.deepcopy(fg_port) + for subnet in new_fg_port['subnets']: + subnet['gateway_ip'] = '19.4.4.2' + router.router[n_const.FLOATINGIP_AGENT_INTF_KEY] = [new_fg_port] + self.assertRaises(n_exc.FloatingIpSetupException, + self.agent._process_updated_router, + router.router) + self.agent._process_updated_router(router.router) + self.assertTrue(fg_device.exists()) + self.assertTrue(fpr_device.exists()) + updated_route = fg_device.route.list_routes( + ip_version=lib_constants.IP_VERSION_4, + table=tbl_index) + expected_route = [{'cidr': '0.0.0.0/0', + 'dev': fg_port_name, + 'table': tbl_index, + u'via': u'19.4.4.2'}] + self.assertEqual(expected_route, updated_route) + self._validate_fips_for_external_network( + router, router.fip_ns.get_name()) + self._delete_router(self.agent, router.router_id) + self._assert_fip_namespace_deleted(external_gw_port) + def test_dvr_update_gateway_port_with_no_gw_port_in_namespace(self): self.agent.conf.agent_mode = 'dvr' diff --git a/neutron/tests/unit/agent/l3/test_dvr_local_router.py b/neutron/tests/unit/agent/l3/test_dvr_local_router.py index 1e3a35bb9b5..f2a2a69ccac 100644 --- a/neutron/tests/unit/agent/l3/test_dvr_local_router.py +++ b/neutron/tests/unit/agent/l3/test_dvr_local_router.py @@ -206,6 +206,7 @@ class TestDvrRouterOperations(base.BaseTestCase): ri.rtr_fip_connect = False ex_gw_port = {'network_id': 'fake_net_id'} ri.create_dvr_external_gateway_on_agent(ex_gw_port) + ri._check_rtr_2_fip_connect = mock.Mock() ri.connect_rtr_2_fip() self.assertTrue(ri._check_if_address_scopes_match.called) if address_scopes_match: @@ -329,6 +330,7 @@ class TestDvrRouterOperations(base.BaseTestCase): ri.fip_ns = mock.Mock() ri.fip_ns.agent_gateway_port = agent_gw_port ri.create_dvr_external_gateway_on_agent(ri.ex_gw_port) + ri._check_rtr_2_fip_connect = mock.Mock() ri.connect_rtr_2_fip() self.assertTrue(ri.rtr_fip_connect) ri.fip_ns.allocate_rule_priority.return_value = FIP_PRI