diff --git a/neutron/agent/l3/ha_router.py b/neutron/agent/l3/ha_router.py
index a2bdfb20823..f1a248963de 100644
--- a/neutron/agent/l3/ha_router.py
+++ b/neutron/agent/l3/ha_router.py
@@ -254,7 +254,9 @@ class HaRouter(router.RouterInfo):
 
     def remove_floating_ip(self, device, ip_cidr):
         self._remove_vip(ip_cidr)
-        if self.ha_state == 'master' and device.addr.list():
+        if self.ha_state == 'master' and device.addr.list(to=ip_cidr):
+            # Delete the floatingip address from external port only after
+            # the ip address has been configured to the device
             super(HaRouter, self).remove_floating_ip(device, ip_cidr)
 
     def internal_network_updated(self, interface_name, ip_cidrs):
diff --git a/neutron/tests/functional/agent/l3/test_ha_router.py b/neutron/tests/functional/agent/l3/test_ha_router.py
index 8f0c68fbd66..fd134af7324 100644
--- a/neutron/tests/functional/agent/l3/test_ha_router.py
+++ b/neutron/tests/functional/agent/l3/test_ha_router.py
@@ -242,6 +242,20 @@ class L3HATestCase(framework.L3AgentTestFramework):
         interface_name = router.get_external_device_name(port['id'])
         router.external_gateway_removed(port, interface_name)
 
+    def test_removing_floatingip_immediately(self):
+        router_info = self.generate_router_info(enable_ha=True)
+        router = self.manage_router(self.agent, router_info)
+        ex_gw_port = router.get_ex_gw_port()
+        interface_name = router.get_external_device_interface_name(ex_gw_port)
+        utils.wait_until_true(lambda: router.ha_state == 'master')
+        self._add_fip(router, '172.168.1.20', fixed_address='10.0.0.3')
+        router.process(self.agent)
+        router.router[l3_constants.FLOATINGIP_KEY] = []
+        # The purpose of the test is to simply make sure no exception is raised
+        # Because router.process will consume the FloatingIpSetupException,
+        # call the configure_fip_addresses directly here
+        router.configure_fip_addresses(interface_name)
+
 
 class L3HATestFailover(framework.L3AgentTestFramework):