diff --git a/neutron/agent/l3/dvr_edge_ha_router.py b/neutron/agent/l3/dvr_edge_ha_router.py
index 57173e05d44..0d349826812 100644
--- a/neutron/agent/l3/dvr_edge_ha_router.py
+++ b/neutron/agent/l3/dvr_edge_ha_router.py
@@ -62,7 +62,7 @@ class DvrEdgeHaRouter(dvr_edge_router.DvrEdgeRouter,
                 self.get_ex_gw_port())
         self._add_vip(fip_cidr, interface_name)
 
-        self.ha_port = self.router.get(constants.HA_INTERFACE_KEY)
+        self.set_ha_port()
         if (self.is_router_master() and self.ha_port and
                 self.ha_port['status'] == constants.PORT_STATUS_ACTIVE):
             return super(DvrEdgeHaRouter, self).add_centralized_floatingip(
diff --git a/neutron/agent/l3/ha_router.py b/neutron/agent/l3/ha_router.py
index 8a74a330917..4642c2517d6 100644
--- a/neutron/agent/l3/ha_router.py
+++ b/neutron/agent/l3/ha_router.py
@@ -123,7 +123,7 @@ class HaRouter(router.RouterInfo):
             raise Exception(msg)
         super(HaRouter, self).initialize(process_monitor)
 
-        self.ha_port = ha_port
+        self.set_ha_port()
         self._init_keepalived_manager(process_monitor)
         self.ha_network_added()
         self.update_initial_state(self.state_change_callback)
@@ -461,10 +461,23 @@ class HaRouter(router.RouterInfo):
         self.ha_network_removed()
         super(HaRouter, self).delete()
 
+    def set_ha_port(self):
+        ha_port = self.router.get(n_consts.HA_INTERFACE_KEY)
+        if not ha_port:
+            return
+        # NOTE: once HA port is set, it MUST remain this value no matter what
+        # the server return. Because there is race condition between l3-agent
+        # side sync router info for processing and server side router deleting.
+        # TODO(liuyulong): make sure router HA ports never change.
+        if not self.ha_port or (self.ha_port and
+                                self.ha_port['status'] != ha_port['status']):
+            self.ha_port = ha_port
+
     def process(self):
         super(HaRouter, self).process()
 
-        self.ha_port = self.router.get(n_consts.HA_INTERFACE_KEY)
+        self.set_ha_port()
+        LOG.debug("Processing HA router with HA port: %s", self.ha_port)
         if (self.ha_port and
                 self.ha_port['status'] == n_consts.PORT_STATUS_ACTIVE):
             self.enable_keepalived()
diff --git a/neutron/tests/unit/agent/l3/test_ha_router.py b/neutron/tests/unit/agent/l3/test_ha_router.py
index e5f58142918..69f2ef0d9c4 100644
--- a/neutron/tests/unit/agent/l3/test_ha_router.py
+++ b/neutron/tests/unit/agent/l3/test_ha_router.py
@@ -15,6 +15,7 @@
 import signal
 
 import mock
+from neutron_lib import constants as n_consts
 from oslo_utils import uuidutils
 
 from neutron.agent.l3 import ha_router
@@ -115,3 +116,31 @@ class TestBasicRouterOperations(base.BaseTestCase):
         calls = ["sig='str(%d)'" % signal.SIGTERM,
                  "sig='str(%d)'" % signal.SIGKILL]
         mock_pm.disable.has_calls(calls)
+
+    def test_set_ha_port(self):
+        ri = self._create_router()
+        self.assertIsNone(ri.ha_port)
+
+        ri.router = {}
+        ri.set_ha_port()
+        self.assertIsNone(ri.ha_port)
+
+        # HA_INTERFACE_KEY from None to some value
+        ri.router = {n_consts.HA_INTERFACE_KEY: {"id": _uuid(),
+                                                 "status": "DOWN"}}
+        ri.set_ha_port()
+        self.assertIsNotNone(ri.ha_port)
+        self.assertEqual('DOWN', ri.ha_port["status"])
+
+        # HA port state change
+        ri.router = {n_consts.HA_INTERFACE_KEY: {"id": _uuid(),
+                                                 "status": "ACTIVE"}}
+        ri.set_ha_port()
+        self.assertIsNotNone(ri.ha_port)
+        self.assertEqual('ACTIVE', ri.ha_port["status"])
+
+        ri.router = {}
+        ri.set_ha_port()
+        # neutron server return empty HA_INTERFACE_KEY, but
+        # agent side router info should remain the original value.
+        self.assertIsNotNone(ri.ha_port)