Filter HA router without HA port bindings after race conditions

Neutron server will not be able to sync ha router data after
race happened between get_ha_router_port_bindings and HA router
deleting API call. Once the ports of L3HARouterAgentPortBinding
were deleted the _process_sync_ha_data may get a None binding
port, and then the _process_sync_ha_data will fail to get the
HA interface port info due to the None port. This patch will
filter the bindings without port.

Change-Id: Ie38baf061d678fc5d768195b25241efbad74e42f
Closes-Bug: #1533457
(cherry picked from commit 179b8301ed)

neutron/db/l3_hamode_db.py

@@ -683,7 +683,15 @@ class L3_HA_NAT_db_mixin(l3_dvr_db.L3_NAT_with_dvr_db_mixin,
                                                     routers_dict.keys(),
                                                     host)
         for binding in bindings:
-            port_dict = self._core_plugin._make_port_dict(binding.port)
+            port = binding.port
+            if not port:
+                # Filter the HA router has no ha port here
+                LOG.info(_LI("HA router %s is missing HA router port "
+                             "bindings. Skipping it."),
+                         binding.router_id)
+                routers_dict.pop(binding.router_id)
+                continue
+            port_dict = self._core_plugin._make_port_dict(port)
 
             router = routers_dict.get(binding.router_id)
             router[constants.HA_INTERFACE_KEY] = port_dict
@@ -694,6 +702,8 @@ class L3_HA_NAT_db_mixin(l3_dvr_db.L3_NAT_with_dvr_db_mixin,
             if interface:
                 self._populate_mtu_and_subnets_for_ports(context, [interface])
 
+        # Could not filter the HA_INTERFACE_KEY here, because a DVR router
+        # with SNAT HA in DVR compute host also does not have that attribute.
         return list(routers_dict.values())
 
     @log_helpers.log_method_call

neutron/tests/unit/db/test_l3_hamode_db.py

@@ -748,6 +748,34 @@ class L3HATestCase(L3HATestFramework):
             self.assertEqual(states[router['id']],
                              router[constants.HA_ROUTER_STATE_KEY])
 
+    def test_sync_ha_router_info_ha_interface_port_concurrently_deleted(self):
+        router1 = self._create_router()
+        router2 = self._create_router()
+
+        # retrieve all router ha port bindings
+        bindings = self.plugin.get_ha_router_port_bindings(
+            self.admin_ctx, [router1['id'], router2['id']])
+        self.assertEqual(4, len(bindings))
+
+        routers = self.plugin.get_ha_sync_data_for_host(
+            self.admin_ctx, self.agent1['host'], self.agent1)
+        self.assertEqual(2, len(routers))
+
+        bindings = self.plugin.get_ha_router_port_bindings(
+            self.admin_ctx, [router1['id'], router2['id']],
+            self.agent1['host'])
+        self.assertEqual(2, len(bindings))
+
+        fake_binding = mock.Mock()
+        fake_binding.router_id = router2['id']
+        fake_binding.port = None
+        with mock.patch.object(
+                self.plugin, "get_ha_router_port_bindings",
+                return_value=[bindings[0], fake_binding]):
+            routers = self.plugin.get_ha_sync_data_for_host(
+                self.admin_ctx, self.agent1['host'], self.agent1)
+            self.assertEqual(1, len(routers))
+
     def test_set_router_states_handles_concurrently_deleted_router(self):
         router1 = self._create_router()
         router2 = self._create_router()