Check for missing port bindings each sync period

When two DVR bindings go to ACTIVE simultaneously, one of the
updates will hit a stale data exception when commit to the DB.
This will ultimately result in update_port_postcommit not getting
called when the second binding's status goes to ACTIVE on retry.

To handle this, we now check for missing port bindings once per
sync period.

Change-Id: I8262bec98993fe80ca3482a47d283aeafea75101

networking_arista/ml2/arista_sync.py

@@ -145,6 +145,14 @@ class AristaSyncWorker(worker.BaseWorker):
         for resource_type in reversed(self.sync_order):
             resource_type.clear_all_data()
 
+    def port_bindings_missing(self):
+        # Check for any DVR port bindings that weren't synced
+        current_port_bindings = self.port_bindings.get_neutron_ids()
+        self.port_bindings.neutron_data_stale = True
+        if current_port_bindings != self.port_bindings.get_neutron_ids():
+            return True
+        return False
+
     def check_if_out_of_sync(self):
         cvx_uuid = self._rpc.get_cvx_uuid()
         out_of_sync = False
@@ -157,6 +165,11 @@ class AristaSyncWorker(worker.BaseWorker):
             self.force_full_sync()
             self._synchronizing_uuid = cvx_uuid
             out_of_sync = True
+        elif self.port_bindings_missing():
+            LOG.info("%(pid)s DVR port bindings out of sync. Triggering sync.",
+                     {'pid': os.getpid()})
+            out_of_sync = True
+
         self._last_sync_time = time.time()
         return out_of_sync
 

networking_arista/tests/unit/ml2/ml2_test_base.py

@@ -21,6 +21,7 @@ from pstats import Stats
 from neutron_lib.api.definitions import portbindings
 from neutron_lib import constants as n_const
 import neutron_lib.context
+from neutron_lib.db import api as db_api
 from neutron_lib.plugins import constants as p_const
 from neutron_lib.plugins import directory
 from oslo_config import cfg
@@ -29,6 +30,7 @@ from neutron.api.rpc.handlers import l3_rpc
 from neutron.common import constants as c_const
 from neutron.common import utils as common_utils
 from neutron.plugins.ml2.drivers import type_vxlan  # noqa
+from neutron.plugins.ml2 import models as ml2_models
 from neutron.services.trunk import constants as trunk_const
 from neutron.tests.common import helpers
 from neutron.tests.unit.plugins.ml2 import test_plugin
@@ -269,16 +271,36 @@ class MechTestBase(test_plugin.Ml2PluginV2TestCase):
                    portbindings.VIF_TYPE: portbindings.VIF_TYPE_UNBOUND}}
         self.plugin.update_port(self.context, port_id, p_dict)
 
-    def bind_dvr_to_host(self, port, host):
-        p_dict = {'port':
-                  {'device_id': port['device_id'],
-                   'device_owner': port['device_owner'],
-                   portbindings.HOST_ID: host}}
-        self.plugin.update_distributed_port_binding(self.context,
-                                                    port['id'], p_dict)
-        self.plugin.update_port_status(self.context, port['id'],
-                                       n_const.PORT_STATUS_ACTIVE,
-                                       host)
+    def bind_dvr_to_host(self, port, host, notify_ml2=True, seg_id=None):
+        if notify_ml2:
+            p_dict = {'port':
+                      {'device_id': port['device_id'],
+                       'device_owner': port['device_owner'],
+                       portbindings.HOST_ID: host}}
+            self.plugin.update_distributed_port_binding(self.context,
+                                                        port['id'], p_dict)
+            self.plugin.update_port_status(self.context, port['id'],
+                                           n_const.PORT_STATUS_ACTIVE,
+                                           host)
+        else:
+            session = db_api.get_writer_session()
+            with session.begin():
+                distributed_binding = {'port_id': port['id'],
+                                       'host': host,
+                                       'router_id': port['device_id'],
+                                       'status': 'ACTIVE',
+                                       'vif_type': 'distributed',
+                                       'vnic_type': portbindings.VNIC_NORMAL,
+                                       'profile': '',
+                                       'vif_details': ''}
+                session.add(ml2_models.DistributedPortBinding(
+                    **distributed_binding))
+                binding_level = {'port_id': port['id'],
+                                 'host': host,
+                                 'segment_id': seg_id,
+                                 'level': 0,
+                                 'driver': 'ovs'}
+                session.add(ml2_models.PortBindingLevel(**binding_level))
         p_ctx = self.plugin.get_bound_port_context(self.context, port['id'],
                                                    host)
         return port, p_ctx

networking_arista/tests/unit/ml2/test_mechanism_arista.py

@@ -191,6 +191,56 @@ class BasicMechDriverTestCase(ml2_test_base.MechTestBase):
         self.assertRouterPortDeleted(port['id'])
         self.assertPortBindingDeleted((port['id'], port_host_1))
 
+    def test_dvr_port_race(self):
+        network_tenant = 'net-ten'
+        net_dict = {'network': {'name': 'net',
+                                'tenant_id': network_tenant,
+                                'admin_state_up': True,
+                                'shared': False,
+                                'provider:physical_network': self.physnet,
+                                'provider:network_type': 'vlan'}}
+        network, net_ctx = self.create_network(net_dict)
+
+        # Create DVR port
+        device_id = 'router-1'
+        port_tenant = 'port-ten'
+        port_host_1 = self.host1
+        port_dict = {'name': 'port1',
+                     'tenant_id': port_tenant,
+                     'network_id': network['id'],
+                     'admin_state_up': True,
+                     'fixed_ips': [],
+                     'device_id': device_id,
+                     'device_owner': n_const.DEVICE_OWNER_DVR_INTERFACE}
+        port, _ = self.create_port(port_dict)
+        port, port_ctx = self.bind_dvr_to_host(port, port_host_1)
+        self.assertTenantCreated(port_tenant)
+        self.assertRouterCreated(device_id)
+        self.assertRouterPortCreated(port['id'])
+        self.assertPortBindingCreated((port['id'], port_host_1))
+
+        # Bring up a second DVR host, but don't notify the mech_plugin
+        port_host_2 = self.host2
+        port, port_ctx = self.bind_dvr_to_host(
+            port, port_host_2, notify_ml2=False,
+            seg_id=net_ctx.network_segments[0]['id'])
+        self.assertPortBindingCreated((port['id'], port_host_2))
+
+        # Removed the second host
+        self.unbind_dvr_from_host(port, port_host_2)
+        self.assertPortBindingDeleted((port['id'], port_host_2))
+        self.assertTenantCreated(port_tenant)
+        self.assertRouterCreated(device_id)
+        self.assertRouterPortCreated(port['id'])
+        self.assertPortBindingCreated((port['id'], port_host_1))
+
+        # Delete the port
+        self.delete_port(port['id'])
+        self.assertTenantDeleted(port_tenant)
+        self.assertRouterDeleted(device_id)
+        self.assertRouterPortDeleted(port['id'])
+        self.assertPortBindingDeleted((port['id'], port_host_1))
+
     def test_basic_vm_port(self):
         network_tenant = 'net-ten'
         net_dict = {'network': {'name': 'net',