ovs: survive errors from check_ovs_status

Instead of allowing an error to bubble up and exit from rpc_loop, catch
it and assume the switch is dead which will make the agent to wait until
the switch is back without failing the service.

Change-Id: Ic3095dd42b386f56b1f75ebb6a125606f295551b
Closes-Bug: #1731494

neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py

@@ -1778,8 +1778,12 @@ class OVSNeutronAgent(l2population_rpc.L2populationRpcCallBackTunnelMixin,
                 port_info.get('updated'))
 
     def check_ovs_status(self):
-        # Check for the canary flow
-        status = self.int_br.check_canary_table()
+        try:
+            # Check for the canary flow
+            status = self.int_br.check_canary_table()
+        except Exception:
+            LOG.exception("Failure while checking for the canary flow")
+            status = constants.OVS_DEAD
         if status == constants.OVS_RESTARTED:
             LOG.warning("OVS is restarted. OVSNeutronAgent will reset "
                         "bridges and recover ports.")

neutron/tests/unit/plugins/ml2/drivers/openvswitch/agent/test_ovs_neutron_agent.py

@@ -3514,6 +3514,14 @@ class TestOvsDvrNeutronAgent(object):
                 pass
         self.assertTrue(all([x.called for x in reset_mocks]))
 
+    def test_rpc_loop_survives_error_in_check_canary_table(self):
+        with mock.patch.object(self.agent.int_br,
+                               'check_canary_table',
+                               side_effect=TypeError('borked')),\
+                mock.patch.object(self.agent, '_check_and_handle_signal',
+                                  side_effect=[True, False]):
+            self.agent.rpc_loop(polling_manager=mock.Mock())
+
     def _test_scan_ports_failure(self, scan_method_name):
         with mock.patch.object(self.agent,
                                'check_ovs_status',