Ensure that re_added ports are DOWN before set back to UP

During e.g. rebuild of the server by Nova, ports plugged to such server
are quickly removed and added again into br-int. In such case, ports are
in the "re_added" ports set in the neutron-ovs-agent.
But it seems that in some cases it may happen that such port isn't
switched to be DOWN first and then, when neutron-ovs-agent treats port
as added/updated and reports to the server that port is UP, there is no
notification to nova-compute send (because port's status was UP and new
status is still UP in the Neutron DB).
As Nova waits for the notification from Neutron in such case server
could ends up in the ERROR state.

To avoid such issue, all ports which are treated as "re_added" by the
neutron-ovs-agent are now first switched to be DOWN on the server side.
That way, when those ports are treated as added/updated in the same
rpc_loop iteration, switching their status to UP will for sure trigger
notification to nova.

Closes-Bug: #1963899
Change-Id: I0df376a80140ead7ff1fbf7f5ffef08a999dbe0b

neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py

@@ -2118,6 +2118,30 @@ class OVSNeutronAgent(l2population_rpc.L2populationRpcCallBackTunnelMixin,
         devices_not_in_datapath = set()
         migrating_devices = set()
         start = time.time()
+        if re_added:
+            # NOTE(slaweq): to make sure that devices which were deleted and
+            # then quickly added back (e.g. during rebuild of the VM by nova),
+            # we will update all such devices to be DOWN first. That way when
+            # we will process those devices as added/updated later in
+            # process_network_ports, we can be sure that update state to UP
+            # will trigger send notification about port status to Nova
+            re_added_devices = self.plugin_rpc.update_device_list(
+                context=self.context,
+                devices_up=[],
+                devices_down=re_added,
+                agent_id=self.agent_id,
+                host=self.conf.host)
+            failed_re_added_down = re_added_devices.get('failed_devices_down')
+            if failed_re_added_down:
+                LOG.debug("Status updated failed for re_added devices: %s",
+                          failed_re_added_down)
+            LOG.info("process_network_ports - iteration:%(iter_num)d - "
+                     "reporting re_added devices DOWN completed. "
+                     "Number of readded devices: %(re_added)s. "
+                     "Time elapsed: %(elapsed).3f",
+                     {'iter_num': self.iter_num,
+                      're_added': len(re_added),
+                      'elapsed': time.time() - start})
         if devices_added_updated:
             (skipped_devices, binding_no_activated_devices,
              need_binding_devices, failed_devices['added'],

neutron/tests/unit/plugins/ml2/drivers/openvswitch/agent/test_ovs_neutron_agent.py

@@ -1137,6 +1137,7 @@ class TestOvsNeutronAgent(object):
         skipped_devices = skipped_devices or []
         binding_no_activated_devices = binding_no_activated_devices or set()
         added_devices = port_info.get('added', set())
+        re_added_devices = port_info.get('re_added', set())
         with mock.patch.object(self.agent.sg_agent,
                                "setup_port_filters") as setup_port_filters,\
                 mock.patch.object(
@@ -1155,7 +1156,9 @@ class TestOvsNeutronAgent(object):
                 mock.patch.object(self.agent,
                                   "treat_devices_skipped",
                                   return_value=(
-                                      skipped_devices)) as device_skipped:
+                                      skipped_devices)) as device_skipped,\
+                mock.patch.object(self.agent.plugin_rpc,
+                                  'update_device_list') as update_device_list:
             self.assertEqual(
                 failed_devices,
                 self.agent.process_network_ports(port_info, False))
@@ -1167,11 +1170,20 @@ class TestOvsNeutronAgent(object):
                                      port_info.get('updated', set()))
             if devices_added_updated:
                 device_added_updated.assert_called_once_with(
-                    devices_added_updated, False, set())
+                    devices_added_updated, False, re_added_devices)
             if port_info.get('removed', set()):
                 device_removed.assert_called_once_with(port_info['removed'])
             if skipped_devices:
                 device_skipped.assert_called_once_with(set(skipped_devices))
+            if port_info.get('re_added'):
+                update_device_list.assert_called_once_with(
+                    context=self.agent.context,
+                    devices_up=[],
+                    devices_down=port_info['re_added'],
+                    agent_id=self.agent.agent_id,
+                    host=self.agent.conf.host)
+            else:
+                update_device_list.assert_not_called()
 
     def test_process_network_ports(self):
         self._test_process_network_ports(
@@ -1203,6 +1215,13 @@ class TestOvsNeutronAgent(object):
     def test_process_network_port_with_empty_port(self):
         self._test_process_network_ports({})
 
+    def test_process_network_ports_with_re_added_ports(self):
+        self._test_process_network_ports(
+            {'current': set(['tap0']),
+             'removed': set([]),
+             'added': set(['eth1']),
+             're_added': set(['eth1'])})
+
     @mock.patch.object(linux_utils, 'execute', return_value=False)
     def test_hybrid_plug_flag_based_on_firewall(self, *args):
         cfg.CONF.set_default(