Clean up orphan tap devices on migrate

A failed migration can leave an orphaned tap device on a host, which
will cause subsequent migrations to fail due to a tap device name
collision.  This change will detect these collisions on a host
during pre_live_migrate and clean them up.

This change will also clean up all orphaned adapters on driver
startup.

Change-Id: I0ea40b0ab15653f115b185823c2681a6758d5c83
Closes-Bug: 1692022

nova_powervm/tests/virt/powervm/fixtures.py

@@ -93,6 +93,7 @@ class PowerVMComputeDriver(fixtures.Fixture):
     @mock.patch('nova_powervm.virt.powervm.disk.localdisk.LocalStorage')
     @mock.patch('nova_powervm.virt.powervm.driver.PowerVMDriver._get_adapter')
     @mock.patch('pypowervm.tasks.partition.get_this_partition')
+    @mock.patch('pypowervm.tasks.cna.find_orphaned_trunks')
     def _init_host(self, *args):
 
         self.mock_sys = self.useFixture(fixtures.MockPatch(

nova_powervm/tests/virt/powervm/test_driver.py

@@ -1954,3 +1954,10 @@ class TestPowerVMDriver(test.TestCase):
     def test_deallocate_networks_on_reschedule(self):
         candeallocate = self.drv.deallocate_networks_on_reschedule(mock.Mock())
         self.assertTrue(candeallocate)
+
+    @mock.patch('pypowervm.tasks.cna.find_orphaned_trunks')
+    def test_cleanup_orphan_adapters(self, mock_find_orphans):
+        mock_orphan = mock.MagicMock()
+        mock_find_orphans.return_value = [mock_orphan]
+        self.drv._cleanup_orphan_adapters('my_vswitch')
+        mock_orphan.delete.assert_called_once_with()

nova_powervm/tests/virt/powervm/test_vif.py

@@ -908,10 +908,11 @@ class TestVifOvsDriver(test.TestCase):
     @mock.patch('nova.network.linux_net.create_ovs_vif_port')
     @mock.patch('nova.utils.execute')
     @mock.patch('pypowervm.tasks.cna.crt_trunk_with_free_vlan')
+    @mock.patch('pypowervm.tasks.cna.find_orphaned_trunks', autospec=True)
     @mock.patch('pypowervm.tasks.partition.get_this_partition')
     def test_pre_live_migrate_at_destination(
-            self, mock_part_get, mock_trunk_crt, mock_execute,
-            mock_crt_ovs_port):
+            self, mock_part_get, mock_find_trunks, mock_trunk_crt,
+            mock_execute, mock_crt_ovs_port):
         # Mock the vif
         vif = {'devname': 'tap-dev', 'address': 'aa:bb:cc:dd:ee:ff',
                'network': {'bridge': 'br-int'}, 'id': 'vif_id'}
@@ -923,17 +924,26 @@ class TestVifOvsDriver(test.TestCase):
 
         mock_trunk_crt.return_value = [mock.Mock(pvid=2)]
 
+        mock_orphan_wrap = mock.MagicMock(mac='aabbccddeeff')
+        mock_find_trunks.return_value = [mock_orphan_wrap]
+
         # Invoke and test the basic response
         vea_vlan_mappings = {}
         self.drv.pre_live_migrate_at_destination(vif, vea_vlan_mappings)
         self.assertEqual(vea_vlan_mappings['aa:bb:cc:dd:ee:ff'], 2)
 
         # Now validate it called the things it needed to
-        mock_execute.assert_called_once_with('ip', 'link', 'set', 'tap-dev',
-                                             'up', run_as_root=True)
+        mock_execute.assert_has_calls(
+            [mock.call('ovs-vsctl', '--timeout=120', '--', '--if-exists',
+                       'del-port', 'br-int', 'tap-dev', run_as_root=True),
+             mock.call('ip', 'link', 'set', 'tap-dev', 'up',
+                       run_as_root=True)])
         mock_trunk_crt.assert_called_once_with(
             self.adpt, 'host_uuid', ['mgmt_uuid'],
             CONF.powervm.pvm_vswitch_for_novalink_io, dev_name='tap-dev')
+        mock_find_trunks.assert_called_once_with(
+            self.adpt, CONF.powervm.pvm_vswitch_for_novalink_io)
+        mock_orphan_wrap.delete.assert_called_once_with()
         mock_crt_ovs_port.assert_called_once_with(
             'br-int', 'tap-dev', 'vif_id', 'aa:bb:cc:dd:ee:ff', self.inst.uuid)
 

nova_powervm/virt/powervm/driver.py

@@ -37,6 +37,7 @@ from pypowervm import const as pvm_const
 from pypowervm import exceptions as pvm_exc
 from pypowervm.helpers import log_helper as log_hlp
 from pypowervm.helpers import vios_busy as vio_hlp
+from pypowervm.tasks import cna as pvm_cna
 from pypowervm.tasks import memory as pvm_mem
 from pypowervm.tasks import partition as pvm_par
 from pypowervm.tasks import power_opts as pvm_popts
@@ -139,6 +140,9 @@ class PowerVMDriver(driver.ComputeDriver):
         # Value: overhead (int MB)
         self._inst_overhead_cache = {}
 
+        # Clean-up any orphan adapters
+        self._cleanup_orphan_adapters(CONF.powervm.pvm_vswitch_for_novalink_io)
+
         LOG.info(_LI("The compute driver has been initialized."))
 
     def cleanup_host(self, host):
@@ -1816,3 +1820,10 @@ class PowerVMDriver(driver.ComputeDriver):
         :returns: Boolean value. If True deallocate networks on reschedule.
         """
         return True
+
+    def _cleanup_orphan_adapters(self, vswitch_name):
+        """Finds and removes trunk VEAs that have no corresponding CNA."""
+        orphans = pvm_cna.find_orphaned_trunks(self.adapter, vswitch_name)
+        for orphan in orphans:
+            LOG.info("Deleting orphan CNA: %s", orphan.dev_name)
+            orphan.delete()

nova_powervm/virt/powervm/vif.py

@@ -805,6 +805,8 @@ class PvmOvsVifDriver(PvmLioVifDriver):
                                   mac address, value is the destination's
                                   target hypervisor VLAN.
         """
+        self._cleanup_orphan_adapters(vif,
+                                      CONF.powervm.pvm_vswitch_for_novalink_io)
         mgmt_wrap = pvm_par.get_this_partition(self.adapter)
         dev = self.get_trunk_dev_name(vif)
 
@@ -907,3 +909,13 @@ class PvmOvsVifDriver(PvmLioVifDriver):
         """
         linux_net.delete_ovs_vif_port(vif['network']['bridge'],
                                       self.get_trunk_dev_name(vif))
+
+    def _cleanup_orphan_adapters(self, vif, vswitch_name):
+        """Finds and removes trunk VEAs that have no corresponding CNA."""
+        # Find and delete orphan adapters with macs matching our vif
+        orphans = pvm_cna.find_orphaned_trunks(self.adapter, vswitch_name)
+        for orphan in orphans:
+            if vm.norm_mac(orphan.mac) == vif['address']:
+                linux_net.delete_ovs_vif_port(vif['network']['bridge'],
+                                              self.get_trunk_dev_name(vif))
+                orphan.delete()