Check for container exists before deletion

When a user creates an instance, the user is able to destroy
the instance before LXD has a chance to start the container.
This results in a bad state between LXD and nova-lxd and
pylxd raising an exception that the container was not found.

Before destroying the conainer, check the status code
of LXD to see if the container exists. If the container
does not exist just cleanup the profile if it exists
and the assoicated vifs if had already been created.

This was discovered by running tempest unit tests.

Change-Id: I7c244e8bca9bab69ead5d45c966674e9e7107f20
Signed-off-by: Chuck Short <chuck.short@canonical.com>

contrib/tempest/run_tempest_lxd.sh

@@ -18,9 +18,6 @@ ignored_tests="|^tempest.api.compute.images"
 
 # Regressions
 ignored_tests="$ignored_tests|.*AttachInterfacesTestJSON.test_create_list_show_delete_interfaces"
-ignored_tests="$ignored_tests|.*DeleteServersAdminTestJSON.test_admin_delete_servers_of_others"
-ignored_tests="$ignored_tests|.*DeleteServersAdminTestJSON.test_delete_server_while_in_error_state"
-ignored_tests="$ignored_tests|.*DeleteServersTestJSON.test_delete_server_while_in_building_state"
 
 # backups are not supported
 ignored_tests="$ignored_tests|.*ServerActionsTestJSON.test_create_backup"

nova/tests/unit/virt/lxd/test_driver.py

@@ -349,10 +349,8 @@ class LXDDriverTest(test.NoDBTestCase):
         self.assertEqual(expected_calls, execute.call_args_list)
 
     def test_destroy(self):
-        mock_profile = mock.Mock()
         mock_container = mock.Mock()
         mock_container.status = 'Running'
-        self.client.profiles.get.return_value = mock_profile
         self.client.containers.get.return_value = mock_container
         ctx = context.get_admin_context()
         instance = fake_instance.fake_instance_obj(ctx, name='test')
@@ -366,17 +364,34 @@ class LXDDriverTest(test.NoDBTestCase):
 
         lxd_driver.cleanup.assert_called_once_with(
             ctx, instance, network_info, None)
-        lxd_driver.client.profiles.get.assert_called_once_with(instance.name)
-        mock_profile.delete.assert_called_once_with()
         lxd_driver.client.containers.get.assert_called_once_with(instance.name)
         mock_container.stop.assert_called_once_with(wait=True)
         mock_container.delete.assert_called_once_with(wait=True)
 
+    def test_destroy_without_instance(self):
+        def side_effect(*args, **kwargs):
+            raise lxdcore_exceptions.LXDAPIException(MockResponse(404))
+        self.client.containers.get.side_effect = side_effect
+
+        ctx = context.get_admin_context()
+        instance = fake_instance.fake_instance_obj(ctx, name='test')
+        network_info = [mock.Mock()]
+
+        lxd_driver = driver.LXDDriver(None)
+        lxd_driver.init_host(None)
+        lxd_driver.cleanup = mock.Mock()  # There is a separate cleanup test
+
+        self.assertRaises(
+            lxdcore_exceptions.LXDAPIException,
+            lxd_driver.destroy, ctx, instance, network_info)
+
     @mock.patch('os.path.exists', mock.Mock(return_value=True))
     @mock.patch('pwd.getpwuid')
     @mock.patch('shutil.rmtree')
     @mock.patch.object(driver.utils, 'execute')
     def test_cleanup(self, execute, rmtree, getpwuid):
+        mock_profile = mock.Mock()
+        self.client.profiles.get.return_value = mock_profile
         pwuid = mock.Mock()
         pwuid.pw_name = 'user'
         getpwuid.return_value = pwuid
@@ -404,6 +419,7 @@ class LXDDriverTest(test.NoDBTestCase):
         execute.assert_called_once_with(
             'chown', '-R', 'user:user', instance_dir, run_as_root=True)
         rmtree.assert_called_once_with(instance_dir)
+        mock_profile.delete.assert_called_once_with()
 
     @mock.patch.object(driver.utils, 'execute')
     @mock.patch('nova.virt.driver.block_device_info_get_ephemerals')

nova/virt/lxd/driver.py

@@ -314,16 +314,20 @@ class LXDDriver(driver.ComputeDriver):
         See `nova.virt.driver.ComputeDriver.destroy` for more
         information.
         """
-        container = self.client.containers.get(instance.name)
         try:
+            container = self.client.containers.get(instance.name)
             if container.status != 'Stopped':
                 container.stop(wait=True)
         except lxd_exceptions.LXDAPIException as e:
             if e.response.status_code != 200:
                 raise
+            elif e.response.status_code == 404:
+                self.cleanup(
+                    context, instance, network_info, block_device_info)
+            else:
+                return
 
         container.delete(wait=True)
-        self.client.profiles.get(instance.name).delete()
 
         self.cleanup(context, instance, network_info, block_device_info)
 
@@ -356,6 +360,8 @@ class LXDDriver(driver.ComputeDriver):
                 container_dir, run_as_root=True)
             shutil.rmtree(container_dir)
 
+        self.client.profiles.get(instance.name).delete()
+
     def reboot(self, context, instance, network_info, reboot_type,
                block_device_info=None, bad_volumes_callback=None):
         """Reboot the container.