Wait for cinder volume to be deleted

If users delete a container with auto-removed cinder volume(s),
it should wait for the cinder volumes to be removed in cinder side
before completing the container deletion process.

Change-Id: I38dc6766b167f66c0e3327d883fdeb5a79bef2ed

zun/compute/manager.py

@@ -197,6 +197,29 @@ class Manager(periodic_task.PeriodicTasks):
         self._fail_container(context, container, msg, unset_host=True)
         raise exception.Conflict(msg)
 
+    def _wait_for_volumes_deleted(self, context, volumes, container,
+                                  timeout=60, poll_interval=1):
+        start_time = time.time()
+        try:
+            volumes = itertools.chain(volumes)
+            volume = next(volumes)
+            while time.time() - start_time < timeout:
+                if not volume.auto_remove:
+                    volume = next(volumes)
+                is_deleted, is_error = self.driver.is_volume_deleted(
+                    context, volume)
+                if is_deleted:
+                    volume = next(volumes)
+                if is_error:
+                    break
+                time.sleep(poll_interval)
+        except StopIteration:
+            return
+        msg = _("Volumes cannot be successfully deleted after "
+                "%d seconds") % (timeout)
+        self._fail_container(context, container, msg, unset_host=True)
+        raise exception.Conflict(msg)
+
     def _check_support_disk_quota(self, context, container):
         base_device_size = self.driver.get_host_default_base_size()
         if base_device_size:
@@ -395,6 +418,7 @@ class Manager(periodic_task.PeriodicTasks):
             self._detach_volume(context, volume, reraise=reraise)
             if volume.auto_remove and len(db_volumes) == 1:
                 self.driver.delete_volume(context, volume)
+        self._wait_for_volumes_deleted(context, volumes, container)
 
     def _detach_volume(self, context, volume, reraise=True):
         context = context.elevated()

zun/container/docker/driver.py

@@ -1045,6 +1045,10 @@ class DockerDriver(driver.ContainerDriver):
         volume_driver = self._get_volume_driver(volume_mapping)
         return volume_driver.is_volume_available(context, volume_mapping)
 
+    def is_volume_deleted(self, context, volume_mapping):
+        volume_driver = self._get_volume_driver(volume_mapping)
+        return volume_driver.is_volume_deleted(context, volume_mapping)
+
     def _get_or_create_docker_network(self, context, network_api,
                                       neutron_net_id):
         docker_net_name = self._get_docker_network_name(context,

zun/container/driver.py

@@ -210,6 +210,9 @@ class ContainerDriver(object):
     def is_volume_available(self, context, volume_mapping):
         raise NotImplementedError()
 
+    def is_volume_deleted(self, context, volume_mapping):
+        raise NotImplementedError()
+
     def add_security_group(self, context, container, security_group, **kwargs):
         raise NotImplementedError()
 

zun/volume/driver.py

@@ -82,6 +82,9 @@ class VolumeDriver(object):
     def is_volume_available(self, context, volume):
         raise NotImplementedError()
 
+    def is_volume_deleted(self, context, volume):
+        raise NotImplementedError()
+
 
 class Local(VolumeDriver):
 
@@ -193,3 +196,19 @@ class Cinder(VolumeDriver):
             is_error = False
 
         return is_available, is_error
+
+    @validate_volume_provider(supported_providers)
+    def is_volume_deleted(self, context, volume):
+        try:
+            volume = cinder_api.CinderAPI(context).search_volume(
+                volume.volume_id)
+            is_deleted = False
+            # Cinder volume error states: 'error', 'error_deleting',
+            # 'error_backing-up', 'error_restoring', 'error_extending',
+            # all of which start with 'error'
+            is_error = True if 'error' in volume.status else False
+        except exception.VolumeNotFound:
+            is_deleted = True
+            is_error = False
+
+        return is_deleted, is_error