diff --git a/zun/compute/manager.py b/zun/compute/manager.py index e99820373..5127e8436 100644 --- a/zun/compute/manager.py +++ b/zun/compute/manager.py @@ -197,6 +197,29 @@ class Manager(periodic_task.PeriodicTasks): self._fail_container(context, container, msg, unset_host=True) raise exception.Conflict(msg) + def _wait_for_volumes_deleted(self, context, volumes, container, + timeout=60, poll_interval=1): + start_time = time.time() + try: + volumes = itertools.chain(volumes) + volume = next(volumes) + while time.time() - start_time < timeout: + if not volume.auto_remove: + volume = next(volumes) + is_deleted, is_error = self.driver.is_volume_deleted( + context, volume) + if is_deleted: + volume = next(volumes) + if is_error: + break + time.sleep(poll_interval) + except StopIteration: + return + msg = _("Volumes cannot be successfully deleted after " + "%d seconds") % (timeout) + self._fail_container(context, container, msg, unset_host=True) + raise exception.Conflict(msg) + def _check_support_disk_quota(self, context, container): base_device_size = self.driver.get_host_default_base_size() if base_device_size: @@ -395,6 +418,7 @@ class Manager(periodic_task.PeriodicTasks): self._detach_volume(context, volume, reraise=reraise) if volume.auto_remove and len(db_volumes) == 1: self.driver.delete_volume(context, volume) + self._wait_for_volumes_deleted(context, volumes, container) def _detach_volume(self, context, volume, reraise=True): context = context.elevated() diff --git a/zun/container/docker/driver.py b/zun/container/docker/driver.py index 2b8494507..e469e7627 100644 --- a/zun/container/docker/driver.py +++ b/zun/container/docker/driver.py @@ -1045,6 +1045,10 @@ class DockerDriver(driver.ContainerDriver): volume_driver = self._get_volume_driver(volume_mapping) return volume_driver.is_volume_available(context, volume_mapping) + def is_volume_deleted(self, context, volume_mapping): + volume_driver = self._get_volume_driver(volume_mapping) + return volume_driver.is_volume_deleted(context, volume_mapping) + def _get_or_create_docker_network(self, context, network_api, neutron_net_id): docker_net_name = self._get_docker_network_name(context, diff --git a/zun/container/driver.py b/zun/container/driver.py index c6f36515b..f837522f4 100644 --- a/zun/container/driver.py +++ b/zun/container/driver.py @@ -210,6 +210,9 @@ class ContainerDriver(object): def is_volume_available(self, context, volume_mapping): raise NotImplementedError() + def is_volume_deleted(self, context, volume_mapping): + raise NotImplementedError() + def add_security_group(self, context, container, security_group, **kwargs): raise NotImplementedError() diff --git a/zun/volume/driver.py b/zun/volume/driver.py index 573a3fcd0..71b6d855f 100644 --- a/zun/volume/driver.py +++ b/zun/volume/driver.py @@ -82,6 +82,9 @@ class VolumeDriver(object): def is_volume_available(self, context, volume): raise NotImplementedError() + def is_volume_deleted(self, context, volume): + raise NotImplementedError() + class Local(VolumeDriver): @@ -193,3 +196,19 @@ class Cinder(VolumeDriver): is_error = False return is_available, is_error + + @validate_volume_provider(supported_providers) + def is_volume_deleted(self, context, volume): + try: + volume = cinder_api.CinderAPI(context).search_volume( + volume.volume_id) + is_deleted = False + # Cinder volume error states: 'error', 'error_deleting', + # 'error_backing-up', 'error_restoring', 'error_extending', + # all of which start with 'error' + is_error = True if 'error' in volume.status else False + except exception.VolumeNotFound: + is_deleted = True + is_error = False + + return is_deleted, is_error