Avoid long wait for volume detach in some tests

When we attach a volume to a server we are going to delete later, we
set ourselves up for having to wait for a graceful detach of that
volume from the server, which is what the API provides. That should
always work, but if the guest has crashed or become unstable, we may
end up waiting for it until we timeout and fail, even though the
end goal is to just delete the server right after. So, this adds a
flag to attach_volume() that effectively says "let this be cleaned up
when the server is deleted." We still make the detach call, but we do
not wait for it to complete. This allows nova to force things during
the server delete process and avoid us getting hung up.

Further, since deleting the server is an asynchronous operation which
returns to the API caller immediately, we could race to delete the
volume before nova has finished server delete, making it available for
deletion. So, this also changes our create_volume() cleanup to use a
special routine that checks to see if the volume is attached and if it
is, waits for it to become available before attempting to delete the
volume.

This bites us in the gate quite a bit, in the force-snapshot-on-live-
instances test scenario, which is what the referenced gate bug is
about.

Change-Id: I027789668d7d9aebcd0418d9a45801771f1eccbd
Related-Bug: #1939108

tempest/api/volume/base.py

@@ -20,6 +20,7 @@ from tempest.lib.common import api_version_utils
 from tempest.lib.common.utils import data_utils
 from tempest.lib.common.utils import test_utils
 from tempest.lib.decorators import cleanup_order
+from tempest.lib import exceptions as lib_exc
 import tempest.test
 
 CONF = config.CONF
@@ -126,12 +127,32 @@ class BaseVolumeTest(api_version_utils.BaseMicroversionTest,
 
         volume = self.volumes_client.create_volume(**kwargs)['volume']
         self.cleanup(test_utils.call_and_ignore_notfound_exc,
-                     self.delete_volume, self.volumes_client, volume['id'])
+                     self._delete_volume_for_cleanup,
+                     self.volumes_client, volume['id'])
         if wait_until:
             waiters.wait_for_volume_resource_status(self.volumes_client,
                                                     volume['id'], wait_until)
         return volume
 
+    @staticmethod
+    def _delete_volume_for_cleanup(volumes_client, volume_id):
+        """Delete a volume (only) for cleanup.
+
+        If it is attached to a server, wait for it to become available,
+        assuming we have already deleted the server and just need nova to
+        complete the delete operation before it is available to be deleted.
+        Otherwise proceed to the regular delete_volume().
+        """
+        try:
+            vol = volumes_client.show_volume(volume_id)['volume']
+            if vol['status'] == 'in-use':
+                waiters.wait_for_volume_resource_status(volumes_client,
+                                                        volume_id,
+                                                        'available')
+        except lib_exc.NotFound:
+            pass
+        BaseVolumeTest.delete_volume(volumes_client, volume_id)
+
     @cleanup_order
     def create_snapshot(self, volume_id=1, **kwargs):
         """Wrapper utility that returns a test snapshot."""
@@ -183,16 +204,17 @@ class BaseVolumeTest(api_version_utils.BaseMicroversionTest,
         snapshots_client.delete_snapshot(snapshot_id)
         snapshots_client.wait_for_resource_deletion(snapshot_id)
 
-    def attach_volume(self, server_id, volume_id):
+    def attach_volume(self, server_id, volume_id, wait_for_detach=True):
         """Attach a volume to a server"""
         self.servers_client.attach_volume(
             server_id, volumeId=volume_id,
             device='/dev/%s' % CONF.compute.volume_device_name)
         waiters.wait_for_volume_resource_status(self.volumes_client,
                                                 volume_id, 'in-use')
-        self.addCleanup(waiters.wait_for_volume_resource_status,
-                        self.volumes_client, volume_id, 'available',
-                        server_id, self.servers_client)
+        if wait_for_detach:
+            self.addCleanup(waiters.wait_for_volume_resource_status,
+                            self.volumes_client, volume_id, 'available',
+                            server_id, self.servers_client)
         self.addCleanup(self.servers_client.detach_volume, server_id,
                         volume_id)
 

tempest/api/volume/test_volumes_snapshots.py

@@ -51,7 +51,10 @@ class VolumesSnapshotTestJSON(base.BaseVolumeTest):
         # Create a test instance
         server = self.create_server(wait_until='SSHABLE')
 
-        self.attach_volume(server['id'], volume['id'])
+        # NOTE(danms): We are attaching this volume to a server, but we do
+        # not need to block on detach during cleanup because we will be
+        # deleting the server anyway.
+        self.attach_volume(server['id'], volume['id'], wait_for_detach=False)
 
         # Snapshot a volume which attached to an instance with force=False
         self.assertRaises(lib_exc.BadRequest, self.create_snapshot,
@@ -83,7 +86,11 @@ class VolumesSnapshotTestJSON(base.BaseVolumeTest):
 
         # Create a server and attach it
         server = self.create_server(wait_until='SSHABLE')
-        self.attach_volume(server['id'], self.volume_origin['id'])
+        # NOTE(danms): We are attaching this volume to a server, but we do
+        # not need to block on detach during cleanup because we will be
+        # deleting the server anyway.
+        self.attach_volume(server['id'], self.volume_origin['id'],
+                           wait_for_detach=False)
 
         # Now that the volume is attached, create other snapshots
         snapshot2 = self.create_snapshot(self.volume_origin['id'], force=True)