Log console on failure to delete attachment

If we timeout waiting for delete attachment to complete, we should
log the guest console to see if it has crashed or has other error
messages that might indicate the reason.

Related-Bug: #1939108

Change-Id: If0ce1b7eb2064a7b91d2e426836c76171bfdff79

tempest/api/volume/base.py

@@ -191,7 +191,8 @@ class BaseVolumeTest(api_version_utils.BaseMicroversionTest,
         waiters.wait_for_volume_resource_status(self.volumes_client,
                                                 volume_id, 'in-use')
         self.addCleanup(waiters.wait_for_volume_resource_status,
-                        self.volumes_client, volume_id, 'available')
+                        self.volumes_client, volume_id, 'available',
+                        server_id, self.servers_client)
         self.addCleanup(self.servers_client.detach_volume, server_id,
                         volume_id)
 

tempest/common/waiters.py

@@ -303,12 +303,16 @@ def wait_for_image_copied_to_stores(client, image_id):
     raise lib_exc.TimeoutException(message)
 
 
-def wait_for_volume_resource_status(client, resource_id, status):
+def wait_for_volume_resource_status(client, resource_id, status,
+                                    server_id=None, servers_client=None):
     """Waits for a volume resource to reach a given status.
 
     This function is a common function for volume, snapshot and backup
     resources. The function extracts the name of the desired resource from
     the client class name of the resource.
+
+    If server_id and servers_client are provided, dump the console for that
+    server on failure.
     """
     resource_name = re.findall(
         r'(volume|group-snapshot|snapshot|backup|group)',
@@ -330,6 +334,11 @@ def wait_for_volume_resource_status(client, resource_id, status):
             raise exceptions.VolumeExtendErrorException(volume_id=resource_id)
 
         if int(time.time()) - start >= client.build_timeout:
+            if server_id and servers_client:
+                console_output = servers_client.get_console_output(
+                    server_id)['output']
+                LOG.debug('Console output for %s\nbody=\n%s',
+                          server_id, console_output)
             message = ('%s %s failed to reach %s status (current %s) '
                        'within the required time (%s s).' %
                        (resource_name, resource_id, status, resource_status,

tempest/tests/common/test_waiters.py

@@ -385,6 +385,29 @@ class TestVolumeWaiters(base.TestCase):
                                     mock.call(volume_id)])
         mock_sleep.assert_called_once_with(1)
 
+    @mock.patch.object(time, 'sleep')
+    def test_wait_for_volume_status_timeout_console(self, mock_sleep):
+        # Tests that the wait method gets the server console log if the
+        # timeout is hit.
+        client = mock.Mock(spec=volumes_client.VolumesClient,
+                           resource_type="volume",
+                           build_interval=1,
+                           build_timeout=1)
+        servers_client = mock.Mock()
+        servers_client.get_console_output.return_value = {
+            'output': 'console log'}
+        volume = {'volume': {'status': 'detaching'}}
+        mock_show = mock.Mock(return_value=volume)
+        client.show_volume = mock_show
+        volume_id = '7532b91e-aa0a-4e06-b3e5-20c0c5ee1caa'
+        self.assertRaises(lib_exc.TimeoutException,
+                          waiters.wait_for_volume_resource_status,
+                          client, volume_id, 'available',
+                          server_id='someserver',
+                          servers_client=servers_client)
+        servers_client.get_console_output.assert_called_once_with(
+            'someserver')
+
     @mock.patch.object(time, 'sleep')
     def test_wait_for_volume_status_error_extending(self, mock_sleep):
         # Tests that the wait method raises VolumeExtendErrorException if