Introduce an attached volume migration test

This change introduces a true cinder host to host attached volume
migration test in addition to the existing attached volume retype test.
To enable this two new calls are introduced to the v3 volume client to
allow all volume backends to be listed per project and to also call for
a direct volume migration between backends.

Related-bug: #1803961
Depends-On: I1bdf3431bda2da98380e0dcaa9f952e6768ca3af
Change-Id: I501eb0cd5eb101b4dc553c2cdbc414693dd5b681

releasenotes/notes/add-migrate-volume-and-list-hosts-to-v3-volume-client-library-ad3529260db58f00.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add list host API support to the volume v3 client library.
+    This feature enables callers to list all hosts for a given project.
+  - |
+    Add migrate volume API support to the volume v3 client library.
+    This features allows callers to migrate volumes between backends.

tempest/common/waiters.py

@@ -213,6 +213,31 @@ def wait_for_volume_resource_status(client, resource_id, status):
              resource_name, resource_id, status, time.time() - start)
 
 
+def wait_for_volume_migration(client, volume_id, new_host):
+    """Waits for a Volume to move to a new host."""
+    body = client.show_volume(volume_id)['volume']
+    host = body['os-vol-host-attr:host']
+    migration_status = body['migration_status']
+    start = int(time.time())
+
+    # new_host is hostname@backend while current_host is hostname@backend#type
+    while migration_status != 'success' or new_host not in host:
+        time.sleep(client.build_interval)
+        body = client.show_volume(volume_id)['volume']
+        host = body['os-vol-host-attr:host']
+        migration_status = body['migration_status']
+
+        if migration_status == 'error':
+            message = ('volume %s failed to migrate.' % (volume_id))
+            raise lib_exc.TempestException(message)
+
+        if int(time.time()) - start >= client.build_timeout:
+            message = ('Volume %s failed to migrate to %s (current %s) '
+                       'within the required time (%s s).' %
+                       (volume_id, new_host, host, client.build_timeout))
+            raise lib_exc.TimeoutException(message)
+
+
 def wait_for_volume_retype(client, volume_id, new_volume_type):
     """Waits for a Volume to have a new volume type."""
     body = client.show_volume(volume_id)['volume']

tempest/lib/services/volume/v3/volumes_client.py

@@ -35,6 +35,16 @@ class VolumesClient(base_client.BaseClient):
             return params
         return urllib.urlencode(params)
 
+    def list_hosts(self):
+        """Lists all hosts summary info that is not disabled.
+
+        https://developer.openstack.org/api-ref/block-storage/v3/index.html#list-all-hosts-for-a-project
+        """
+        resp, body = self.get('os-hosts')
+        body = json.loads(body)
+        self.expected_success(200, resp.status)
+        return rest_client.ResponseBody(resp, body)
+
     def list_volumes(self, detail=False, params=None):
         """List all the volumes created.
 
@@ -55,6 +65,19 @@ class VolumesClient(base_client.BaseClient):
         self.expected_success(200, resp.status)
         return rest_client.ResponseBody(resp, body)
 
+    def migrate_volume(self, volume_id, **kwargs):
+        """Migrate a volume to a new backend
+
+        For a full list of available parameters please refer to the offical
+        API reference:
+
+        https://developer.openstack.org/api-ref/block-storage/v3/index.html#migrate-a-volume
+        """
+        post_body = json.dumps({'os-migrate_volume': kwargs})
+        resp, body = self.post('volumes/%s/action' % volume_id, post_body)
+        self.expected_success(202, resp.status)
+        return rest_client.ResponseBody(resp, body)
+
     def show_volume(self, volume_id):
         """Returns the details of a single volume."""
         url = "volumes/%s" % volume_id

tempest/scenario/test_volume_migrate_attached.py

@@ -97,7 +97,7 @@ class TestVolumeMigrateRetypeAttached(manager.ScenarioTest):
     @decorators.attr(type='slow')
     @decorators.idempotent_id('deadd2c2-beef-4dce-98be-f86765ff311b')
     @utils.services('compute', 'volume')
-    def test_volume_migrate_attached(self):
+    def test_volume_retype_attached(self):
         LOG.info("Creating keypair and security group")
         keypair = self.create_keypair()
         security_group = self._create_security_group()
@@ -149,3 +149,68 @@ class TestVolumeMigrateRetypeAttached(manager.ScenarioTest):
         attached_volumes = self.servers_client.list_volume_attachments(
             instance['id'])['volumeAttachments']
         self.assertEqual(volume_id, attached_volumes[0]['id'])
+
+    @decorators.attr(type='slow')
+    @decorators.idempotent_id('fe47b1ed-640e-4e3b-a090-200e25607362')
+    @utils.services('compute', 'volume')
+    def test_volume_migrate_attached(self):
+        LOG.info("Creating keypair and security group")
+        keypair = self.create_keypair()
+        security_group = self._create_security_group()
+
+        LOG.info("Creating volume")
+        # Create a unique volume type to avoid using the backend default
+        migratable_type = self.create_volume_type()['name']
+        volume_id = self.create_volume(imageRef=CONF.compute.image_ref,
+                                       volume_type=migratable_type)['id']
+        volume = self.admin_volumes_client.show_volume(volume_id)
+
+        LOG.info("Booting instance from volume")
+        instance = self._boot_instance_from_volume(volume_id, keypair,
+                                                   security_group)
+
+        # Identify the source and destination hosts for the migration
+        src_host = volume['volume']['os-vol-host-attr:host']
+
+        # Select the first c-vol host that isn't hosting the volume as the dest
+        # host['host_name'] should take the format of host@backend.
+        # src_host should take the format of host@backend#type
+        hosts = self.admin_volumes_client.list_hosts()['hosts']
+        for host in hosts:
+            if (host['service'] == 'cinder-volume' and
+                not src_host.startswith(host['host_name'])):
+                dest_host = host['host_name']
+                break
+
+        ip_instance = self.get_server_ip(instance)
+        timestamp = self.create_timestamp(ip_instance,
+                                          private_key=keypair['private_key'],
+                                          server=instance)
+
+        LOG.info("Migrating Volume %s from host %s to host %s",
+                 volume_id, src_host, dest_host)
+        self.admin_volumes_client.migrate_volume(volume_id, host=dest_host)
+
+        # This waiter asserts that the migration_status is success and that
+        # the volume has moved to the dest_host
+        waiters.wait_for_volume_migration(self.admin_volumes_client, volume_id,
+                                          dest_host)
+
+        # check the content of written file
+        LOG.info("Getting timestamp in postmigrated instance %s",
+                 instance['id'])
+        timestamp2 = self.get_timestamp(ip_instance,
+                                        private_key=keypair['private_key'],
+                                        server=instance)
+        self.assertEqual(timestamp, timestamp2)
+
+        # Assert that the volume is in-use
+        volume = self.admin_volumes_client.show_volume(volume_id)['volume']
+        self.assertEqual('in-use', volume['status'])
+
+        # Assert that the same volume id is attached to the instance, ensuring
+        # the os-migrate_volume_completion Cinder API has been called
+        attached_volumes = self.servers_client.list_volume_attachments(
+            instance['id'])['volumeAttachments']
+        attached_volume_id = attached_volumes[0]['id']
+        self.assertEqual(volume_id, attached_volume_id)

tempest/tests/common/test_waiters.py

@@ -148,3 +148,68 @@ class TestInterfaceWaiters(base.TestCase):
         list_interfaces.assert_has_calls([mock.call('server_id'),
                                           mock.call('server_id')])
         sleep.assert_called_once_with(client.build_interval)
+
+
+class TestVolumeWaiters(base.TestCase):
+    vol_migrating_src_host = {
+        'volume': {'migration_status': 'migrating',
+                   'os-vol-host-attr:host': 'src_host@backend#type'}}
+    vol_migrating_dst_host = {
+        'volume': {'migration_status': 'migrating',
+                   'os-vol-host-attr:host': 'dst_host@backend#type'}}
+    vol_migration_success = {
+        'volume': {'migration_status': 'success',
+                   'os-vol-host-attr:host': 'dst_host@backend#type'}}
+    vol_migration_error = {
+        'volume': {'migration_status': 'error',
+                   'os-vol-host-attr:host': 'src_host@backend#type'}}
+
+    def test_wait_for_volume_migration_timeout(self):
+        show_volume = mock.MagicMock(return_value=self.vol_migrating_src_host)
+        client = mock.Mock(spec=volumes_client.VolumesClient,
+                           resource_type="volume",
+                           build_interval=1,
+                           build_timeout=1,
+                           show_volume=show_volume)
+        self.patch('time.time', side_effect=[0., client.build_timeout + 1.])
+        self.patch('time.sleep')
+        self.assertRaises(lib_exc.TimeoutException,
+                          waiters.wait_for_volume_migration,
+                          client, mock.sentinel.volume_id, 'dst_host')
+
+    def test_wait_for_volume_migration_error(self):
+        show_volume = mock.MagicMock(side_effect=[
+            self.vol_migrating_src_host,
+            self.vol_migrating_src_host,
+            self.vol_migration_error])
+        client = mock.Mock(spec=volumes_client.VolumesClient,
+                           resource_type="volume",
+                           build_interval=1,
+                           build_timeout=1,
+                           show_volume=show_volume)
+        self.patch('time.time', return_value=0.)
+        self.patch('time.sleep')
+        self.assertRaises(lib_exc.TempestException,
+                          waiters.wait_for_volume_migration,
+                          client, mock.sentinel.volume_id, 'dst_host')
+
+    def test_wait_for_volume_migration_success_and_dst(self):
+        show_volume = mock.MagicMock(side_effect=[
+            self.vol_migrating_src_host,
+            self.vol_migrating_dst_host,
+            self.vol_migration_success])
+        client = mock.Mock(spec=volumes_client.VolumesClient,
+                           resource_type="volume",
+                           build_interval=1,
+                           build_timeout=1,
+                           show_volume=show_volume)
+        self.patch('time.time', return_value=0.)
+        self.patch('time.sleep')
+        waiters.wait_for_volume_migration(
+            client, mock.sentinel.volume_id, 'dst_host')
+
+        # Assert that we wait until migration_status is success and dst_host is
+        # part of the returned os-vol-host-attr:host.
+        show_volume.assert_has_calls([mock.call(mock.sentinel.volume_id),
+                                      mock.call(mock.sentinel.volume_id),
+                                      mock.call(mock.sentinel.volume_id)])