Move Share Migration code to Data Service

Removed functionality of Share Migration relying on Manila Share
Service node, moved code to Data Service node for copy phase.

Added parameter 'notify' and share/api methods for future
implementation (see dependent patches).

Added new copy operation statuses, in order to implement future
API calls to obtain progress and cancel migration.

Added possibility of 2-phase migration for driver migration and
generic (fallback) migration.

Added admin export location support and removed approach of
replacing IP with config parameter.

Added Admin-only API entry points to:
- Migration Cancel (only during copying)
- Reset Task State field
- Migration Get Progress (only during copying)
- Migration Complete (2nd phase migration)
- Notify parameter on Migrate Share

APIImpact
DocImpact

Implements: blueprint data-service-migration
Change-Id: I1d65aac2f36942cd70eb214be561d59a15a4ba26

manila_tempest_tests/config.py

@@ -36,7 +36,7 @@ ShareGroup = [
                help="The minimum api microversion is configured to be the "
                     "value of the minimum microversion supported by Manila."),
     cfg.StrOpt("max_api_microversion",
-               default="2.14",
+               default="2.15",
                help="The maximum api microversion is configured to be the "
                     "value of the latest microversion supported by Manila."),
     cfg.StrOpt("region",
@@ -183,7 +183,7 @@ ShareGroup = [
                default="100",
                help="Flavor used for client vm in scenario tests."),
     cfg.IntOpt("migration_timeout",
-               default=1200,
+               default=1500,
                help="Time to wait for share migration before "
                     "timing out (seconds)."),
     cfg.StrOpt("default_share_type_name",

manila_tempest_tests/services/share/v2/json/shares_client.py

@@ -945,16 +945,19 @@ class SharesV2Client(shares_client.SharesClient):
 
 ###############
 
-    def migrate_share(self, share_id, host, version=LATEST_MICROVERSION,
-                      action_name=None):
+    def migrate_share(self, share_id, host, notify,
+                      version=LATEST_MICROVERSION, action_name=None):
         if action_name is None:
-            if utils.is_microversion_gt(version, "2.6"):
+            if utils.is_microversion_lt(version, "2.7"):
+                action_name = 'os-migrate_share'
+            elif utils.is_microversion_lt(version, "2.15"):
                 action_name = 'migrate_share'
             else:
-                action_name = 'os-migrate_share'
+                action_name = 'migration_start'
         post_body = {
             action_name: {
                 'host': host,
+                'notify': notify,
             }
         }
         body = json.dumps(post_body)
@@ -962,27 +965,72 @@ class SharesV2Client(shares_client.SharesClient):
                          headers=EXPERIMENTAL, extra_headers=True,
                          version=version)
 
-    def wait_for_migration_completed(self, share_id, dest_host,
-                                     version=LATEST_MICROVERSION):
+    def migration_complete(self, share_id, version=LATEST_MICROVERSION,
+                           action_name='migration_complete'):
+        post_body = {
+            action_name: None,
+        }
+        body = json.dumps(post_body)
+        return self.post('shares/%s/action' % share_id, body,
+                         headers=EXPERIMENTAL, extra_headers=True,
+                         version=version)
+
+    def migration_cancel(self, share_id, version=LATEST_MICROVERSION,
+                         action_name='migration_cancel'):
+        post_body = {
+            action_name: None,
+        }
+        body = json.dumps(post_body)
+        return self.post('shares/%s/action' % share_id, body,
+                         headers=EXPERIMENTAL, extra_headers=True,
+                         version=version)
+
+    def migration_get_progress(self, share_id, version=LATEST_MICROVERSION,
+                               action_name='migration_get_progress'):
+        post_body = {
+            action_name: None,
+        }
+        body = json.dumps(post_body)
+        return self.post('shares/%s/action' % share_id, body,
+                         headers=EXPERIMENTAL, extra_headers=True,
+                         version=version)
+
+    def reset_task_state(
+            self, share_id, task_state, version=LATEST_MICROVERSION,
+            action_name='reset_task_state'):
+        post_body = {
+            action_name: {
+                'task_state': task_state,
+            }
+        }
+        body = json.dumps(post_body)
+        return self.post('shares/%s/action' % share_id, body,
+                         headers=EXPERIMENTAL, extra_headers=True,
+                         version=version)
+
+    def wait_for_migration_status(self, share_id, dest_host, status,
+                                  version=LATEST_MICROVERSION):
         """Waits for a share to migrate to a certain host."""
         share = self.get_share(share_id, version=version)
         migration_timeout = CONF.share.migration_timeout
         start = int(time.time())
-        while share['task_state'] != 'migration_success':
+        while share['task_state'] != status:
             time.sleep(self.build_interval)
             share = self.get_share(share_id, version=version)
-            if share['task_state'] == 'migration_success':
+            if share['task_state'] == status:
                 return share
             elif share['task_state'] == 'migration_error':
                 raise share_exceptions.ShareMigrationException(
                     share_id=share['id'], src=share['host'], dest=dest_host)
             elif int(time.time()) - start >= migration_timeout:
-                message = ('Share %(share_id)s failed to migrate from '
-                           'host %(src)s to host %(dest)s within the required '
-                           'time %(timeout)s.' % {
+                message = ('Share %(share_id)s failed to reach status '
+                           '%(status)s when migrating from host %(src)s to '
+                           'host %(dest)s within the required time '
+                           '%(timeout)s.' % {
                                'src': share['host'],
                                'dest': dest_host,
                                'share_id': share['id'],
-                               'timeout': self.build_timeout
+                               'timeout': self.build_timeout,
+                               'status': status,
                            })
                 raise exceptions.TimeoutException(message)

manila_tempest_tests/tests/api/admin/test_admin_actions.py

@@ -29,6 +29,8 @@ class AdminActionsTest(base.BaseSharesAdminTest):
     def resource_setup(cls):
         super(AdminActionsTest, cls).resource_setup()
         cls.states = ["error", "available"]
+        cls.task_states = ["migration_starting", "data_copying_in_progress",
+                           "migration_success"]
         cls.bad_status = "error_deleting"
         cls.sh = cls.create_share()
         cls.sh_instance = (
@@ -116,3 +118,11 @@ class AdminActionsTest(base.BaseSharesAdminTest):
         # Snapshot with status 'error_deleting' should be deleted
         self.shares_v2_client.force_delete(sn["id"], s_type="snapshots")
         self.shares_v2_client.wait_for_resource_deletion(snapshot_id=sn["id"])
+
+    @test.attr(type=["gate", ])
+    @base.skip_if_microversion_lt("2.15")
+    def test_reset_share_task_state(self):
+        for task_state in self.task_states:
+            self.shares_v2_client.reset_task_state(self.sh["id"], task_state)
+            self.shares_v2_client.wait_for_share_status(
+                self.sh["id"], task_state, 'task_state')

manila_tempest_tests/tests/api/admin/test_admin_actions_negative.py

@@ -166,3 +166,24 @@ class AdminActionsNegativeTest(base.BaseSharesAdminTest):
         self.assertRaises(lib_exc.Forbidden,
                           self.member_shares_v2_client.get_instances_of_share,
                           self.sh['id'])
+
+    @test.attr(type=["gate", "negative", ])
+    @base.skip_if_microversion_lt("2.15")
+    def test_reset_task_state_share_not_found(self):
+        self.assertRaises(
+            lib_exc.NotFound, self.shares_v2_client.reset_task_state,
+            'fake_share', 'migration_error')
+
+    @test.attr(type=["gate", "negative", ])
+    @base.skip_if_microversion_lt("2.15")
+    def test_reset_task_state_empty(self):
+        self.assertRaises(
+            lib_exc.BadRequest, self.shares_v2_client.reset_task_state,
+            self.sh['id'], None)
+
+    @test.attr(type=["gate", "negative", ])
+    @base.skip_if_microversion_lt("2.15")
+    def test_reset_task_state_invalid_state(self):
+        self.assertRaises(
+            lib_exc.BadRequest, self.shares_v2_client.reset_task_state,
+            self.sh['id'], 'fake_state')

manila_tempest_tests/tests/api/admin/test_migration.py

@@ -17,6 +17,7 @@ from tempest import config  # noqa
 from tempest import test  # noqa
 
 from manila_tempest_tests.tests.api import base
+from manila_tempest_tests import utils
 
 CONF = config.CONF
 
@@ -39,8 +40,45 @@ class MigrationNFSTest(base.BaseSharesAdminTest):
             raise cls.skipException("Migration tests disabled. Skipping.")
 
     @test.attr(type=["gate", ])
+    @base.skip_if_microversion_lt("2.5")
     def test_migration_empty_v2_5(self):
 
+        share, dest_pool = self._setup_migration()
+
+        old_exports = share['export_locations']
+
+        share = self.migrate_share(share['id'], dest_pool, version='2.5')
+
+        self._validate_migration_successful(dest_pool, share, old_exports,
+                                            version='2.5')
+
+    @test.attr(type=["gate", ])
+    @base.skip_if_microversion_lt("2.15")
+    def test_migration_completion_empty_v2_15(self):
+
+        share, dest_pool = self._setup_migration()
+
+        old_exports = self.shares_v2_client.list_share_export_locations(
+            share['id'], version='2.15')
+        self.assertNotEmpty(old_exports)
+        old_exports = [x['path'] for x in old_exports
+                       if x['is_admin_only'] is False]
+        self.assertNotEmpty(old_exports)
+
+        share = self.migrate_share(
+            share['id'], dest_pool, version='2.15', notify=False,
+            wait_for_status='data_copying_completed')
+
+        self._validate_migration_successful(dest_pool, share,
+                                            old_exports, '2.15', notify=False)
+
+        share = self.migration_complete(share['id'], dest_pool, version='2.15')
+
+        self._validate_migration_successful(dest_pool, share, old_exports,
+                                            version='2.15')
+
+    def _setup_migration(self):
+
         pools = self.shares_client.list_pools()['pools']
 
         if len(pools) < 2:
@@ -51,6 +89,18 @@ class MigrationNFSTest(base.BaseSharesAdminTest):
         share = self.create_share(self.protocol)
         share = self.shares_client.get_share(share['id'])
 
+        self.shares_v2_client.create_access_rule(
+            share['id'], access_to="50.50.50.50", access_level="rw")
+
+        self.shares_v2_client.wait_for_share_status(
+            share['id'], 'active', status_attr='access_rules_status')
+
+        self.shares_v2_client.create_access_rule(
+            share['id'], access_to="51.51.51.51", access_level="ro")
+
+        self.shares_v2_client.wait_for_share_status(
+            share['id'], 'active', status_attr='access_rules_status')
+
         dest_pool = next((x for x in pools if x['name'] != share['host']),
                          None)
 
@@ -59,10 +109,30 @@ class MigrationNFSTest(base.BaseSharesAdminTest):
 
         dest_pool = dest_pool['name']
 
-        old_export_location = share['export_locations'][0]
+        return share, dest_pool
 
-        share = self.migrate_share(share['id'], dest_pool, version='2.5')
+    def _validate_migration_successful(self, dest_pool, share,
+                                       old_exports, version, notify=True):
+        if utils.is_microversion_lt(version, '2.9'):
+            new_exports = share['export_locations']
+            self.assertNotEmpty(new_exports)
+        else:
+            new_exports = self.shares_v2_client.list_share_export_locations(
+                share['id'], version='2.9')
+            self.assertNotEmpty(new_exports)
+            new_exports = [x['path'] for x in new_exports if
+                           x['is_admin_only'] is False]
+            self.assertNotEmpty(new_exports)
 
-        self.assertEqual(dest_pool, share['host'])
-        self.assertNotEqual(old_export_location, share['export_locations'][0])
-        self.assertEqual('migration_success', share['task_state'])
+        # Share migrated
+        if notify:
+            self.assertEqual(dest_pool, share['host'])
+            for export in old_exports:
+                self.assertFalse(export in new_exports)
+            self.assertEqual('migration_success', share['task_state'])
+        # Share not migrated yet
+        else:
+            self.assertNotEqual(dest_pool, share['host'])
+            for export in old_exports:
+                self.assertTrue(export in new_exports)
+            self.assertEqual('data_copying_completed', share['task_state'])

manila_tempest_tests/tests/api/admin/test_migration_negative.py

@@ -0,0 +1,97 @@
+# Copyright 2015 Hitachi Data Systems.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from tempest import config  # noqa
+from tempest.lib import exceptions as lib_exc  # noqa
+from tempest import test  # noqa
+
+from manila_tempest_tests.tests.api import base
+
+CONF = config.CONF
+
+
+class MigrationNFSTest(base.BaseSharesAdminTest):
+    """Tests Share Migration.
+
+    Tests migration in multi-backend environment.
+    """
+
+    protocol = "nfs"
+
+    @classmethod
+    def resource_setup(cls):
+        super(MigrationNFSTest, cls).resource_setup()
+        if not CONF.share.run_migration_tests:
+            raise cls.skipException("Migration tests disabled. Skipping.")
+
+        cls.share = cls.create_share(cls.protocol)
+        cls.share = cls.shares_client.get_share(cls.share['id'])
+        pools = cls.shares_client.list_pools()['pools']
+
+        if len(pools) < 2:
+            raise cls.skipException("At least two different pool entries "
+                                    "are needed to run migration tests. "
+                                    "Skipping.")
+        cls.dest_pool = next((x for x in pools
+                              if x['name'] != cls.share['host']), None)
+
+    @test.attr(type=["gate", "negative", ])
+    @base.skip_if_microversion_lt("2.15")
+    def test_migration_cancel_invalid(self):
+        self.assertRaises(
+            lib_exc.BadRequest, self.shares_v2_client.migration_cancel,
+            self.share['id'])
+
+    @test.attr(type=["gate", "negative", ])
+    @base.skip_if_microversion_lt("2.15")
+    def test_migration_get_progress_invalid(self):
+        self.assertRaises(
+            lib_exc.BadRequest, self.shares_v2_client.migration_get_progress,
+            self.share['id'])
+
+    @test.attr(type=["gate", "negative", ])
+    @base.skip_if_microversion_lt("2.15")
+    def test_migration_complete_invalid(self):
+        self.assertRaises(
+            lib_exc.BadRequest, self.shares_v2_client.migration_complete,
+            self.share['id'])
+
+    @test.attr(type=["gate", "negative", ])
+    @base.skip_if_microversion_lt("2.5")
+    def test_migrate_share_with_snapshot_v2_5(self):
+        snap = self.create_snapshot_wait_for_active(self.share['id'])
+        self.assertRaises(
+            lib_exc.BadRequest, self.shares_v2_client.migrate_share,
+            self.share['id'], self.dest_pool, True, version='2.5')
+        self.shares_client.delete_snapshot(snap['id'])
+        self.shares_client.wait_for_resource_deletion(snapshot_id=snap["id"])
+
+    @test.attr(type=["gate", "negative", ])
+    @base.skip_if_microversion_lt("2.5")
+    def test_migrate_share_same_host_v2_5(self):
+        self.assertRaises(
+            lib_exc.BadRequest, self.shares_v2_client.migrate_share,
+            self.share['id'], self.share['host'], True, version='2.5')
+
+    @test.attr(type=["gate", "negative", ])
+    @base.skip_if_microversion_lt("2.5")
+    def test_migrate_share_not_available_v2_5(self):
+        self.shares_client.reset_state(self.share['id'], 'error')
+        self.shares_client.wait_for_share_status(self.share['id'], 'error')
+        self.assertRaises(
+            lib_exc.BadRequest, self.shares_v2_client.migrate_share,
+            self.share['id'], self.dest_pool, True, version='2.5')
+        self.shares_client.reset_state(self.share['id'], 'available')
+        self.shares_client.wait_for_share_status(self.share['id'], 'available')

manila_tempest_tests/tests/api/base.py

@@ -343,11 +343,22 @@ class BaseSharesTest(test.BaseTestCase):
         return share
 
     @classmethod
-    def migrate_share(cls, share_id, dest_host, client=None, **kwargs):
+    def migrate_share(cls, share_id, dest_host, client=None, notify=True,
+                      wait_for_status='migration_success', **kwargs):
         client = client or cls.shares_v2_client
-        client.migrate_share(share_id, dest_host, **kwargs)
-        share = client.wait_for_migration_completed(
-            share_id, dest_host, version=kwargs.get('version'))
+        client.migrate_share(share_id, dest_host, notify, **kwargs)
+        share = client.wait_for_migration_status(
+            share_id, dest_host, wait_for_status,
+            version=kwargs.get('version'))
+        return share
+
+    @classmethod
+    def migration_complete(cls, share_id, dest_host, client=None, **kwargs):
+        client = client or cls.shares_v2_client
+        client.migration_complete(share_id, **kwargs)
+        share = client.wait_for_migration_status(
+            share_id, dest_host, 'migration_success',
+            version=kwargs.get('version'))
         return share
 
     @classmethod

manila_tempest_tests/tests/scenario/manager_share.py

@@ -196,8 +196,9 @@ class ShareScenarioTest(manager.NetworkScenarioTest):
 
     def _migrate_share(self, share_id, dest_host, client=None):
         client = client or self.shares_admin_v2_client
-        client.migrate_share(share_id, dest_host)
-        share = client.wait_for_migration_completed(share_id, dest_host)
+        client.migrate_share(share_id, dest_host, True)
+        share = client.wait_for_migration_status(share_id, dest_host,
+                                                 'migration_success')
         return share
 
     def _create_share_type(self, name, is_public=True, **kwargs):