Browse Source

Fix Share Migration improper behavior for drivers

Tempest tests were not appropriate for driver-assisted migration,
so this was fixed.

Also, improved docstrings and fixed workflow for drivers when
implementing 2-phase migration to be accurate with tempest and
handle AZs, which were previously locked to the source share's
AZ.

Driver-assisted migration now creates an additional
share instance to better handle and support driver methods.

Updated allow_access and deny_access APIs to allow users to mount
migrating shares before issuing 'migration-complete'.

APIImpact

Closes-bug: #1594922
Change-Id: If4bfaf7e9d963b83c13a6fea241c2eda14f7f409
changes/67/332267/74
Rodrigo Barbieri 5 years ago
parent
commit
c7fe51e79b
  1. 8
      manila/api/views/shares.py
  2. 8
      manila/data/manager.py
  3. 9
      manila/exception.py
  4. 6
      manila/scheduler/manager.py
  5. 245
      manila/share/api.py
  6. 181
      manila/share/driver.py
  7. 631
      manila/share/manager.py
  8. 22
      manila/share/migration.py
  9. 76
      manila/share/rpcapi.py
  10. 5
      manila/tests/api/v2/test_shares.py
  11. 4
      manila/tests/data/test_manager.py
  12. 9
      manila/tests/scheduler/test_manager.py
  13. 307
      manila/tests/share/test_api.py
  14. 43
      manila/tests/share/test_driver.py
  15. 858
      manila/tests/share/test_manager.py
  16. 47
      manila/tests/share/test_migration.py
  17. 64
      manila/tests/share/test_rpcapi.py
  18. 16
      manila_tempest_tests/common/constants.py
  19. 22
      manila_tempest_tests/services/share/v2/json/shares_client.py
  20. 82
      manila_tempest_tests/tests/api/admin/test_migration.py
  21. 43
      manila_tempest_tests/tests/api/admin/test_migration_negative.py
  22. 8
      manila_tempest_tests/tests/api/base.py
  23. 32
      manila_tempest_tests/tests/scenario/test_share_basic_ops.py
  24. 15
      manila_tempest_tests/utils.py

8
manila/api/views/shares.py

@ -96,11 +96,9 @@ class ViewBuilder(common.ViewBuilder):
return {'share': share_dict}
def migration_get_progress(self, progress):
result = {
'total_progress': progress['total_progress'],
'current_file_path': progress['current_file_path'],
'current_file_progress': progress['current_file_progress']
}
result = {'total_progress': progress['total_progress']}
return result
@common.ViewBuilder.versioned_method("2.2")

8
manila/data/manager.py

@ -73,6 +73,8 @@ class DataManager(manager.Manager):
'dest_instance_id': dest_share_instance_id})
share_ref = self.db.share_get(context, share_id)
share_instance_ref = self.db.share_instance_get(
context, share_instance_id, with_share_data=True)
share_rpcapi = share_rpc.ShareAPI()
@ -90,7 +92,7 @@ class DataManager(manager.Manager):
migration_info_dest)
except exception.ShareDataCopyCancelled:
share_rpcapi.migration_complete(
context, share_ref, share_instance_id, dest_share_instance_id)
context, share_instance_ref, dest_share_instance_id)
return
except Exception:
self.db.share_update(
@ -101,7 +103,7 @@ class DataManager(manager.Manager):
'dest': dest_share_instance_id}
LOG.exception(msg)
share_rpcapi.migration_complete(
context, share_ref, share_instance_id, dest_share_instance_id)
context, share_instance_ref, dest_share_instance_id)
raise exception.ShareDataCopyFailed(reason=msg)
finally:
self.busy_tasks_shares.pop(share_id, None)
@ -121,7 +123,7 @@ class DataManager(manager.Manager):
'dest_instance_id': dest_share_instance_id})
share_rpcapi.migration_complete(
context, share_ref, share_instance_id, dest_share_instance_id)
context, share_instance_ref, dest_share_instance_id)
def data_copy_cancel(self, context, share_id):
LOG.info(_LI("Received request to cancel share migration "

9
manila/exception.py

@ -242,6 +242,10 @@ class InvalidShareServer(Invalid):
message = _("Share server %(share_server_id)s is not valid.")
class ShareMigrationError(ManilaException):
message = _("Error in share migration: %(reason)s")
class ShareMigrationFailed(ManilaException):
message = _("Share migration failed: %(reason)s")
@ -267,6 +271,11 @@ class ShareServerNotCreated(ManilaException):
message = _("Share server %(share_server_id)s failed on creation.")
class ShareServerNotReady(ManilaException):
message = _("Share server %(share_server_id)s failed to reach '%(state)s' "
"within %(time)s seconds.")
class ServiceNotFound(NotFound):
message = _("Service %(service_id)s could not be found.")

6
manila/scheduler/manager.py

@ -159,9 +159,6 @@ class SchedulerManager(manager.Manager):
request_spec,
filter_properties)
except exception.NoValidHost as ex:
with excutils.save_and_reraise_exception():
_migrate_share_set_error(self, context, ex, request_spec)
except Exception as ex:
with excutils.save_and_reraise_exception():
_migrate_share_set_error(self, context, ex, request_spec)
@ -169,7 +166,8 @@ class SchedulerManager(manager.Manager):
share_ref = db.share_get(context, share_id)
try:
share_rpcapi.ShareAPI().migration_start(
context, share_ref, tgt_host, force_host_copy, notify)
context, share_ref, tgt_host.host, force_host_copy,
notify)
except Exception as ex:
with excutils.save_and_reraise_exception():
_migrate_share_set_error(self, context, ex, request_spec)

245
manila/share/api.py

@ -271,7 +271,7 @@ class API(base.Base):
policy.check_policy(context, 'share', 'create')
request_spec, share_instance = (
self._create_share_instance_and_get_request_spec(
self.create_share_instance_and_get_request_spec(
context, share, availability_zone=availability_zone,
consistency_group=consistency_group, host=host,
share_network_id=share_network_id))
@ -307,7 +307,7 @@ class API(base.Base):
return share_instance
def _create_share_instance_and_get_request_spec(
def create_share_instance_and_get_request_spec(
self, context, share, availability_zone=None,
consistency_group=None, host=None, share_network_id=None):
@ -393,7 +393,7 @@ class API(base.Base):
raise exception.ReplicationException(reason=msg % share['id'])
request_spec, share_replica = (
self._create_share_instance_and_get_request_spec(
self.create_share_instance_and_get_request_spec(
context, share, availability_zone=availability_zone,
share_network_id=share_network_id))
@ -874,7 +874,7 @@ class API(base.Base):
return snapshot
def migration_start(self, context, share, host, force_host_copy,
def migration_start(self, context, share, dest_host, force_host_copy,
notify=True):
"""Migrates share to a new host."""
@ -899,10 +899,10 @@ class API(base.Base):
self._check_is_share_busy(share)
# Make sure the destination host is different than the current one
if host == share_instance['host']:
if dest_host == share_instance['host']:
msg = _('Destination host %(dest_host)s must be different '
'than the current host %(src_host)s.') % {
'dest_host': host,
'dest_host': dest_host,
'src_host': share_instance['host']}
raise exception.InvalidHost(reason=msg)
@ -912,8 +912,23 @@ class API(base.Base):
msg = _("Share %s must not have snapshots.") % share['id']
raise exception.InvalidShare(reason=msg)
dest_host_host = share_utils.extract_host(dest_host)
# Make sure the host is in the list of available hosts
utils.validate_service_host(context, share_utils.extract_host(host))
utils.validate_service_host(context, dest_host_host)
service = self.db.service_get_by_args(
context, dest_host_host, 'manila-share')
share_type = {}
share_type_id = share['share_type_id']
if share_type_id:
share_type = share_types.get_share_type(context, share_type_id)
request_spec = self._get_request_spec_dict(
share,
share_type,
availability_zone_id=service['availability_zone_id'])
# NOTE(ganso): there is the possibility of an error between here and
# manager code, which will cause the share to be stuck in
@ -925,21 +940,14 @@ class API(base.Base):
context, share,
{'task_state': constants.TASK_STATE_MIGRATION_STARTING})
share_type = {}
share_type_id = share['share_type_id']
if share_type_id:
share_type = share_types.get_share_type(context, share_type_id)
request_spec = self._get_request_spec_dict(share, share_type)
try:
self.scheduler_rpcapi.migrate_share_to_host(context, share['id'],
host, force_host_copy,
notify, request_spec)
self.scheduler_rpcapi.migrate_share_to_host(
context, share['id'], dest_host, force_host_copy, notify,
request_spec)
except Exception:
msg = _('Destination host %(dest_host)s did not pass validation '
'for migration of share %(share)s.') % {
'dest_host': host,
'dest_host': dest_host,
'share': share['id']}
raise exception.InvalidHost(reason=msg)
@ -948,64 +956,150 @@ class API(base.Base):
if share['task_state'] not in (
constants.TASK_STATE_DATA_COPYING_COMPLETED,
constants.TASK_STATE_MIGRATION_DRIVER_PHASE1_DONE):
msg = _("First migration phase of share %s not completed"
" yet.") % share['id']
msg = self._migration_validate_error_message(share)
if msg is None:
msg = _("First migration phase of share %s not completed"
" yet.") % share['id']
LOG.error(msg)
raise exception.InvalidShare(reason=msg)
share_instance_id, new_share_instance_id = (
self.get_migrating_instances(share))
share_instance_ref = self.db.share_instance_get(
context, share_instance_id, with_share_data=True)
self.share_rpcapi.migration_complete(context, share_instance_ref,
new_share_instance_id)
def get_migrating_instances(self, share):
share_instance_id = None
new_share_instance_id = None
if share['task_state'] == (
constants.TASK_STATE_DATA_COPYING_COMPLETED):
for instance in share.instances:
if instance['status'] == constants.STATUS_MIGRATING:
share_instance_id = instance['id']
if instance['status'] == constants.STATUS_MIGRATING_TO:
new_share_instance_id = instance['id']
if None in (share_instance_id, new_share_instance_id):
msg = _("Share instances %(instance_id)s and "
"%(new_instance_id)s in inconsistent states, cannot"
" continue share migration for share %(share_id)s"
".") % {'instance_id': share_instance_id,
'new_instance_id': new_share_instance_id,
'share_id': share['id']}
raise exception.ShareMigrationFailed(reason=msg)
share_rpc = share_rpcapi.ShareAPI()
share_rpc.migration_complete(context, share, share_instance_id,
new_share_instance_id)
for instance in share.instances:
if instance['status'] == constants.STATUS_MIGRATING:
share_instance_id = instance['id']
if instance['status'] == constants.STATUS_MIGRATING_TO:
new_share_instance_id = instance['id']
if None in (share_instance_id, new_share_instance_id):
msg = _("Share instances %(instance_id)s and "
"%(new_instance_id)s in inconsistent states, cannot"
" continue share migration for share %(share_id)s"
".") % {'instance_id': share_instance_id,
'new_instance_id': new_share_instance_id,
'share_id': share['id']}
raise exception.ShareMigrationFailed(reason=msg)
return share_instance_id, new_share_instance_id
def migration_get_progress(self, context, share):
if share['task_state'] == (
constants.TASK_STATE_MIGRATION_DRIVER_IN_PROGRESS):
share_rpc = share_rpcapi.ShareAPI()
return share_rpc.migration_get_progress(context, share)
share_instance_id, migrating_instance_id = (
self.get_migrating_instances(share))
share_instance_ref = self.db.share_instance_get(
context, share_instance_id, with_share_data=True)
service_host = share_utils.extract_host(share_instance_ref['host'])
service = self.db.service_get_by_args(
context, service_host, 'manila-share')
if utils.service_is_up(service):
try:
result = self.share_rpcapi.migration_get_progress(
context, share_instance_ref, migrating_instance_id)
except Exception:
msg = _("Failed to obtain migration progress of share "
"%s.") % share['id']
LOG.exception(msg)
raise exception.ShareMigrationError(reason=msg)
else:
result = None
elif share['task_state'] == (
constants.TASK_STATE_DATA_COPYING_IN_PROGRESS):
data_rpc = data_rpcapi.DataAPI()
LOG.info(_LI("Sending request to get share migration information"
" of share %s.") % share['id'])
return data_rpc.data_copy_get_progress(context, share['id'])
services = self.db.service_get_all_by_topic(context, 'manila-data')
if len(services) > 0 and utils.service_is_up(services[0]):
try:
result = data_rpc.data_copy_get_progress(
context, share['id'])
except Exception:
msg = _("Failed to obtain migration progress of share "
"%s.") % share['id']
LOG.exception(msg)
raise exception.ShareMigrationError(reason=msg)
else:
result = None
else:
msg = _("Migration of share %s data copy progress cannot be "
"obtained at this moment.") % share['id']
result = None
if not (result and result.get('total_progress') is not None):
msg = self._migration_validate_error_message(share)
if msg is None:
msg = _("Migration progress of share %s cannot be obtained at "
"this moment.") % share['id']
LOG.error(msg)
raise exception.InvalidShare(reason=msg)
return result
def _migration_validate_error_message(self, share):
task_state = share['task_state']
if task_state == constants.TASK_STATE_MIGRATION_SUCCESS:
msg = _("Migration of share %s has already "
"completed.") % share['id']
elif task_state in (None, constants.TASK_STATE_MIGRATION_ERROR):
msg = _("There is no migration being performed for share %s "
"at this moment.") % share['id']
elif task_state == constants.TASK_STATE_MIGRATION_CANCELLED:
msg = _("Migration of share %s was already "
"cancelled.") % share['id']
elif task_state in (constants.TASK_STATE_MIGRATION_DRIVER_PHASE1_DONE,
constants.TASK_STATE_DATA_COPYING_COMPLETED):
msg = _("Migration of share %s has already completed first "
"phase.") % share['id']
else:
return None
return msg
def migration_cancel(self, context, share):
if share['task_state'] == (
migrating = True
if share['task_state'] in (
constants.TASK_STATE_DATA_COPYING_COMPLETED,
constants.TASK_STATE_MIGRATION_DRIVER_PHASE1_DONE,
constants.TASK_STATE_MIGRATION_DRIVER_IN_PROGRESS):
share_rpc = share_rpcapi.ShareAPI()
share_rpc.migration_cancel(context, share)
share_instance_id, migrating_instance_id = (
self.get_migrating_instances(share))
share_instance_ref = self.db.share_instance_get(
context, share_instance_id, with_share_data=True)
service_host = share_utils.extract_host(share_instance_ref['host'])
service = self.db.service_get_by_args(
context, service_host, 'manila-share')
if utils.service_is_up(service):
self.share_rpcapi.migration_cancel(
context, share_instance_ref, migrating_instance_id)
else:
migrating = False
elif share['task_state'] == (
constants.TASK_STATE_DATA_COPYING_IN_PROGRESS):
@ -1013,11 +1107,28 @@ class API(base.Base):
data_rpc = data_rpcapi.DataAPI()
LOG.info(_LI("Sending request to cancel migration of "
"share %s.") % share['id'])
data_rpc.data_copy_cancel(context, share['id'])
services = self.db.service_get_all_by_topic(context, 'manila-data')
if len(services) > 0 and utils.service_is_up(services[0]):
try:
data_rpc.data_copy_cancel(context, share['id'])
except Exception:
msg = _("Failed to cancel migration of share "
"%s.") % share['id']
LOG.exception(msg)
raise exception.ShareMigrationError(reason=msg)
else:
migrating = False
else:
msg = _("Data copy for migration of share %s cannot be cancelled"
" at this moment.") % share['id']
migrating = False
if not migrating:
msg = self._migration_validate_error_message(share)
if msg is None:
msg = _("Migration of share %s cannot be cancelled at this "
"moment.") % share['id']
LOG.error(msg)
raise exception.InvalidShare(reason=msg)
@ -1186,7 +1297,20 @@ class API(base.Base):
policy.check_policy(ctx, 'share', 'allow_access')
share = self.db.share_get(ctx, share['id'])
if share['status'] != constants.STATUS_AVAILABLE:
msg = _("Share status must be %s") % constants.STATUS_AVAILABLE
if not (share['status'] in (constants.STATUS_MIGRATING,
constants.STATUS_MIGRATING_TO) and
share['task_state'] in (
constants.TASK_STATE_DATA_COPYING_ERROR,
constants.TASK_STATE_MIGRATION_ERROR,
constants.TASK_STATE_MIGRATION_DRIVER_PHASE1_DONE,
constants.TASK_STATE_DATA_COPYING_COMPLETED)):
msg = _("Share status must be %(available)s, or %(migrating)s "
"while first phase of migration is completed.") % {
'available': constants.STATUS_AVAILABLE,
'migrating': constants.STATUS_MIGRATING
}
else:
msg = _("Share status must be %s") % constants.STATUS_AVAILABLE
raise exception.InvalidShare(reason=msg)
values = {
'share_id': share['id'],
@ -1258,7 +1382,20 @@ class API(base.Base):
msg = _("Share doesn't have any instances")
raise exception.InvalidShare(reason=msg)
if share['status'] != constants.STATUS_AVAILABLE:
msg = _("Share status must be %s") % constants.STATUS_AVAILABLE
if not (share['status'] in (constants.STATUS_MIGRATING,
constants.STATUS_MIGRATING_TO) and
share['task_state'] in (
constants.TASK_STATE_DATA_COPYING_ERROR,
constants.TASK_STATE_MIGRATION_ERROR,
constants.TASK_STATE_MIGRATION_DRIVER_PHASE1_DONE,
constants.TASK_STATE_DATA_COPYING_COMPLETED)):
msg = _("Share status must be %(available)s, or %(migrating)s "
"while first phase of migration is completed.") % {
'available': constants.STATUS_AVAILABLE,
'migrating': constants.STATUS_MIGRATING
}
else:
msg = _("Share status must be %s") % constants.STATUS_AVAILABLE
raise exception.InvalidShare(reason=msg)
for share_instance in share.instances:

181
manila/share/driver.py

@ -316,86 +316,156 @@ class ShareDriver(object):
{'actual': self.driver_handles_share_servers,
'allowed': driver_handles_share_servers})
def migration_start(self, context, share_ref, share_server, host,
dest_driver_migration_info, notify):
"""Is called to perform 1st phase of driver migration of a given share.
def migration_check_compatibility(
self, context, source_share, destination_share,
share_server=None, destination_share_server=None):
"""Checks destination compatibility for migration of a given share.
.. note::
Is called to test compatibility with destination backend.
Based on destination_driver_migration_info, driver should check if it
is compatible with destination backend so optimized migration can
proceed.
:param context: The 'context.RequestContext' object for the request.
:param source_share: Reference to the share to be migrated.
:param destination_share: Reference to the share model to be used by
migrated share.
:param share_server: Share server model or None.
:param destination_share_server: Destination Share server model or
None.
:return: A dictionary containing values indicating if destination
backend is compatible and if share can remain writable during
migration.
Example::
{
'compatible': True,
'writable': True,
}
"""
return {
'compatible': False,
'writable': False,
}
def migration_start(
self, context, source_share, destination_share,
share_server=None, destination_share_server=None):
"""Starts migration of a given share to another host.
.. note::
Is called in source share's backend to start migration.
Driver should implement this method if willing to perform migration
in an optimized way, useful for when driver understands destination
backend.
in an optimized way, useful for when source share's backend driver
is compatible with destination backend driver. This method should
start the migration procedure in the backend and end. Following steps
should be done in 'migration_continue'.
:param context: The 'context.RequestContext' object for the request.
:param share_ref: Reference to the share being migrated.
:param source_share: Reference to the original share model.
:param destination_share: Reference to the share model to be used by
migrated share.
:param share_server: Share server model or None.
:param host: Destination host and its capabilities.
:param dest_driver_migration_info: Migration information provided by
destination host.
:param notify: whether the migration should complete or wait for
2nd phase call. Driver may throw exception when validating this
parameter, exception if does not support 1-phase or 2-phase approach.
:returns: Boolean value indicating if driver migration succeeded.
:returns: Dictionary containing a model update with relevant data to
be updated after migration, such as export locations.
:param destination_share_server: Destination Share server model or
None.
"""
return None, None
raise NotImplementedError()
def migration_complete(self, context, share_ref, share_server,
dest_driver_migration_info):
"""Is called to perform 2nd phase of driver migration of a given share.
def migration_continue(
self, context, source_share, destination_share,
share_server=None, destination_share_server=None):
"""Continues migration of a given share to another host.
.. note::
Is called in source share's backend to continue migration.
Driver should implement this method to continue monitor the migration
progress in storage and perform following steps until 1st phase is
completed.
If driver is implementing 2-phase migration, this method should
perform tasks related to the 2nd phase of migration, thus completing
it.
:param context: The 'context.RequestContext' object for the request.
:param share_ref: Reference to the share being migrated.
:param source_share: Reference to the original share model.
:param destination_share: Reference to the share model to be used by
migrated share.
:param share_server: Share server model or None.
:param dest_driver_migration_info: Migration information provided by
destination host.
:returns: Dictionary containing a model update with relevant data to
be updated after migration, such as export locations.
:param destination_share_server: Destination Share server model or
None.
:return: Boolean value to indicate if 1st phase is finished.
"""
return None
raise NotImplementedError()
def migration_cancel(self, context, share_ref, share_server,
dest_driver_migration_info):
"""Is called to cancel driver migration.
def migration_complete(
self, context, source_share, destination_share,
share_server=None, destination_share_server=None):
"""Completes migration of a given share to another host.
.. note::
Is called in source share's backend to complete migration.
If driver is implementing 2-phase migration, this method should
perform the disruptive tasks related to the 2nd phase of migration,
thus completing it. Driver should also delete all original share data
from source backend.
If possible, driver can implement a way to cancel an in-progress
migration.
:param context: The 'context.RequestContext' object for the request.
:param share_ref: Reference to the share being migrated.
:param source_share: Reference to the original share model.
:param destination_share: Reference to the share model to be used by
migrated share.
:param share_server: Share server model or None.
:param dest_driver_migration_info: Migration information provided by
destination host.
:param destination_share_server: Destination Share server model or
None.
:return: List of export locations to update the share with.
"""
raise NotImplementedError()
def migration_get_progress(self, context, share_ref, share_server,
dest_driver_migration_info):
"""Is called to get migration progress.
def migration_cancel(
self, context, source_share, destination_share,
share_server=None, destination_share_server=None):
"""Cancels migration of a given share to another host.
.. note::
Is called in source share's backend to cancel migration.
If possible, driver can implement a way to cancel an in-progress
migration.
If possible, driver can implement a way to return migration progress
information.
:param context: The 'context.RequestContext' object for the request.
:param share_ref: Reference to the share being migrated.
:param source_share: Reference to the original share model.
:param destination_share: Reference to the share model to be used by
migrated share.
:param share_server: Share server model or None.
:param dest_driver_migration_info: Migration information provided by
destination host.
:return: A dictionary with 'total_progress' field containing the
percentage value.
:param destination_share_server: Destination Share server model or
None.
"""
raise NotImplementedError()
def migration_get_driver_info(self, context, share, share_server):
"""Is called to provide necessary driver migration logic.
def migration_get_progress(
self, context, source_share, destination_share,
share_server=None, destination_share_server=None):
"""Obtains progress of migration of a given share to another host.
.. note::
Is called in source share's backend to obtain migration progress.
If possible, driver can implement a way to return migration progress
information.
:param context: The 'context.RequestContext' object for the request.
:param share: Reference to the share being migrated.
:param source_share: Reference to the original share model.
:param destination_share: Reference to the share model to be used by
migrated share.
:param share_server: Share server model or None.
:return: A dictionary with migration information.
:param destination_share_server: Destination Share server model or
None.
:return: A dictionary with at least 'total_progress' field containing
the percentage value.
"""
return None
raise NotImplementedError()
def migration_get_info(self, context, share, share_server):
def migration_get_info(self, context, share, share_server=None):
"""Is called to provide necessary generic migration logic.
:param context: The 'context.RequestContext' object for the request.
@ -411,7 +481,7 @@ class ShareDriver(object):
return {'mount': mount_template,
'unmount': unmount_template}
def _get_mount_command(self, context, share_instance, share_server):
def _get_mount_command(self, context, share_instance, share_server=None):
"""Is called to delegate mounting share logic."""
mount_template = self.configuration.safe_get('share_mount_template')
@ -424,7 +494,7 @@ class ShareDriver(object):
return mount_template % format_template
def _get_mount_export(self, share_instance, share_server):
def _get_mount_export(self, share_instance, share_server=None):
# NOTE(ganso): If drivers want to override the export_location IP,
# they can do so using this configuration. This method can also be
# overridden if necessary.
@ -434,7 +504,8 @@ class ShareDriver(object):
path = share_instance['export_locations'][0]['path']
return path
def _get_unmount_command(self, context, share_instance, share_server):
def _get_unmount_command(self, context, share_instance,
share_server=None):
return self.configuration.safe_get('share_unmount_template')
def create_share(self, context, share, share_server=None):

631
manila/share/manager.py

@ -22,6 +22,7 @@
import copy
import datetime
import functools
import time
from oslo_config import cfg
from oslo_log import log
@ -44,6 +45,7 @@ from manila.i18n import _LW
from manila import manager
from manila import quota
from manila.share import access
from manila.share import api
import manila.share.configuration
from manila.share import drivers_private_data
from manila.share import migration
@ -182,7 +184,7 @@ def add_hooks(f):
class ShareManager(manager.SchedulerDependentManager):
"""Manages NAS storages."""
RPC_API_VERSION = '1.11'
RPC_API_VERSION = '1.12'
def __init__(self, share_driver=None, service_name=None, *args, **kwargs):
"""Load the driver from args, or from flags."""
@ -284,6 +286,14 @@ class ShareManager(manager.SchedulerDependentManager):
LOG.debug("Re-exporting %s shares", len(share_instances))
for share_instance in share_instances:
share_ref = self.db.share_get(ctxt, share_instance['share_id'])
if (share_ref['task_state'] == (
constants.TASK_STATE_MIGRATION_DRIVER_IN_PROGRESS) and
share_instance['status'] == constants.STATUS_MIGRATING):
rpcapi = share_rpcapi.ShareAPI()
rpcapi.migration_driver_recovery(ctxt, share_ref, self.host)
continue
if share_ref.is_busy:
LOG.info(
_LI("Share instance %(id)s: skipping export, "
@ -343,7 +353,8 @@ class ShareManager(manager.SchedulerDependentManager):
def _provide_share_server_for_share(self, context, share_network_id,
share_instance, snapshot=None,
consistency_group=None):
consistency_group=None,
create_on_backend=True):
"""Gets or creates share_server and updates share with its id.
Active share_server can be deleted if there are no dependent shares
@ -362,6 +373,9 @@ class ShareManager(manager.SchedulerDependentManager):
share_network_id from provided snapshot.
:param share_instance: Share Instance model
:param snapshot: Optional -- Snapshot model
:param create_on_backend: Boolean. If True, driver will be asked to
create the share server if no share server
is available.
:returns: dict, dict -- first value is share_server, that
has been chosen for share schedule. Second value is
@ -461,20 +475,74 @@ class ShareManager(manager.SchedulerDependentManager):
{'share_server_id': compatible_share_server['id']},
with_share_data=True
)
if create_on_backend:
compatible_share_server = (
self._create_share_server_in_backend(
context, compatible_share_server))
if compatible_share_server['status'] == constants.STATUS_CREATING:
# Create share server on backend with data from db.
compatible_share_server = self._setup_server(
context, compatible_share_server)
LOG.info(_LI("Share server created successfully."))
else:
LOG.info(_LI("Used preexisting share server "
"'%(share_server_id)s'"),
{'share_server_id': compatible_share_server['id']})
return compatible_share_server, share_instance_ref
return _provide_share_server_for_share()
def _create_share_server_in_backend(self, context, share_server):
if share_server['status'] == constants.STATUS_CREATING:
# Create share server on backend with data from db.
share_server = self._setup_server(context, share_server)
LOG.info(_LI("Share server created successfully."))
else:
LOG.info(_LI("Using preexisting share server: "
"'%(share_server_id)s'"),
{'share_server_id': share_server['id']})
return share_server
def create_share_server(self, context, share_server_id):
"""Invoked to create a share server in this backend.
This method is invoked to create the share server defined in the model
obtained by the supplied id.
:param context: The 'context.RequestContext' object for the request.
:param share_server_id: The id of the server to be created.
"""
share_server = self.db.share_server_get(context, share_server_id)
self._create_share_server_in_backend(context, share_server)
def provide_share_server(self, context, share_instance_id,
share_network_id, snapshot_id=None):
"""Invoked to provide a compatible share server.
This method is invoked to find a compatible share server among the
existing ones or create a share server database instance with the share
server properties that will be used to create the share server later.
:param context: The 'context.RequestContext' object for the request.
:param share_instance_id: The id of the share instance whose model
attributes will be used to provide the share server.
:param share_network_id: The id of the share network the share server
to be provided has to be related to.
:param snapshot_id: The id of the snapshot to be used to obtain the
share server if applicable.
:return: The id of the share server that is being provided.
"""
share_instance = self.db.share_instance_get(context, share_instance_id,
with_share_data=True)
snapshot_ref = None
if snapshot_id:
snapshot_ref = self.db.share_snapshot_get(context, snapshot_id)
consistency_group_ref = None
if share_instance.get('consistency_group_id'):
consistency_group_ref = self.db.consistency_group_get(
context, share_instance['consistency_group_id'])
share_server, share_instance = self._provide_share_server_for_share(
context, share_network_id, share_instance, snapshot_ref,
consistency_group_ref, create_on_backend=False)
return share_server['id']
def _provide_share_server_for_cg(self, context, share_network_id,
cg_ref, cgsnapshot=None):
"""Gets or creates share_server and updates share with its id.
@ -592,21 +660,187 @@ class ShareManager(manager.SchedulerDependentManager):
return self.driver.migration_get_info(context, share_instance,
share_server)
def _migration_start_driver(self, context, share_ref, src_share_instance,
dest_host, notify, new_az_id):
share_server = self._get_share_server(context, src_share_instance)
share_api = api.API()
request_spec, dest_share_instance = (
share_api.create_share_instance_and_get_request_spec(
context, share_ref, new_az_id, None, dest_host,
src_share_instance['share_network_id']))
self.db.share_instance_update(
context, dest_share_instance['id'],
{'status': constants.STATUS_MIGRATING_TO})
# refresh and obtain proxified properties
dest_share_instance = self.db.share_instance_get(
context, dest_share_instance['id'], with_share_data=True)
helper = migration.ShareMigrationHelper(context, self.db, share_ref)
try:
if dest_share_instance['share_network_id']:
rpcapi = share_rpcapi.ShareAPI()
# NOTE(ganso): Obtaining the share_server_id asynchronously so
# we can wait for it to be ready.
dest_share_server_id = rpcapi.provide_share_server(
context, dest_share_instance,
dest_share_instance['share_network_id'])
rpcapi.create_share_server(
context, dest_share_instance, dest_share_server_id)
dest_share_server = helper.wait_for_share_server(
dest_share_server_id)
else:
dest_share_server = None
compatibility = self.driver.migration_check_compatibility(
context, src_share_instance, dest_share_instance,
share_server, dest_share_server)
if not compatibility.get('compatible'):
msg = _("Destination host %(host)s is not compatible with "
"share %(share)s's source backend for driver-assisted "
"migration.") % {
'host': dest_host,
'share': share_ref['id'],
}
raise exception.ShareMigrationFailed(reason=msg)
if not compatibility.get('writable'):
readonly_support = self.driver.configuration.safe_get(
'migration_readonly_rules_support')
helper.change_to_read_only(src_share_instance, share_server,
readonly_support, self.driver)
LOG.debug("Initiating driver migration for share %s.",
share_ref['id'])
self.db.share_update(
context, share_ref['id'],
{'task_state': (
constants.TASK_STATE_MIGRATION_DRIVER_IN_PROGRESS)})
self.driver.migration_start(
context, src_share_instance, dest_share_instance,
share_server, dest_share_server)
# prevent invoking _migration_driver_continue immediately
time.sleep(5)
self._migration_driver_continue(
context, share_ref, src_share_instance, dest_share_instance,
share_server, dest_share_server, notify)
except Exception:
# NOTE(ganso): Cleaning up error'ed destination share instance from
# database. It is assumed that driver cleans up leftovers in
# backend when migration fails.
self._migration_delete_instance(context, dest_share_instance['id'])
# NOTE(ganso): For now source share instance should remain in
# migrating status for fallback migration.
msg = _("Driver optimized migration of share %s "
"failed.") % share_ref['id']
LOG.exception(msg)
raise exception.ShareMigrationFailed(reason=msg)
return True
def _migration_driver_continue(
self, context, share_ref, src_share_instance, dest_share_instance,
src_share_server, dest_share_server, notify=False):
finished = False
share_ref = self.db.share_get(context, share_ref['id'])
while (not finished and share_ref['task_state'] ==
constants.TASK_STATE_MIGRATION_DRIVER_IN_PROGRESS):
finished = self.driver.migration_continue(
context, src_share_instance, dest_share_instance,
src_share_server, dest_share_server)
time.sleep(5)
share_ref = self.db.share_get(context, share_ref['id'])
if finished:
self.db.share_update(
context, share_ref['id'],
{'task_state':
constants.TASK_STATE_MIGRATION_DRIVER_PHASE1_DONE})
if notify:
self._migration_complete_driver(
context, share_ref, src_share_instance,
dest_share_instance)
LOG.info(_LI("Share Migration for share %s"
" completed successfully."), share_ref['id'])
else:
LOG.info(_LI("Share Migration for share %s completed "
"first phase successfully."), share_ref['id'])
else:
if (share_ref['task_state'] ==
constants.TASK_STATE_MIGRATION_CANCELLED):
LOG.warning(_LW("Share Migration for share %s was cancelled."),
share_ref['id'])
else:
msg = (_("Share Migration for share %s did not complete "
"first phase successfully."), share_ref['id'])
raise exception.ShareMigrationFailed(reason=msg)
@utils.require_driver_initialized
def migration_get_driver_info(self, context, share_instance_id):
share_instance = self.db.share_instance_get(
context, share_instance_id, with_share_data=True)
def migration_driver_recovery(self, context, share_id):
"""Resumes a migration after a service restart."""
share_server = None
if share_instance.get('share_server_id'):
share_server = self.db.share_server_get(
context, share_instance['share_server_id'])
share = self.db.share_get(context, share_id)
share_api = api.API()
src_share_instance_id, dest_share_instance_id = (
share_api.get_migrating_instances(share))
src_share_instance = self.db.share_instance_get(
context, src_share_instance_id, with_share_data=True)
dest_share_instance = self.db.share_instance_get(
context, dest_share_instance_id, with_share_data=True)
src_share_server = self._get_share_server(context, src_share_instance)
return self.driver.migration_get_driver_info(context, share_instance,
share_server)
dest_share_server = self._get_share_server(
context, dest_share_instance)
try:
self._migration_driver_continue(
context, share, src_share_instance, dest_share_instance,
src_share_server, dest_share_server)
except Exception:
# NOTE(ganso): Cleaning up error'ed destination share instance from
# database. It is assumed that driver cleans up leftovers in
# backend when migration fails.
self._migration_delete_instance(context, dest_share_instance['id'])
self.db.share_instance_update(
context, src_share_instance['id'],
{'status': constants.STATUS_AVAILABLE})
self.db.share_update(
context, share['id'],
{'task_state': constants.TASK_STATE_MIGRATION_ERROR})
msg = _("Driver optimized migration of share %s "
"failed.") % share['id']
LOG.exception(msg)
raise exception.ShareMigrationFailed(reason=msg)
@utils.require_driver_initialized
def migration_start(self, context, share_id, host, force_host_copy,
def migration_start(self, context, share_id, dest_host, force_host_copy,
notify=True):
"""Migrates a share from current host to another host."""
LOG.debug("Entered migration_start method for share %s.", share_id)
@ -615,10 +849,14 @@ class ShareManager(manager.SchedulerDependentManager):
context, share_id,
{'task_state': constants.TASK_STATE_MIGRATION_IN_PROGRESS})
rpcapi = share_rpcapi.ShareAPI()
share_ref = self.db.share_get(context, share_id)
share_instance = self._get_share_instance(context, share_ref)
moved = False
success = False
host_value = share_utils.extract_host(dest_host)
service = self.db.service_get_by_args(
context, host_value, 'manila-share')
new_az_id = service['availability_zone_id']
self.db.share_instance_update(context, share_instance['id'],
{'status': constants.STATUS_MIGRATING})
@ -626,49 +864,27 @@ class ShareManager(manager.SchedulerDependentManager):
if not force_host_copy:
try:
dest_driver_migration_info = rpcapi.migration_get_driver_info(
context, share_instance)
share_server = self._get_share_server(context.elevated(),
share_instance)
LOG.debug("Calling driver migration for share %s.", share_id)
self.db.share_update(
context, share_id,
{'task_state': (
constants.TASK_STATE_MIGRATION_DRIVER_IN_PROGRESS)})
moved, model_update = self.driver.migration_start(
context, share_instance, share_server, host,
dest_driver_migration_info, notify)
# NOTE(ganso): Here we are allowing the driver to perform
# changes even if it has not performed migration. While this
# scenario may not be valid, I do not think it should be
# forcefully prevented.
if model_update:
self.db.share_instance_update(
context, share_instance['id'], model_update)
success = self._migration_start_driver(
context, share_ref, share_instance, dest_host, notify,
new_az_id)
except Exception as e:
msg = six.text_type(e)
LOG.exception(msg)
LOG.warning(_LW("Driver did not migrate share %s. Proceeding "
"with generic migration approach.") % share_id)
if not isinstance(e, NotImplementedError):
LOG.exception(
_LE("The driver could not migrate the share %(shr)s"),
{'shr': share_id})
if not moved:
LOG.debug("Starting generic migration "
"for share %s.", share_id)
if not success:
LOG.info(_LI("Starting generic migration for share %s."), share_id)
self.db.share_update(
context, share_id,
{'task_state': constants.TASK_STATE_MIGRATION_IN_PROGRESS})
try:
self._migration_start_generic(context, share_ref,
share_instance, host, notify)
self._migration_start_generic(
context, share_ref, share_instance, dest_host, notify,
new_az_id)
except Exception:
msg = _("Generic migration failed for share %s.") % share_id
LOG.exception(msg)
@ -679,52 +895,36 @@ class ShareManager(manager.SchedulerDependentManager):
context, share_instance['id'],
{'status': constants.STATUS_AVAILABLE})
raise exception.ShareMigrationFailed(reason=msg)
elif not notify:
self.db.share_update(
context, share_ref['id'],
{'task_state':
constants.TASK_STATE_MIGRATION_DRIVER_PHASE1_DONE})
else:
self.db.share_instance_update(
context, share_instance['id'],
{'status': constants.STATUS_AVAILABLE,
'host': host['host']})
self.db.share_update(
context, share_ref['id'],
{'task_state': constants.TASK_STATE_MIGRATION_SUCCESS})
LOG.info(_LI("Share Migration for share %s"
" completed successfully."), share_ref['id'])
def _migration_start_generic(self, context, share, share_instance, host,
notify):
def _migration_start_generic(self, context, share, src_share_instance,
dest_host, notify, new_az_id):
rpcapi = share_rpcapi.ShareAPI()
helper = migration.ShareMigrationHelper(context, self.db, share)
share_server = self._get_share_server(context.elevated(),
share_instance)
src_share_instance)
readonly_support = self.driver.configuration.safe_get(
'migration_readonly_rules_support')
helper.change_to_read_only(share_instance, share_server,
helper.change_to_read_only(src_share_instance, share_server,
readonly_support, self.driver)
try:
new_share_instance = helper.create_instance_and_wait(
share, share_instance, host)
dest_share_instance = helper.create_instance_and_wait(
share, src_share_instance, dest_host, new_az_id)
self.db.share_instance_update(
context, new_share_instance['id'],
context, dest_share_instance['id'],
{'status': constants.STATUS_MIGRATING_TO})
except Exception:
msg = _("Failed to create instance on destination "
"backend during migration of share %s.") % share['id']
LOG.exception(msg)
helper.cleanup_access_rules(share_instance, share_server,
helper.cleanup_access_rules(src_share_instance, share_server,
self.driver)
raise exception.ShareMigrationFailed(reason=msg)
@ -735,17 +935,17 @@ class ShareManager(manager.SchedulerDependentManager):
try:
src_migration_info = self.driver.migration_get_info(
context, share_instance, share_server)
context, src_share_instance, share_server)
dest_migration_info = rpcapi.migration_get_info(
context, new_share_instance)
context, dest_share_instance)
LOG.debug("Time to start copying in migration"
" for share %s.", share['id'])
data_rpc.migration_start(
context, share['id'], ignore_list, share_instance['id'],
new_share_instance['id'], src_migration_info,
context, share['id'], ignore_list, src_share_instance['id'],
dest_share_instance['id'], src_migration_info,
dest_migration_info, notify)
except Exception:
@ -753,77 +953,128 @@ class ShareManager(manager.SchedulerDependentManager):
" invoking Data Service for migration of "
"share %s.") % share['id']
LOG.exception(msg)
helper.cleanup_new_instance(new_share_instance)
helper.cleanup_access_rules(share_instance, share_server,
helper.cleanup_new_instance(dest_share_instance)
helper.cleanup_access_rules(src_share_instance, share_server,
self.driver)
raise exception.ShareMigrationFailed(reason=msg)
@utils.require_driver_initialized
def migration_complete(self, context, share_id, share_instance_id,
new_share_instance_id):
def _migration_complete_driver(
self, context, share_ref, src_share_instance, dest_share_instance):
LOG.info(_LI("Received request to finish Share Migration for "
"share %s."), share_id)
share_server = self._get_share_server(context, src_share_instance)
dest_share_server = self._get_share_server(
context, dest_share_instance)
share_ref = self.db.share_get(context, share_id)
export_locations = self.driver.migration_complete(
context, src_share_instance, dest_share_instance,
share_server, dest_share_server)
if share_ref['task_state'] == (
constants.TASK_STATE_MIGRATION_DRIVER_PHASE1_DONE):
if export_locations:
self.db.share_export_locations_update(
context, dest_share_instance['id'], export_locations)
rpcapi = share_rpcapi.ShareAPI()
helper = migration.ShareMigrationHelper(context, self.db, share_ref)
share_instance = self._get_share_instance(context, share_ref)
helper.apply_new_access_rules(dest_share_instance)
share_server = self._get_share_server(context, share_instance)
self.db.share_instance_update(
context, dest_share_instance['id'],
{'status': constants.STATUS_AVAILABLE})
self._migration_delete_instance(context, src_share_instance['id'])
self.db.share_update(
context, dest_share_instance['share_id'],
{'task_state': constants.TASK_STATE_MIGRATION_SUCCESS})
def _migration_delete_instance(self, context, instance_id):
share_instance = self.db.share_instance_get(
context, instance_id, with_share_data=True)
self.db.share_instance_update(
context, instance_id, {'status': constants.STATUS_INACTIVE})
rules = self.db.share_access_get_all_for_instance(
context, instance_id)
for rule in rules:
access_mapping = self.db.share_instance_access_get(
context, rule['id'], instance_id)
self.db.share_instance_access_delete(
context, access_mapping['id'])
self.db.share_instance_delete(context, instance_id)
LOG.info(_LI("Share instance %s: deleted successfully."),
instance_id)
self._check_delete_share_server(context, share_instance)
@utils.require_driver_initialized
def migration_complete(self, context, src_instance_id, dest_instance_id):
src_share_instance = self.db.share_instance_get(
context, src_instance_id, with_share_data=True)
dest_share_instance = self.db.share_instance_get(
context, dest_instance_id, with_share_data=True)
share_ref = self.db.share_get(context, src_share_instance['share_id'])
LOG.info(_LI("Received request to finish Share Migration for "
"share %s."), share_ref['id'])
if share_ref['task_state'] == (
constants.TASK_STATE_MIGRATION_DRIVER_PHASE1_DONE):
try:
dest_driver_migration_info = rpcapi.migration_get_driver_info(
context, share_instance)
self._migration_complete_driver(
context, share_ref, src_share_instance,
dest_share_instance)
model_update = self.driver.migration_complete(
context, share_instance, share_server,
dest_driver_migration_info)
if model_update:
self.db.share_instance_update(
context, share_instance['id'], model_update)
self.db.share_update(
context, share_id,
{'task_state': constants.TASK_STATE_MIGRATION_SUCCESS})
except Exception:
msg = _("Driver migration completion failed for"
" share %s.") % share_id
" share %s.") % share_ref['id']
LOG.exception(msg)
self.db.share_instance_update(
context, src_instance_id,
{'status': constants.STATUS_AVAILABLE})
self.db.share_instance_update(
context, dest_instance_id,
{'status': constants.STATUS_ERROR})
self.db.share_update(
context, share_id,
context, share_ref['id'],
{'task_state': constants.TASK_STATE_MIGRATION_ERROR})
raise exception.ShareMigrationFailed(reason=msg)
else:
try:
self._migration_complete(
context, share_ref, share_instance_id,
new_share_instance_id)
self._migration_complete_generic(
context, share_ref, src_instance_id,
dest_instance_id)
except Exception:
msg = _("Generic migration completion failed for"
" share %s.") % share_id
" share %s.") % share_ref['id']
LOG.exception(msg)
self.db.share_update(
context, share_id,
context, share_ref['id'],
{'task_state': constants.TASK_STATE_MIGRATION_ERROR})
self.db.share_instance_update(
context, share_instance_id,
context, src_instance_id,
{'status': constants.STATUS_AVAILABLE})
raise exception.ShareMigrationFailed(reason=msg)
def _migration_complete(self, context, share_ref, share_instance_id,
new_share_instance_id):
LOG.info(_LI("Share Migration for share %s"
" completed successfully."), share_ref['id'])
share_instance = self.db.share_instance_get(
context, share_instance_id, with_share_data=True)
new_share_instance = self.db.share_instance_get(
context, new_share_instance_id, with_share_data=True)
def _migration_complete_generic(self, context, share_ref,
src_instance_id, dest_instance_id):
share_server = self._get_share_server(context, share_instance)
src_share_instance = self.db.share_instance_get(
context, src_instance_id, with_share_data=True)
dest_share_instance = self.db.share_instance_get(
context, dest_instance_id, with_share_data=True)
share_server = self._get_share_server(context, src_share_instance)
helper = migration.ShareMigrationHelper(context, self.db, share_ref)
@ -833,13 +1084,13 @@ class ShareManager(manager.SchedulerDependentManager):
msg = _("Data copy of generic migration for share %s has not "
"completed successfully.") % share_ref['id']
LOG.warning(msg)
helper.cleanup_new_instance(new_share_instance)
helper.cleanup_new_instance(dest_share_instance)
helper.cleanup_access_rules(share_instance, share_server,
helper.cleanup_access_rules(src_share_instance, share_server,
self.driver)
if task_state == constants.TASK_STATE_DATA_COPYING_CANCELLED:
self.db.share_instance_update(
context, share_instance_id,
context, src_instance_id,
{'status': constants.STATUS_AVAILABLE})
self.db.share_update(
context, share_ref['id'],
@ -858,13 +1109,13 @@ class ShareManager(manager.SchedulerDependentManager):
raise exception.ShareMigrationFailed(reason=msg)
try:
helper.apply_new_access_rules(new_share_instance)
helper.apply_new_access_rules(dest_share_instance)
except Exception:
msg = _("Failed to apply new access rules during migration "
"of share %s.") % share_ref['id']
LOG.exception(msg)
helper.cleanup_new_instance(new_share_instance)
helper.cleanup_access_rules(share_instance, share_server,
helper.cleanup_new_instance(dest_share_instance)
helper.cleanup_access_rules(src_share_instance, share_server,
self.driver)
raise exception.ShareMigrationFailed(reason=msg)
@ -872,75 +1123,107 @@ class ShareManager(manager.SchedulerDependentManager):