Browse Source

Create share from snapshot in another pool or backend

This patch enables the creation of a share from snapshot
specifying another pool or backend. In the scheduler, a
new filter and weigher were implemented in order to consider
this operation if the backend supports it. Also, a new
field called 'progress' was added in the share and share
instance. The 'progress' field indicates the status
of the operation create share from snapshot (in percentage).
Finally, a new periodic task was added in order to constantly
check the share status.

Partially-implements: bp create-share-from-snapshot-in-another-pool-or-backend

DOCImpact
Change-Id: Iab13a0961eb4a387a502246e5d4b79bc9046e04b
Co-authored-by: carloss <ces.eduardo98@gmail.com>
Co-authored-by: dviroel <viroel@gmail.com>
changes/97/709697/12
Douglas Viroel 2 years ago
parent
commit
6c47b193b0
  1. 20
      api-ref/source/parameters.yaml
  2. 1
      api-ref/source/samples/share-create-response.json
  3. 15
      api-ref/source/samples/share-instances-list-response.json
  4. 1
      api-ref/source/samples/share-show-instance-response.json
  5. 1
      api-ref/source/samples/share-show-response.json
  6. 2
      api-ref/source/samples/shares-list-detailed-response.json
  7. 2
      api-ref/source/share-instances.inc
  8. 5
      api-ref/source/shares.inc
  9. 5
      manila/api/openstack/api_version_request.py
  10. 5
      manila/api/openstack/rest_api_version_history.rst
  11. 15
      manila/api/views/share_instance.py
  12. 13
      manila/api/views/shares.py
  13. 1
      manila/common/constants.py
  14. 5
      manila/db/api.py
  15. 62
      manila/db/migrations/alembic/versions/e6d88547b381_add_progress_field_to_share_instance.py
  16. 8
      manila/db/sqlalchemy/api.py
  17. 6
      manila/db/sqlalchemy/models.py
  18. 11
      manila/message/message_field.py
  19. 26
      manila/scheduler/drivers/filter.py
  20. 71
      manila/scheduler/filters/create_from_snapshot.py
  21. 2
      manila/scheduler/host_manager.py
  22. 65
      manila/scheduler/weighers/host_affinity.py
  23. 46
      manila/share/api.py
  24. 125
      manila/share/driver.py
  25. 2
      manila/share/drivers/dell_emc/driver.py
  26. 2
      manila/share/drivers/dell_emc/plugins/powermax/connection.py
  27. 2
      manila/share/drivers/dell_emc/plugins/unity/connection.py
  28. 2
      manila/share/drivers/dell_emc/plugins/vnx/connection.py
  29. 2
      manila/share/drivers/generic.py
  30. 2
      manila/share/drivers/glusterfs/layout.py
  31. 2
      manila/share/drivers/glusterfs/layout_directory.py
  32. 2
      manila/share/drivers/glusterfs/layout_volume.py
  33. 2
      manila/share/drivers/hdfs/hdfs_native.py
  34. 2
      manila/share/drivers/hitachi/hnas/driver.py
  35. 2
      manila/share/drivers/hpe/hpe_3par_driver.py
  36. 2
      manila/share/drivers/huawei/base.py
  37. 2
      manila/share/drivers/huawei/huawei_nas.py
  38. 2
      manila/share/drivers/huawei/v3/connection.py
  39. 2
      manila/share/drivers/ibm/gpfs.py
  40. 2
      manila/share/drivers/infinidat/infinibox.py
  41. 2
      manila/share/drivers/inspur/as13000/as13000_nas.py
  42. 2
      manila/share/drivers/lvm.py
  43. 2
      manila/share/drivers/maprfs/maprfs_native.py
  44. 2
      manila/share/drivers/netapp/dataontap/cluster_mode/lib_base.py
  45. 2
      manila/share/drivers/nexenta/ns4/nexenta_nas.py
  46. 2
      manila/share/drivers/nexenta/ns5/nexenta_nas.py
  47. 2
      manila/share/drivers/qnap/qnap.py
  48. 2
      manila/share/drivers/tegile/tegile.py
  49. 2
      manila/share/drivers/veritas/veritas_isa.py
  50. 2
      manila/share/drivers/zfsonlinux/driver.py
  51. 2
      manila/share/drivers/zfssa/zfssashare.py
  52. 289
      manila/share/manager.py
  53. 11
      manila/tests/api/v2/test_share_instances.py
  54. 27
      manila/tests/api/views/test_shares.py
  55. 30
      manila/tests/db/migrations/alembic/migrations_data_checks.py
  56. 12
      manila/tests/db/sqlalchemy/test_api.py
  57. 14
      manila/tests/fake_driver.py
  58. 92
      manila/tests/scheduler/filters/test_create_from_snapshot.py
  59. 138
      manila/tests/scheduler/weighers/test_host_affinity.py
  60. 15
      manila/tests/share/drivers/dummy.py
  61. 2
      manila/tests/share/drivers/glusterfs/test_layout.py
  62. 53
      manila/tests/share/test_api.py
  63. 98
      manila/tests/share/test_driver.py
  64. 339
      manila/tests/share/test_manager.py
  65. 14
      releasenotes/notes/add-create-share-from-snapshot-another-pool-or-backend-98d61fe753b85632.yaml
  66. 2
      setup.cfg

20
api-ref/source/parameters.yaml

@ -468,8 +468,8 @@ source_share_group_snapshot_id_query:
status_query:
description: |
Filters by a share status. A valid value is
``creating``, ``error``, ``available``, ``deleting``,
``error_deleting``, ``manage_starting``, ``manage_error``,
``creating``, ``creating_from_snapshot``, ``error``, ``available``,
``deleting``, ``error_deleting``, ``manage_starting``, ``manage_error``,
``unmanage_starting``, ``unmanage_error``, ``migrating``,
``extending``, ``extending_error``, ``shrinking``,
``shrinking_error``, or ``shrinking_possible_data_loss_error``.
@ -1540,6 +1540,13 @@ progress:
in: body
required: true
type: string
progress_share_instance:
description: |
The progress of the share creation.
in: body
min_version: 2.54
required: true
type: string
project:
description: |
The UUID of the project to which access to the
@ -2713,7 +2720,7 @@ status_1:
type: string
status_16:
description: |
The share status, which is ``creating``,
The share status, which is ``creating``, ``creating_from_snapshot``,
``error``, ``available``, ``deleting``, ``error_deleting``,
``manage_starting``, ``manage_error``, ``unmanage_starting``,
``unmanage_error``, ``unmanaged``, ``extend``,
@ -2737,15 +2744,16 @@ status_3:
``extending_error``. Extend share failed. - ``shrinking``. Share
is being shrunk. - ``shrinking_error``. Failed to update quota on
share shrinking. - ``shrinking_possible_data_loss_error``. Shrink
share failed due to possible data loss.
share failed due to possible data loss. - ``creating_from_snapshot``.
The share is being created from a parent snapshot.
in: body
required: true
type: string
status_5:
description: |
The share instance status. A valid value is
``available``, ``error``, ``creating``, ``deleting``, and
``error_deleting``.
``available``, ``error``, ``creating``, ``deleting``,
``creating_from_snapshot``, or ``error_deleting``.
in: body
required: true
type: string

1
api-ref/source/samples/share-create-response.json

@ -1,6 +1,7 @@
{
"share": {
"status": null,
"progress": null,
"share_server_id": null,
"project_id": "16e1ab15c35a457e9c2b2aa189f544e1",
"name": "share_London",

15
api-ref/source/samples/share-instances-list-response.json

@ -2,6 +2,7 @@
"share_instances": [
{
"status": "error",
"progress": null,
"share_id": "406ea93b-32e9-4907-a117-148b3945749f",
"availability_zone": "nova",
"replica_state": null,
@ -14,6 +15,7 @@
},
{
"status": "available",
"progress": "100%",
"share_id": "d94a8548-2079-4be0-b21c-0a887acd31ca",
"availability_zone": "nova",
"replica_state": null,
@ -23,6 +25,19 @@
"share_server_id": "ba11930a-bf1a-4aa7-bae4-a8dfbaa3cc73",
"host": "manila2@generic1#GENERIC1",
"id": "75559a8b-c90c-42a7-bda2-edbe86acfb7b"
},
{
"status": "creating_from_snapshot",
"progress": "30%",
"share_id": "9bb15af4-27e5-4174-ae15-dc549d4a3b51",
"availability_zone": "nova",
"replica_state": null,
"created_at": "2015-09-07T09:01:15.000000",
"share_network_id": "713df749-aac0-4a54-af52-10f6c991e80c",
"cast_rules_to_readonly": false,
"share_server_id": "ba11930a-bf1a-4aa7-bae4-a8dfbaa3cc73",
"host": "manila2@generic1#GENERIC1",
"id": "48155648-2fd3-480d-b02b-44b995c24bab"
}
]
}

1
api-ref/source/samples/share-show-instance-response.json

@ -1,6 +1,7 @@
{
"share_instance": {
"status": "available",
"progress": "100%",
"share_id": "d94a8548-2079-4be0-b21c-0a887acd31ca",
"availability_zone": "nova",
"replica_state": null,

1
api-ref/source/samples/share-show-response.json

@ -27,6 +27,7 @@
"aim": "doc"
},
"status": "available",
"progress": "100%",
"description": "My custom share London",
"host": "manila2@generic1#GENERIC1",
"access_rules_status": "active",

2
api-ref/source/samples/shares-list-detailed-response.json

@ -25,6 +25,7 @@
"project_id": "16e1ab15c35a457e9c2b2aa189f544e1",
"metadata": {},
"status": "error",
"progress": null,
"access_rules_status": "active",
"description": "There is a share description.",
"host": "manila2@generic1#GENERIC1",
@ -64,6 +65,7 @@
"project_id": "16e1ab15c35a457e9c2b2aa189f544e1",
"metadata": {},
"status": "available",
"progress": "100%",
"access_rules_status": "active",
"description": "Changed description.",
"host": "manila2@generic1#GENERIC1",

2
api-ref/source/share-instances.inc

@ -48,6 +48,7 @@ Response parameters
- status: status_5
- access_rules_status: access_rules_status
- share_id: share_id_2
- progress: progress_share_instance
- availability_zone: availability_zone_1
- created_at: created_at
- replica_state: replica_state
@ -105,6 +106,7 @@ Response parameters
- status: status_5
- access_rules_status: access_rules_status
- share_id: share_id_2
- progress: progress_share_instance
- availability_zone: availability_zone_1
- created_at: created_at
- replica_state: replica_state

5
api-ref/source/shares.inc

@ -39,6 +39,8 @@ A share has one of these status values:
+----------------------------------------+--------------------------------------------------------+
| ``creating`` | The share is being created. |
+----------------------------------------+--------------------------------------------------------+
| ``creating_from_snapshot`` | The share is being created from a parent snapshot. |
+----------------------------------------+--------------------------------------------------------+
| ``deleting`` | The share is being deleted. |
+----------------------------------------+--------------------------------------------------------+
| ``deleted`` | The share was deleted. |
@ -220,6 +222,7 @@ Response parameters
- project_id: project_id
- metadata: metadata
- status: status_16
- progress: progress_share_instance
- description: description
- host: host_1
- access_rules_status: access_rules_status
@ -291,6 +294,7 @@ Response parameters
- project_id: project_id
- metadata: metadata
- status: status_16
- progress: progress_share_instance
- description: description
- host: host_9
- access_rules_status: access_rules_status
@ -369,6 +373,7 @@ Response parameters
- id: id_4
- status: status_3
- progress: progress_share_instance
- links: links
- project_id: project_id
- share_proto: share_proto

5
manila/api/openstack/api_version_request.py

@ -144,13 +144,16 @@ REST_API_VERSION_HISTORY = """
filters, support querying user messages within the specified time
period.
* 2.53 - Added quota control to share replicas.
* 2.54 - Share and share instance objects include a new field called
"progress" which indicates the completion of a share creation
operation as a percentage.
"""
# The minimum and maximum versions of the API supported
# The default api version request is defined to be the
# minimum version of the API supported.
_MIN_API_VERSION = "2.0"
_MAX_API_VERSION = "2.53"
_MAX_API_VERSION = "2.54"
DEFAULT_API_VERSION = _MIN_API_VERSION

5
manila/api/openstack/rest_api_version_history.rst

@ -296,3 +296,8 @@ user documentation.
2.53
----
Added quota control for share replicas and replica gigabytes.
2.54
----
Share and share instance objects include a new field called "progress" which
indicates the completion of a share creation operation as a percentage.

15
manila/api/views/share_instance.py

@ -11,6 +11,7 @@
# under the License.
from manila.api import common
from manila.common import constants
class ViewBuilder(common.ViewBuilder):
@ -25,6 +26,8 @@ class ViewBuilder(common.ViewBuilder):
"add_replication_fields",
"add_share_type_field",
"add_cast_rules_to_readonly_field",
"add_progress_field",
"translate_creating_from_snapshot_status",
]
def detail_list(self, request, instances):
@ -47,6 +50,7 @@ class ViewBuilder(common.ViewBuilder):
'export_location': share_instance.get('export_location'),
'export_locations': export_locations,
}
self.update_versioned_resource_dict(
request, instance_dict, share_instance)
return {'share_instance': instance_dict}
@ -91,3 +95,14 @@ class ViewBuilder(common.ViewBuilder):
share_instance):
instance_dict['cast_rules_to_readonly'] = share_instance.get(
'cast_rules_to_readonly', False)
@common.ViewBuilder.versioned_method("1.0", "2.53")
def translate_creating_from_snapshot_status(self, context, instance_dict,
share_instance):
if (share_instance.get('status') ==
constants.STATUS_CREATING_FROM_SNAPSHOT):
instance_dict['status'] = constants.STATUS_CREATING
@common.ViewBuilder.versioned_method("2.54")
def add_progress_field(self, context, instance_dict, share_instance):
instance_dict['progress'] = share_instance.get('progress')

13
manila/api/views/shares.py

@ -34,6 +34,8 @@ class ViewBuilder(common.ViewBuilder):
"translate_access_rules_status",
"add_share_group_fields",
"add_mount_snapshot_support_field",
"add_progress_field",
"translate_creating_from_snapshot_status",
]
def summary_list(self, request, shares, count=None):
@ -92,6 +94,7 @@ class ViewBuilder(common.ViewBuilder):
'is_public': share.get('is_public'),
'export_locations': export_locations,
}
self.update_versioned_resource_dict(request, share_dict, share)
if context.is_admin:
@ -184,3 +187,13 @@ class ViewBuilder(common.ViewBuilder):
shares_dict['shares_links'] = shares_links
return shares_dict
@common.ViewBuilder.versioned_method("1.0", "2.53")
def translate_creating_from_snapshot_status(self, context, share_dict,
share):
if share.get('status') == constants.STATUS_CREATING_FROM_SNAPSHOT:
share_dict['status'] = constants.STATUS_CREATING
@common.ViewBuilder.versioned_method("2.54")
def add_progress_field(self, context, share_dict, share):
share_dict['progress'] = share.get('progress')

1
manila/common/constants.py

@ -18,6 +18,7 @@ DB_MAX_INT = 0x7FFFFFFF
# SHARE AND GENERAL STATUSES
STATUS_CREATING = 'creating'
STATUS_CREATING_FROM_SNAPSHOT = 'creating_from_snapshot'
STATUS_DELETING = 'deleting'
STATUS_DELETED = 'deleted'
STATUS_ERROR = 'error'

5
manila/db/api.py

@ -340,10 +340,11 @@ def share_instances_get_all_by_share_server(context, share_server_id):
share_server_id)
def share_instances_get_all_by_host(context, host, with_share_data=False):
def share_instances_get_all_by_host(context, host, with_share_data=False,
status=None):
"""Returns all share instances with given host."""
return IMPL.share_instances_get_all_by_host(
context, host, with_share_data=with_share_data)
context, host, with_share_data=with_share_data, status=status)
def share_instances_get_all_by_share_network(context, share_network_id):

62
manila/db/migrations/alembic/versions/e6d88547b381_add_progress_field_to_share_instance.py

@ -0,0 +1,62 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""add-progress-field-to-share-instance
Revision ID: e6d88547b381
Revises: 805685098bd2
Create Date: 2020-01-31 14:06:15.952747
"""
# revision identifiers, used by Alembic.
revision = 'e6d88547b381'
down_revision = '805685098bd2'
from alembic import op
from manila.common import constants
from manila.db.migrations import utils
from oslo_log import log
import sqlalchemy as sa
LOG = log.getLogger(__name__)
def upgrade():
try:
connection = op.get_bind()
op.add_column('share_instances',
sa.Column('progress', sa.String(32), nullable=True,
default=None))
share_instances_table = utils.load_table('share_instances', connection)
updated_data = {'progress': '100%'}
# pylint: disable=no-value-for-parameter
op.execute(
share_instances_table.update().where(
share_instances_table.c.status == constants.STATUS_AVAILABLE,
).values(updated_data)
)
except Exception:
LOG.error("Column share_instances.progress not created.")
raise
def downgrade():
try:
op.drop_column('share_instances', 'progress')
except Exception:
LOG.error("Column share_instances.progress not dropped.")
raise

8
manila/db/sqlalchemy/api.py

@ -1579,7 +1579,7 @@ def _set_instances_share_data(context, instances, session):
@require_admin_context
def share_instances_get_all_by_host(context, host, with_share_data=False,
session=None):
status=None, session=None):
"""Retrieves all share instances hosted on a host."""
session = session or get_session()
instances = (
@ -1588,8 +1588,12 @@ def share_instances_get_all_by_host(context, host, with_share_data=False,
models.ShareInstance.host == host,
models.ShareInstance.host.like("{0}#%".format(host))
)
).all()
)
)
if status is not None:
instances = instances.filter(models.ShareInstance.status == status)
# Returns list of all instances that satisfy filters.
instances = instances.all()
if with_share_data:
instances = _set_instances_share_data(context, instances, session)

6
manila/db/sqlalchemy/models.py

@ -257,6 +257,11 @@ class Share(BASE, ManilaBase):
return len(replicas) > 1
return False
@property
def progress(self):
if len(self.instances) > 0:
return self.instance.progress
@property
def instance(self):
# NOTE(gouthamr): The order of preference: status 'replication_change',
@ -364,6 +369,7 @@ class ShareInstance(BASE, ManilaBase):
deleted = Column(String(36), default='False')
host = Column(String(255))
status = Column(String(255))
progress = Column(String(32))
ACCESS_STATUS_PRIORITIES = {
constants.STATUS_ACTIVE: 0,

11
manila/message/message_field.py

@ -80,6 +80,12 @@ class Detail(object):
"set to extending_error. This action cannot be re-attempted until "
"the status has been rectified. Contact your administrator to "
"determine the cause of this failure."))
FILTER_CREATE_FROM_SNAPSHOT = ('016', FILTER_MSG % 'CreateFromSnapshot')
DRIVER_FAILED_CREATING_FROM_SNAP = (
'017',
_("Share Driver has failed to create the share from snapshot. This "
"operation can be re-attempted by creating a new share. Contact "
"your administrator to determine the cause of this failure."))
ALL = (UNKNOWN_ERROR,
NO_VALID_HOST,
@ -95,7 +101,9 @@ class Detail(object):
FILTER_JSON,
FILTER_RETRY,
FILTER_REPLICATION,
DRIVER_FAILED_EXTEND)
DRIVER_FAILED_EXTEND,
FILTER_CREATE_FROM_SNAPSHOT,
DRIVER_FAILED_CREATING_FROM_SNAP)
# Exception and detail mappings
EXCEPTION_DETAIL_MAPPINGS = {
@ -113,6 +121,7 @@ class Detail(object):
'JsonFilter': FILTER_JSON,
'RetryFilter': FILTER_RETRY,
'ShareReplicationFilter': FILTER_REPLICATION,
'CreateFromSnapshotFilter': FILTER_CREATE_FROM_SNAPSHOT,
}

26
manila/scheduler/drivers/filter.py

@ -162,21 +162,27 @@ class FilterScheduler(base.Scheduler):
share_group = request_spec.get('share_group')
# NOTE(gouthamr): If 'active_replica_host' is present in the request
# spec, pass that host's 'replication_domain' to the
# ShareReplication filter.
# NOTE(gouthamr): If 'active_replica_host' or 'snapshot_host' is
# present in the request spec, pass that host's 'replication_domain' to
# the ShareReplication and CreateFromSnapshot filters.
active_replica_host = request_spec.get('active_replica_host')
replication_domain = None
snapshot_host = request_spec.get('snapshot_host')
allowed_hosts = []
if active_replica_host:
allowed_hosts.append(active_replica_host)
if snapshot_host:
allowed_hosts.append(snapshot_host)
replication_domain = None
if active_replica_host or snapshot_host:
temp_hosts = self.host_manager.get_all_host_states_share(elevated)
ar_host = next((host for host in temp_hosts
if host.host == active_replica_host), None)
if ar_host:
replication_domain = ar_host.replication_domain
matching_host = next((host for host in temp_hosts
if host.host in allowed_hosts), None)
if matching_host:
replication_domain = matching_host.replication_domain
# NOTE(zengyingzhe): remove the 'share_backend_name' extra spec,
# let scheduler choose the available host for this replica
# creation request.
# let scheduler choose the available host for this replica or
# snapshot clone creation request.
share_type.get('extra_specs', {}).pop('share_backend_name', None)
if filter_properties is None:

71
manila/scheduler/filters/create_from_snapshot.py

@ -0,0 +1,71 @@
# Copyright 2019 NetApp, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
from manila.scheduler.filters import base_host
from manila.share import utils as share_utils
LOG = log.getLogger(__name__)
class CreateFromSnapshotFilter(base_host.BaseHostFilter):
"""CreateFromSnapshotFilter filters hosts based on replication_domain."""
def host_passes(self, host_state, filter_properties):
"""Return True if new share's host is compatible with snapshot's host.
Design of this filter:
- Creating shares from snapshots in another pool or backend needs
to match with one of the below conditions:
- The backend of the new share must be the same as its parent
snapshot.
- Both new share and snapshot are in the same replication_domain
"""
snapshot_id = filter_properties.get('request_spec', {}).get(
'snapshot_id')
snapshot_host = filter_properties.get(
'request_spec', {}).get('snapshot_host')
if None in [snapshot_id, snapshot_host]:
# NOTE(silvacarlose): if the request does not contain a snapshot_id
# or a snapshot_host, the user is not creating a share from a
# snapshot and we don't need to filter out the host.
return True
snapshot_backend = share_utils.extract_host(snapshot_host, 'backend')
snapshot_rep_domain = filter_properties.get('replication_domain')
host_backend = share_utils.extract_host(host_state.host, 'backend')
host_rep_domain = host_state.replication_domain
# Same backend
if host_backend == snapshot_backend:
return True
# Same replication domain
if snapshot_rep_domain and snapshot_rep_domain == host_rep_domain:
return True
msg = ("The parent's snapshot %(snapshot_id)s back end and "
"replication domain don't match with the back end and "
"replication domain of the Host %(host)s.")
kwargs = {
"snapshot_id": snapshot_id,
"host": host_state.host
}
LOG.debug(msg, kwargs)
return False

2
manila/scheduler/host_manager.py

@ -48,6 +48,7 @@ host_manager_opts = [
'CapabilitiesFilter',
'DriverFilter',
'ShareReplicationFilter',
'CreateFromSnapshotFilter',
],
help='Which filter class names to use for filtering hosts '
'when not specified in the request.'),
@ -55,6 +56,7 @@ host_manager_opts = [
default=[
'CapacityWeigher',
'GoodnessWeigher',
'HostAffinityWeigher',
],
help='Which weigher class names to use for weighing hosts.'),
cfg.ListOpt(

65
manila/scheduler/weighers/host_affinity.py

@ -0,0 +1,65 @@
# Copyright 2019 NetApp, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from manila import context
from manila.db import api as db_api
from manila.scheduler.weighers import base_host
from manila.share import utils as share_utils
class HostAffinityWeigher(base_host.BaseHostWeigher):
def _weigh_object(self, obj, weight_properties):
"""Weigh hosts based on their proximity to the source's share pool.
If no snapshot_id was provided will return 0, otherwise, if source and
destination hosts are located on:
1. same back ends and pools: host is a perfect choice (100)
2. same back ends and different pools: host is a very good choice (75)
3. different back ends with the same AZ: host is a good choice (50)
4. different back ends and AZs: host isn't so good choice (25)
"""
ctx = context.get_admin_context()
request_spec = weight_properties.get('request_spec')
snapshot_id = request_spec.get('snapshot_id')
snapshot_host = request_spec.get('snapshot_host')
if None in [snapshot_id, snapshot_host]:
# NOTE(silvacarlose): if the request does not contain a snapshot_id
# or a snapshot_host, the user is not creating a share from a
# snapshot and we don't need to weigh the host.
return 0
snapshot_ref = db_api.share_snapshot_get(ctx, snapshot_id)
# Source host info: pool, backend and availability zone
src_pool = share_utils.extract_host(snapshot_host, 'pool')
src_backend = share_utils.extract_host(
request_spec.get('snapshot_host'), 'backend')
src_az = snapshot_ref['share']['availability_zone']
# Destination host info: pool, backend and availability zone
dst_pool = share_utils.extract_host(obj.host, 'pool')
dst_backend = share_utils.extract_host(obj.host, 'backend')
# NOTE(dviroel): All hosts were already filtered by the availability
# zone parameter.
dst_az = None
if weight_properties['availability_zone_id']:
dst_az = db_api.availability_zone_get(
ctx, weight_properties['availability_zone_id']).name
if src_backend == dst_backend:
return 100 if (src_pool and src_pool == dst_pool) else 75
else:
return 50 if (src_az and src_az == dst_az) else 25

46
manila/share/api.py

@ -46,7 +46,11 @@ share_api_opts = [
default=False,
help='If set to False, then share creation from snapshot will '
'be performed on the same host. '
'If set to True, then scheduling step will be used.')
'If set to True, then scheduler will be used.'
'When enabling this option make sure that filter '
'CreateShareFromSnapshot is enabled and to have hosts '
'reporting replication_domain option.'
)
]
CONF = cfg.CONF
@ -190,7 +194,18 @@ class API(base.Base):
share_type_id = share_type['id'] if share_type else None
else:
source_share = self.db.share_get(context, snapshot['share_id'])
availability_zone = source_share['instance']['availability_zone']
source_share_az = source_share['instance']['availability_zone']
if availability_zone is None:
availability_zone = source_share_az
elif (availability_zone != source_share_az
and not CONF.use_scheduler_creating_share_from_snapshot):
LOG.error("The specified availability zone must be the same "
"as parent share when you have the configuration "
"option 'use_scheduler_creating_share_from_snapshot'"
" set to False.")
msg = _("The specified availability zone must be the same "
"as the parent share when creating from snapshot.")
raise exception.InvalidInput(reason=msg)
if share_type is None:
# Grab the source share's share_type if no new share type
# has been provided.
@ -327,19 +342,23 @@ class API(base.Base):
context, reservations, share_type_id=share_type_id)
host = None
if snapshot and not CONF.use_scheduler_creating_share_from_snapshot:
# Shares from snapshots with restriction - source host only.
# It is common situation for different types of backends.
host = snapshot['share']['instance']['host']
elif share_group:
snapshot_host = None
if snapshot:
snapshot_host = snapshot['share']['instance']['host']
if not CONF.use_scheduler_creating_share_from_snapshot:
# Shares from snapshots with restriction - source host only.
# It is common situation for different types of backends.
host = snapshot['share']['instance']['host']
if share_group and host is None:
host = share_group['host']
self.create_instance(
context, share, share_network_id=share_network_id, host=host,
availability_zone=availability_zone, share_group=share_group,
share_group_snapshot_member=share_group_snapshot_member,
share_type_id=share_type_id, availability_zones=availability_zones)
share_type_id=share_type_id, availability_zones=availability_zones,
snapshot_host=snapshot_host)
# Retrieve the share with instance details
share = self.db.share_get(context, share['id'])
@ -415,14 +434,16 @@ class API(base.Base):
def create_instance(self, context, share, share_network_id=None,
host=None, availability_zone=None,
share_group=None, share_group_snapshot_member=None,
share_type_id=None, availability_zones=None):
share_type_id=None, availability_zones=None,
snapshot_host=None):
request_spec, share_instance = (
self.create_share_instance_and_get_request_spec(
context, share, availability_zone=availability_zone,
share_group=share_group, host=host,
share_network_id=share_network_id,
share_type_id=share_type_id,
availability_zones=availability_zones))
availability_zones=availability_zones,
snapshot_host=snapshot_host))
if share_group_snapshot_member:
# Inherit properties from the share_group_snapshot_member
@ -461,7 +482,7 @@ class API(base.Base):
self, context, share, availability_zone=None,
share_group=None, host=None, share_network_id=None,
share_type_id=None, cast_rules_to_readonly=False,
availability_zones=None):
availability_zones=None, snapshot_host=None):
availability_zone_id = None
if availability_zone:
@ -528,6 +549,7 @@ class API(base.Base):
'share_proto': share['share_proto'],
'share_id': share['id'],
'snapshot_id': share['snapshot_id'],
'snapshot_host': snapshot_host,
'share_type': share_type,
'share_group': share_group,
'availability_zone_id': availability_zone_id,

125
manila/share/driver.py

@ -24,6 +24,7 @@ import time
from oslo_config import cfg
from oslo_log import log
from manila.common import constants
from manila import exception
from manila.i18n import _
from manila import network
@ -649,8 +650,46 @@ class ShareDriver(object):
raise NotImplementedError()
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
"""Is called to create share from snapshot."""
share_server=None, parent_share=None):
"""Is called to create share from snapshot.
Creating a share from snapshot can take longer than a simple clone
operation if data copy is required from one host to another. For this
reason driver will be able complete this creation asynchronously, by
providing a 'creating_from_snapshot' status in the model update.
When answering asynchronously, drivers must implement the call
'get_share_status' in order to provide updates for shares with
'creating_from_snapshot' status.
It is expected that the driver returns a model update to the share
manager that contains: share status and a list of export_locations.
A list of 'export_locations' is mandatory only for share in 'available'
status.
The current supported status are 'available' and
'creating_from_snapshot'.
:param context: Current context
:param share: Share instance model with share data.
:param snapshot: Snapshot instance model .
:param share_server: Share server model or None.
:param parent_share: Share model from parent snapshot with share data
and share server model.
:returns: a dictionary of updates containing current share status and
its export_location (if available).
Example::
{
'status': 'available',
'export_locations': [{...}, {...}],
}
:raises: ShareBackendException.
A ShareBackendException in this method will set the instance to
'error' and the operation will end.
"""
raise NotImplementedError()
def create_snapshot(self, context, snapshot, share_server=None):
@ -1311,6 +1350,15 @@ class ShareDriver(object):
share_server=None):
"""Create a share group from a share group snapshot.
When creating a share from snapshot operation takes longer than a
simple clone operation, drivers will be able to complete this creation
asynchronously, by providing a 'creating_from_snapshot' status in the
returned model update. The current supported status are 'available' and
'creating_from_snapshot'.
In order to provide updates for shares with 'creating_from_snapshot'
status, drivers must implement the call 'get_share_status'.
:param context:
:param share_group_dict: The share group details
EXAMPLE:
@ -1372,12 +1420,27 @@ class ShareDriver(object):
share_update_list - a list of dictionaries containing dicts for
every share created in the share group. Any share dicts should at a
minimum contain the 'id' key and 'export_locations'.
minimum contain the 'id' key and, for synchronous creation, the
'export_locations'. For asynchronous share creation this dict
must also contain the key 'status' with the value set to
'creating_from_snapshot'. The current supported status are
'available' and 'creating_from_snapshot'.
Export locations should be in the same format as returned by
a share_create. This list may be empty or None. EXAMPLE:
.. code::
[{'id': 'uuid', 'export_locations': [{...}, {...}]}]
[
{
'id': 'uuid',
'export_locations': [{...}, {...}],
},
{
'id': 'uuid',
'export_locations': [],
'status': 'creating_from_snapshot',
},
]
"""
# Ensure that the share group snapshot has members
if not share_group_snapshot_dict['share_group_snapshot_members']:
@ -1389,18 +1452,38 @@ class ShareDriver(object):
LOG.debug('Creating share group from group snapshot %s.',
share_group_snapshot_dict['id'])
for clone in clone_list:
kwargs = {}
share_update_info = {}
if self.driver_handles_share_servers:
kwargs['share_server'] = share_server
export_locations = (
model_update = (
self.create_share_from_snapshot(
context, clone['share'], clone['snapshot'], **kwargs))
share_update_list.append({
if isinstance(model_update, dict):
status = model_update.get('status')
# NOTE(dviroel): share status is mandatory when answering
# a model update. If not provided, won't be possible to
# determine if was successfully created.
if status is None:
msg = _("Driver didn't provide a share status.")
raise exception.InvalidShareInstance(reason=msg)
if status not in [constants.STATUS_AVAILABLE,
constants.STATUS_CREATING_FROM_SNAPSHOT]:
msg = _('Driver returned an invalid status: %s') % status
raise exception.InvalidShareInstance(reason=msg)
share_update_info.update({'status': status})
export_locations = model_update.get('export_locations', [])
else:
# NOTE(dviroel): the driver that doesn't implement the new
# model_update will return only the export locations
export_locations = model_update
share_update_info.update({
'id': clone['share']['id'],
'export_locations': export_locations,
})
share_update_list.append(share_update_info)
return None, share_update_list
def delete_share_group(self, context, share_group_dict, share_server=None):
@ -2742,3 +2825,31 @@ class ShareDriver(object):
this share server.
"""
raise NotImplementedError()
def get_share_status(self, share, share_server=None):
"""Invoked periodically to get the current status of a given share.
Driver can use this method to update the status of a share that is
still pending from other operations.
This method is expected to be called in a periodic interval set by the
'periodic_interval' configuration in seconds.
:param share: share to get updated status from.
:param share_server: share server model or None.
:returns: a dictionary of updates with the current share status, that
must be 'available', 'creating_from_snapshot' or 'error', a list of
export locations, if available, and a progress field which
indicates the completion of the share creation operation.
EXAMPLE::
{
'status': 'available',
'export_locations': [{...}, {...}],
'progress': '50%'
}
:raises: ShareBackendException.
A ShareBackendException in this method will set the instance status
to 'error'.
"""
raise NotImplementedError()

2
manila/share/drivers/dell_emc/driver.py

@ -107,7 +107,7 @@ class EMCShareDriver(driver.ShareDriver):
return location
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Is called to create share from snapshot."""
location = self.plugin.create_share_from_snapshot(
context, share, snapshot, share_server)

2
manila/share/drivers/dell_emc/plugins/powermax/connection.py

@ -214,7 +214,7 @@ class PowerMaxStorageConnection(driver.StorageConnection):
'share_name': share_name})
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Create a share from a snapshot - clone a snapshot."""
share_name = share['id']

2
manila/share/drivers/dell_emc/plugins/unity/connection.py

@ -223,7 +223,7 @@ class UnityStorageConnection(driver.StorageConnection):
return locations
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Create a share from a snapshot - clone a snapshot."""
share_name = share['id']

2
manila/share/drivers/dell_emc/plugins/vnx/connection.py

@ -211,7 +211,7 @@ class VNXStorageConnection(driver.StorageConnection):
'share_name': share_name})
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Create a share from a snapshot - clone a snapshot."""
share_name = share['id']

2
manila/share/drivers/generic.py

@ -640,7 +640,7 @@ class GenericShareDriver(driver.ExecuteMixin, driver.ShareDriver):
@ensure_server
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Is called to create share from snapshot."""
return self._create_share(
context, share,

2
manila/share/drivers/glusterfs/layout.py

@ -235,7 +235,7 @@ class GlusterfsShareLayoutBase(object):
@abc.abstractmethod
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Is called to create share from snapshot."""
@abc.abstractmethod

2
manila/share/drivers/glusterfs/layout_directory.py

@ -188,7 +188,7 @@ class GlusterfsDirectoryMappedLayout(layout.GlusterfsShareLayoutBase):
pass
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
raise NotImplementedError
def create_snapshot(self, context, snapshot, share_server=None):

2
manila/share/drivers/glusterfs/layout_volume.py

@ -463,7 +463,7 @@ class GlusterfsVolumeMappedLayout(layout.GlusterfsShareLayoutBase):
return backend_snapshot_name
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
old_gmgr = self._share_manager(snapshot['share_instance'])
# Snapshot clone feature in GlusterFS server essential to support this

2
manila/share/drivers/hdfs/hdfs_native.py

@ -227,7 +227,7 @@ class HDFSNativeShareDriver(driver.ExecuteMixin, driver.ShareDriver):
return self._get_share_path(share)
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Creates a snapshot."""
self._create_share(share)
share_path = '/' + share['name']

2
manila/share/drivers/hitachi/hnas/driver.py

@ -448,7 +448,7 @@ class HitachiHNASDriver(driver.ShareDriver):
{'id': snapshot['id']})
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
r"""Creates a new share from snapshot.
:param context: The `context.RequestContext` object for the request

2
manila/share/drivers/hpe/hpe_3par_driver.py

@ -501,7 +501,7 @@ class HPE3ParShareDriver(driver.ShareDriver):
return self._hpe3par.build_export_locations(protocol, ips, path)
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Is called to create share from snapshot."""
fpg, vfs, ips = self._get_pool_location(share, share_server)

2
manila/share/drivers/huawei/base.py

@ -66,7 +66,7 @@ class HuaweiBase(object):
@abc.abstractmethod
def create_share_from_snapshot(self, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Create share from snapshot."""
@abc.abstractmethod

2
manila/share/drivers/huawei/huawei_nas.py

@ -119,7 +119,7 @@ class HuaweiNasDriver(driver.ShareDriver):
self.plugin.extend_share(share, new_size, share_server)
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Create a share from snapshot."""
LOG.debug("Create a share from snapshot %s.", snapshot['snapshot_id'])
location = self.plugin.create_share_from_snapshot(share, snapshot)

2
manila/share/drivers/huawei/v3/connection.py

@ -383,7 +383,7 @@ class V3StorageConnection(driver.HuaweiBase):
return share
def create_share_from_snapshot(self, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Create a share from snapshot."""
share_fs_id = self.helper.get_fsid_by_name(snapshot['share_name'])
if not share_fs_id:

2
manila/share/drivers/ibm/gpfs.py

@ -493,7 +493,7 @@ class GPFSShareDriver(driver.ExecuteMixin, driver.GaneshaMixin,
return location
def create_share_from_snapshot(self, ctx, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Is called to create share from a snapshot."""
share_path = self._get_share_path(share)
self._create_share_from_snapshot(share, snapshot, share_path)

2
manila/share/drivers/infinidat/infinibox.py

@ -403,7 +403,7 @@ class InfiniboxShareDriver(driver.ShareDriver):
@infinisdk_to_manila_exceptions
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
name = self._make_share_name(share)
infinidat_snapshot = self._get_infinidat_snapshot(snapshot)
infinidat_new_share = infinidat_snapshot.create_snapshot(

2
manila/share/drivers/inspur/as13000/as13000_nas.py

@ -343,7 +343,7 @@ class AS13000ShareDriver(driver.ShareDriver):
@inspur_driver_debug_trace
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Create a share from snapshot."""
pool, name, size, proto = self._get_share_instance_pnsp(share)

2
manila/share/drivers/lvm.py

@ -242,7 +242,7 @@ class LVMShareDriver(LVMMixin, driver.ShareDriver):
return location
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Is called to create share from snapshot."""
self._allocate_container(share)
snapshot_device_name = self._get_local_path(snapshot)

2
manila/share/drivers/maprfs/maprfs_native.py

@ -206,7 +206,7 @@ class MapRFSNativeShareDriver(driver.ExecuteMixin, driver.ShareDriver):
raise exception.ShareResourceNotFound(share_id=share['share_id'])
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Creates a share from snapshot."""
metadata = self.api.get_share_metadata(context,
{'id': share['share_id']})

2
manila/share/drivers/netapp/dataontap/cluster_mode/lib_base.py

@ -485,7 +485,7 @@ class NetAppCmodeFileStorageLibrary(object):
@na_utils.trace
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Creates new share from snapshot."""
vserver, vserver_client = self._get_vserver(share_server=share_server)
self._allocate_container_from_snapshot(

2
manila/share/drivers/nexenta/ns4/nexenta_nas.py

@ -77,7 +77,7 @@ class NexentaNasDriver(driver.ShareDriver):
return self.helper.create_filesystem(share)
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Is called to create share from snapshot."""
LOG.debug('Creating share from snapshot %s.', snapshot['name'])
return self.helper.create_share_from_snapshot(share, snapshot)

2
manila/share/drivers/nexenta/ns5/nexenta_nas.py

@ -189,7 +189,7 @@ class NexentaNasDriver(driver.ShareDriver):
return '%s:%s' % (self.nas_host, filesystem['mountPoint'])
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Is called to create share from snapshot."""
snapshot_path = self._get_snapshot_path(snapshot)
LOG.debug('Creating share from snapshot %s.', snapshot_path)

2
manila/share/drivers/qnap/qnap.py

@ -519,7 +519,7 @@ class QnapShareDriver(driver.ShareDriver):
retries=5)
@utils.synchronized('qnap-create_share_from_snapshot')
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Create a share from a snapshot."""
LOG.debug('Entering create_share_from_snapshot. The source '
'snapshot=%(snap)s. The created share=%(share)s',

2
manila/share/drivers/tegile/tegile.py

@ -281,7 +281,7 @@ class TegileShareDriver(driver.ShareDriver):
@debugger
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Create a share from a snapshot - clone a snapshot."""
pool, project, share_name = self._get_pool_project_share_name(share)

2
manila/share/drivers/veritas/veritas_isa.py

@ -524,7 +524,7 @@ class ACCESSShareDriver(driver.ExecuteMixin, driver.ShareDriver):
json.dumps(data2), 'DELETE')
def create_share_from_snapshot(self, ctx, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""create share from a snapshot."""
sharename = snapshot['share_name']
va_sharename = self._get_va_share_name(sharename)

2
manila/share/drivers/zfsonlinux/driver.py

@ -580,7 +580,7 @@ class ZFSonLinuxShareDriver(zfs_utils.ExecuteMixin, driver.ShareDriver):
@ensure_share_server_not_provided
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Is called to create a share from snapshot."""
dataset_name = self._get_dataset_name(share)
ssh_cmd = '%(username)s@%(host)s' % {

2
manila/share/drivers/zfssa/zfssashare.py

@ -242,7 +242,7 @@ class ZFSSAShareDriver(driver.ShareDriver):
snapshot['id'])
def create_share_from_snapshot(self, context, share, snapshot,
share_server=None):
share_server=None, parent_share=None):
"""Create a share from a snapshot - clone a snapshot."""
lcfg = self.configuration
LOG.debug("ZFSSAShareDriver.create_share_from_snapshot: clone=%s",

289
manila/share/manager.py

@ -403,7 +403,7 @@ class ShareManager(manager.SchedulerDependentManager):
self._ensure_share_instance_has_pool(ctxt, share_instance)
share_instance = self.db.share_instance_get(
ctxt, share_instance['id'], with_share_data=True)
share_instance_dict = self._get_share_replica_dict(
share_instance_dict = self._get_share_instance_dict(
ctxt, share_instance)