Create share from snapshot in another pool or backend

This patch enables the creation of a share from snapshot
specifying another pool or backend. In the scheduler, a
new filter and weigher were implemented in order to consider
this operation if the backend supports it. Also, a new
field called 'progress' was added in the share and share
instance. The 'progress' field indicates the status
of the operation create share from snapshot (in percentage).
Finally, a new periodic task was added in order to constantly
check the share status.

Partially-implements: bp create-share-from-snapshot-in-another-pool-or-backend

DOCImpact
Change-Id: Iab13a0961eb4a387a502246e5d4b79bc9046e04b
Co-authored-by: carloss <ces.eduardo98@gmail.com>
Co-authored-by: dviroel <viroel@gmail.com>

api-ref/source/parameters.yaml

@@ -468,8 +468,8 @@ source_share_group_snapshot_id_query:
 status_query:
   description: |
     Filters by a share status. A valid value is
-    ``creating``, ``error``, ``available``, ``deleting``,
-    ``error_deleting``, ``manage_starting``, ``manage_error``,
+    ``creating``, ``creating_from_snapshot``, ``error``, ``available``,
+    ``deleting``, ``error_deleting``,  ``manage_starting``, ``manage_error``,
     ``unmanage_starting``, ``unmanage_error``, ``migrating``,
     ``extending``, ``extending_error``, ``shrinking``,
     ``shrinking_error``, or ``shrinking_possible_data_loss_error``.
@@ -1540,6 +1540,13 @@ progress:
   in: body
   required: true
   type: string
+progress_share_instance:
+  description: |
+    The progress of the share creation.
+  in: body
+  min_version: 2.54
+  required: true
+  type: string
 project:
   description: |
     The UUID of the project to which access to the
@@ -2713,7 +2720,7 @@ status_1:
   type: string
 status_16:
   description: |
-    The share status, which is ``creating``,
+    The share status, which is ``creating``, ``creating_from_snapshot``,
     ``error``, ``available``, ``deleting``, ``error_deleting``,
     ``manage_starting``, ``manage_error``, ``unmanage_starting``,
     ``unmanage_error``, ``unmanaged``, ``extend``,
@@ -2737,15 +2744,16 @@ status_3:
     ``extending_error``. Extend share failed.  - ``shrinking``. Share
     is being shrunk.  - ``shrinking_error``. Failed to update quota on
     share shrinking.  - ``shrinking_possible_data_loss_error``. Shrink
-    share failed due to   possible data loss.
+    share failed due to   possible data loss. - ``creating_from_snapshot``.
+    The share is being created from a parent snapshot.
   in: body
   required: true
   type: string
 status_5:
   description: |
     The share instance status. A valid value is
-    ``available``, ``error``, ``creating``, ``deleting``, and
-    ``error_deleting``.
+    ``available``, ``error``, ``creating``, ``deleting``,
+    ``creating_from_snapshot``, or ``error_deleting``.
   in: body
   required: true
   type: string

api-ref/source/samples/share-create-response.json

@@ -1,6 +1,7 @@
 {
     "share": {
         "status": null,
+        "progress": null,
         "share_server_id": null,
         "project_id": "16e1ab15c35a457e9c2b2aa189f544e1",
         "name": "share_London",

api-ref/source/samples/share-instances-list-response.json

@@ -2,6 +2,7 @@
     "share_instances": [
         {
             "status": "error",
+            "progress": null,
             "share_id": "406ea93b-32e9-4907-a117-148b3945749f",
             "availability_zone": "nova",
             "replica_state": null,
@@ -14,6 +15,7 @@
         },
         {
             "status": "available",
+            "progress": "100%",
             "share_id": "d94a8548-2079-4be0-b21c-0a887acd31ca",
             "availability_zone": "nova",
             "replica_state": null,
@@ -23,6 +25,19 @@
             "share_server_id": "ba11930a-bf1a-4aa7-bae4-a8dfbaa3cc73",
             "host": "manila2@generic1#GENERIC1",
             "id": "75559a8b-c90c-42a7-bda2-edbe86acfb7b"
+        },
+        {
+            "status": "creating_from_snapshot",
+            "progress": "30%",
+            "share_id": "9bb15af4-27e5-4174-ae15-dc549d4a3b51",
+            "availability_zone": "nova",
+            "replica_state": null,
+            "created_at": "2015-09-07T09:01:15.000000",
+            "share_network_id": "713df749-aac0-4a54-af52-10f6c991e80c",
+            "cast_rules_to_readonly": false,
+            "share_server_id": "ba11930a-bf1a-4aa7-bae4-a8dfbaa3cc73",
+            "host": "manila2@generic1#GENERIC1",
+            "id": "48155648-2fd3-480d-b02b-44b995c24bab"
         }
     ]
 }

api-ref/source/samples/share-show-instance-response.json

@@ -1,6 +1,7 @@
 {
     "share_instance": {
         "status": "available",
+        "progress": "100%",
         "share_id": "d94a8548-2079-4be0-b21c-0a887acd31ca",
         "availability_zone": "nova",
         "replica_state": null,

api-ref/source/samples/share-show-response.json

@@ -27,6 +27,7 @@
             "aim": "doc"
         },
         "status": "available",
+        "progress": "100%",
         "description": "My custom share London",
         "host": "manila2@generic1#GENERIC1",
         "access_rules_status": "active",

api-ref/source/samples/shares-list-detailed-response.json

@@ -25,6 +25,7 @@
             "project_id": "16e1ab15c35a457e9c2b2aa189f544e1",
             "metadata": {},
             "status": "error",
+            "progress": null,
             "access_rules_status": "active",
             "description": "There is a share description.",
             "host": "manila2@generic1#GENERIC1",
@@ -64,6 +65,7 @@
             "project_id": "16e1ab15c35a457e9c2b2aa189f544e1",
             "metadata": {},
             "status": "available",
+            "progress": "100%",
             "access_rules_status": "active",
             "description": "Changed description.",
             "host": "manila2@generic1#GENERIC1",

api-ref/source/share-instances.inc

@@ -48,6 +48,7 @@ Response parameters
    - status: status_5
    - access_rules_status: access_rules_status
    - share_id: share_id_2
+   - progress: progress_share_instance
    - availability_zone: availability_zone_1
    - created_at: created_at
    - replica_state: replica_state
@@ -105,6 +106,7 @@ Response parameters
    - status: status_5
    - access_rules_status: access_rules_status
    - share_id: share_id_2
+   - progress: progress_share_instance
    - availability_zone: availability_zone_1
    - created_at: created_at
    - replica_state: replica_state

api-ref/source/shares.inc

@@ -39,6 +39,8 @@ A share has one of these status values:
 +----------------------------------------+--------------------------------------------------------+
 | ``creating``                           | The share is being created.                            |
 +----------------------------------------+--------------------------------------------------------+
+| ``creating_from_snapshot``             | The share is being created from a parent snapshot.     |
++----------------------------------------+--------------------------------------------------------+
 | ``deleting``                           | The share is being deleted.                            |
 +----------------------------------------+--------------------------------------------------------+
 | ``deleted``                            | The share was deleted.                                 |
@@ -220,6 +222,7 @@ Response parameters
    - project_id: project_id
    - metadata: metadata
    - status: status_16
+   - progress: progress_share_instance
    - description: description
    - host: host_1
    - access_rules_status: access_rules_status
@@ -291,6 +294,7 @@ Response parameters
    - project_id: project_id
    - metadata: metadata
    - status: status_16
+   - progress: progress_share_instance
    - description: description
    - host: host_9
    - access_rules_status: access_rules_status
@@ -369,6 +373,7 @@ Response parameters
 
    - id: id_4
    - status: status_3
+   - progress: progress_share_instance
    - links: links
    - project_id: project_id
    - share_proto: share_proto

manila/api/openstack/api_version_request.py

@@ -144,13 +144,16 @@ REST_API_VERSION_HISTORY = """
              filters, support querying user messages within the specified time
              period.
     * 2.53 - Added quota control to share replicas.
+    * 2.54 - Share and share instance objects include a new field called
+             "progress" which indicates the completion of a share creation
+             operation as a percentage.
 """
 
 # The minimum and maximum versions of the API supported
 # The default api version request is defined to be the
 # minimum version of the API supported.
 _MIN_API_VERSION = "2.0"
-_MAX_API_VERSION = "2.53"
+_MAX_API_VERSION = "2.54"
 DEFAULT_API_VERSION = _MIN_API_VERSION
 
 

manila/api/openstack/rest_api_version_history.rst

@@ -296,3 +296,8 @@ user documentation.
 2.53
 ----
   Added quota control for share replicas and replica gigabytes.
+
+2.54
+----
+  Share and share instance objects include a new field called "progress" which
+  indicates the completion of a share creation operation as a percentage.

manila/api/views/share_instance.py

@@ -11,6 +11,7 @@
 #    under the License.
 
 from manila.api import common
+from manila.common import constants
 
 
 class ViewBuilder(common.ViewBuilder):
@@ -25,6 +26,8 @@ class ViewBuilder(common.ViewBuilder):
         "add_replication_fields",
         "add_share_type_field",
         "add_cast_rules_to_readonly_field",
+        "add_progress_field",
+        "translate_creating_from_snapshot_status",
     ]
 
     def detail_list(self, request, instances):
@@ -47,6 +50,7 @@ class ViewBuilder(common.ViewBuilder):
             'export_location': share_instance.get('export_location'),
             'export_locations': export_locations,
         }
+
         self.update_versioned_resource_dict(
             request, instance_dict, share_instance)
         return {'share_instance': instance_dict}
@@ -91,3 +95,14 @@ class ViewBuilder(common.ViewBuilder):
                                          share_instance):
         instance_dict['cast_rules_to_readonly'] = share_instance.get(
             'cast_rules_to_readonly', False)
+
+    @common.ViewBuilder.versioned_method("1.0", "2.53")
+    def translate_creating_from_snapshot_status(self, context, instance_dict,
+                                                share_instance):
+        if (share_instance.get('status') ==
+                constants.STATUS_CREATING_FROM_SNAPSHOT):
+            instance_dict['status'] = constants.STATUS_CREATING
+
+    @common.ViewBuilder.versioned_method("2.54")
+    def add_progress_field(self, context, instance_dict, share_instance):
+        instance_dict['progress'] = share_instance.get('progress')

manila/api/views/shares.py

@@ -34,6 +34,8 @@ class ViewBuilder(common.ViewBuilder):
         "translate_access_rules_status",
         "add_share_group_fields",
         "add_mount_snapshot_support_field",
+        "add_progress_field",
+        "translate_creating_from_snapshot_status",
     ]
 
     def summary_list(self, request, shares, count=None):
@@ -92,6 +94,7 @@ class ViewBuilder(common.ViewBuilder):
             'is_public': share.get('is_public'),
             'export_locations': export_locations,
         }
+
         self.update_versioned_resource_dict(request, share_dict, share)
 
         if context.is_admin:
@@ -184,3 +187,13 @@ class ViewBuilder(common.ViewBuilder):
             shares_dict['shares_links'] = shares_links
 
         return shares_dict
+
+    @common.ViewBuilder.versioned_method("1.0", "2.53")
+    def translate_creating_from_snapshot_status(self, context, share_dict,
+                                                share):
+        if share.get('status') == constants.STATUS_CREATING_FROM_SNAPSHOT:
+            share_dict['status'] = constants.STATUS_CREATING
+
+    @common.ViewBuilder.versioned_method("2.54")
+    def add_progress_field(self, context, share_dict, share):
+        share_dict['progress'] = share.get('progress')

manila/common/constants.py

@@ -18,6 +18,7 @@ DB_MAX_INT = 0x7FFFFFFF
 
 # SHARE AND GENERAL STATUSES
 STATUS_CREATING = 'creating'
+STATUS_CREATING_FROM_SNAPSHOT = 'creating_from_snapshot'
 STATUS_DELETING = 'deleting'
 STATUS_DELETED = 'deleted'
 STATUS_ERROR = 'error'

manila/db/api.py

@@ -340,10 +340,11 @@ def share_instances_get_all_by_share_server(context, share_server_id):
                                                         share_server_id)
 
 
-def share_instances_get_all_by_host(context, host, with_share_data=False):
+def share_instances_get_all_by_host(context, host, with_share_data=False,
+                                    status=None):
     """Returns all share instances with given host."""
     return IMPL.share_instances_get_all_by_host(
-        context, host, with_share_data=with_share_data)
+        context, host, with_share_data=with_share_data, status=status)
 
 
 def share_instances_get_all_by_share_network(context, share_network_id):

manila/db/migrations/alembic/versions/e6d88547b381_add_progress_field_to_share_instance.py

@@ -0,0 +1,62 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""add-progress-field-to-share-instance
+
+Revision ID: e6d88547b381
+Revises: 805685098bd2
+Create Date: 2020-01-31 14:06:15.952747
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'e6d88547b381'
+down_revision = '805685098bd2'
+
+from alembic import op
+from manila.common import constants
+from manila.db.migrations import utils
+from oslo_log import log
+import sqlalchemy as sa
+
+
+LOG = log.getLogger(__name__)
+
+
+def upgrade():
+
+    try:
+        connection = op.get_bind()
+        op.add_column('share_instances',
+                      sa.Column('progress', sa.String(32), nullable=True,
+                                default=None))
+        share_instances_table = utils.load_table('share_instances', connection)
+
+        updated_data = {'progress': '100%'}
+
+        # pylint: disable=no-value-for-parameter
+        op.execute(
+            share_instances_table.update().where(
+                share_instances_table.c.status == constants.STATUS_AVAILABLE,
+            ).values(updated_data)
+        )
+    except Exception:
+        LOG.error("Column share_instances.progress not created.")
+        raise
+
+
+def downgrade():
+    try:
+        op.drop_column('share_instances', 'progress')
+    except Exception:
+        LOG.error("Column share_instances.progress not dropped.")
+        raise

manila/db/sqlalchemy/api.py

@@ -1579,7 +1579,7 @@ def _set_instances_share_data(context, instances, session):
 
 @require_admin_context
 def share_instances_get_all_by_host(context, host, with_share_data=False,
-                                    session=None):
+                                    status=None, session=None):
     """Retrieves all share instances hosted on a host."""
     session = session or get_session()
     instances = (
@@ -1588,8 +1588,12 @@ def share_instances_get_all_by_host(context, host, with_share_data=False,
                 models.ShareInstance.host == host,
                 models.ShareInstance.host.like("{0}#%".format(host))
             )
-        ).all()
+        )
     )
+    if status is not None:
+        instances = instances.filter(models.ShareInstance.status == status)
+    # Returns list of all instances that satisfy filters.
+    instances = instances.all()
 
     if with_share_data:
         instances = _set_instances_share_data(context, instances, session)

manila/db/sqlalchemy/models.py

@@ -257,6 +257,11 @@ class Share(BASE, ManilaBase):
             return len(replicas) > 1
         return False
 
+    @property
+    def progress(self):
+        if len(self.instances) > 0:
+            return self.instance.progress
+
     @property
     def instance(self):
         # NOTE(gouthamr): The order of preference: status 'replication_change',
@@ -364,6 +369,7 @@ class ShareInstance(BASE, ManilaBase):
     deleted = Column(String(36), default='False')
     host = Column(String(255))
     status = Column(String(255))
+    progress = Column(String(32))
 
     ACCESS_STATUS_PRIORITIES = {
         constants.STATUS_ACTIVE: 0,

manila/message/message_field.py

@@ -80,6 +80,12 @@ class Detail(object):
           "set to extending_error. This action cannot be re-attempted until "
           "the status has been rectified. Contact your administrator to "
           "determine the cause of this failure."))
+    FILTER_CREATE_FROM_SNAPSHOT = ('016', FILTER_MSG % 'CreateFromSnapshot')
+    DRIVER_FAILED_CREATING_FROM_SNAP = (
+        '017',
+        _("Share Driver has failed to create the share from snapshot. This "
+          "operation can be re-attempted by creating a new share. Contact "
+          "your administrator to determine the cause of this failure."))
 
     ALL = (UNKNOWN_ERROR,
            NO_VALID_HOST,
@@ -95,7 +101,9 @@ class Detail(object):
            FILTER_JSON,
            FILTER_RETRY,
            FILTER_REPLICATION,
-           DRIVER_FAILED_EXTEND)
+           DRIVER_FAILED_EXTEND,
+           FILTER_CREATE_FROM_SNAPSHOT,
+           DRIVER_FAILED_CREATING_FROM_SNAP)
 
     # Exception and detail mappings
     EXCEPTION_DETAIL_MAPPINGS = {
@@ -113,6 +121,7 @@ class Detail(object):
         'JsonFilter': FILTER_JSON,
         'RetryFilter': FILTER_RETRY,
         'ShareReplicationFilter': FILTER_REPLICATION,
+        'CreateFromSnapshotFilter': FILTER_CREATE_FROM_SNAPSHOT,
     }
 
 

manila/scheduler/drivers/filter.py

@@ -162,21 +162,27 @@ class FilterScheduler(base.Scheduler):
 
         share_group = request_spec.get('share_group')
 
-        # NOTE(gouthamr): If 'active_replica_host' is present in the request
-        # spec, pass that host's 'replication_domain' to the
-        # ShareReplication filter.
+        # NOTE(gouthamr): If 'active_replica_host' or 'snapshot_host' is
+        # present in the request spec, pass that host's 'replication_domain' to
+        # the ShareReplication and CreateFromSnapshot filters.
         active_replica_host = request_spec.get('active_replica_host')
-        replication_domain = None
+        snapshot_host = request_spec.get('snapshot_host')
+        allowed_hosts = []
         if active_replica_host:
+            allowed_hosts.append(active_replica_host)
+        if snapshot_host:
+            allowed_hosts.append(snapshot_host)
+        replication_domain = None
+        if active_replica_host or snapshot_host:
             temp_hosts = self.host_manager.get_all_host_states_share(elevated)
-            ar_host = next((host for host in temp_hosts
-                            if host.host == active_replica_host), None)
-            if ar_host:
-                replication_domain = ar_host.replication_domain
+            matching_host = next((host for host in temp_hosts
+                                  if host.host in allowed_hosts), None)
+            if matching_host:
+                replication_domain = matching_host.replication_domain
 
             # NOTE(zengyingzhe): remove the 'share_backend_name' extra spec,
-            # let scheduler choose the available host for this replica
-            # creation request.
+            # let scheduler choose the available host for this replica or
+            # snapshot clone creation request.
             share_type.get('extra_specs', {}).pop('share_backend_name', None)
 
         if filter_properties is None:

manila/scheduler/filters/create_from_snapshot.py

@@ -0,0 +1,71 @@
+# Copyright 2019 NetApp, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from oslo_log import log
+
+from manila.scheduler.filters import base_host
+from manila.share import utils as share_utils
+
+LOG = log.getLogger(__name__)
+
+
+class CreateFromSnapshotFilter(base_host.BaseHostFilter):
+    """CreateFromSnapshotFilter filters hosts based on replication_domain."""
+
+    def host_passes(self, host_state, filter_properties):
+        """Return True if new share's host is compatible with snapshot's host.
+
+        Design of this filter:
+
+            - Creating shares from snapshots in another pool or backend needs
+              to match with one of the below conditions:
+              - The backend of the new share must be the same as its parent
+                snapshot.
+              - Both new share and snapshot are in the same replication_domain
+        """
+        snapshot_id = filter_properties.get('request_spec', {}).get(
+            'snapshot_id')
+        snapshot_host = filter_properties.get(
+            'request_spec', {}).get('snapshot_host')
+
+        if None in [snapshot_id, snapshot_host]:
+            # NOTE(silvacarlose): if the request does not contain a snapshot_id
+            # or a snapshot_host, the user is not creating a share from a
+            # snapshot and we don't need to filter out the host.
+            return True
+
+        snapshot_backend = share_utils.extract_host(snapshot_host, 'backend')
+        snapshot_rep_domain = filter_properties.get('replication_domain')
+
+        host_backend = share_utils.extract_host(host_state.host, 'backend')
+        host_rep_domain = host_state.replication_domain
+
+        # Same backend
+        if host_backend == snapshot_backend:
+            return True
+        # Same replication domain
+        if snapshot_rep_domain and snapshot_rep_domain == host_rep_domain:
+            return True
+
+        msg = ("The parent's snapshot %(snapshot_id)s back end and "
+               "replication domain don't match with the back end and "
+               "replication domain of the Host %(host)s.")
+        kwargs = {
+            "snapshot_id": snapshot_id,
+            "host": host_state.host
+        }
+        LOG.debug(msg, kwargs)
+        return False

manila/scheduler/host_manager.py

@@ -48,6 +48,7 @@ host_manager_opts = [
                     'CapabilitiesFilter',
                     'DriverFilter',
                     'ShareReplicationFilter',
+                    'CreateFromSnapshotFilter',
                 ],
                 help='Which filter class names to use for filtering hosts '
                      'when not specified in the request.'),
@@ -55,6 +56,7 @@ host_manager_opts = [
                 default=[
                     'CapacityWeigher',
                     'GoodnessWeigher',
+                    'HostAffinityWeigher',
                 ],
                 help='Which weigher class names to use for weighing hosts.'),
     cfg.ListOpt(

manila/scheduler/weighers/host_affinity.py

@@ -0,0 +1,65 @@
+# Copyright 2019 NetApp, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from manila import context
+from manila.db import api as db_api
+from manila.scheduler.weighers import base_host
+from manila.share import utils as share_utils
+
+
+class HostAffinityWeigher(base_host.BaseHostWeigher):
+
+    def _weigh_object(self, obj, weight_properties):
+        """Weigh hosts based on their proximity to the source's share pool.
+
+        If no snapshot_id was provided will return 0, otherwise, if source and
+        destination hosts are located on:
+        1. same back ends and pools: host is a perfect choice (100)
+        2. same back ends and different pools: host is a very good choice (75)
+        3. different back ends with the same AZ: host is a good choice (50)
+        4. different back ends and AZs: host isn't so good choice (25)
+        """
+
+        ctx = context.get_admin_context()
+        request_spec = weight_properties.get('request_spec')
+        snapshot_id = request_spec.get('snapshot_id')
+        snapshot_host = request_spec.get('snapshot_host')
+
+        if None in [snapshot_id, snapshot_host]:
+            # NOTE(silvacarlose): if the request does not contain a snapshot_id
+            # or a snapshot_host, the user is not creating a share from a
+            # snapshot and we don't need to weigh the host.
+            return 0
+
+        snapshot_ref = db_api.share_snapshot_get(ctx, snapshot_id)
+        # Source host info: pool, backend and availability zone
+        src_pool = share_utils.extract_host(snapshot_host, 'pool')
+        src_backend = share_utils.extract_host(
+            request_spec.get('snapshot_host'), 'backend')
+        src_az = snapshot_ref['share']['availability_zone']
+        # Destination host info: pool, backend and availability zone
+        dst_pool = share_utils.extract_host(obj.host, 'pool')
+        dst_backend = share_utils.extract_host(obj.host, 'backend')
+        # NOTE(dviroel): All hosts were already filtered by the availability
+        # zone parameter.
+        dst_az = None
+        if weight_properties['availability_zone_id']:
+            dst_az = db_api.availability_zone_get(
+                ctx, weight_properties['availability_zone_id']).name
+
+        if src_backend == dst_backend:
+            return 100 if (src_pool and src_pool == dst_pool) else 75
+        else:
+            return 50 if (src_az and src_az == dst_az) else 25

manila/share/api.py

@@ -46,7 +46,11 @@ share_api_opts = [
                 default=False,
                 help='If set to False, then share creation from snapshot will '
                      'be performed on the same host. '
-                     'If set to True, then scheduling step will be used.')
+                     'If set to True, then scheduler will be used.'
+                     'When enabling this option make sure that filter '
+                     'CreateShareFromSnapshot is enabled and to have hosts '
+                     'reporting replication_domain option.'
+                )
 ]
 
 CONF = cfg.CONF
@@ -190,7 +194,18 @@ class API(base.Base):
             share_type_id = share_type['id'] if share_type else None
         else:
             source_share = self.db.share_get(context, snapshot['share_id'])
-            availability_zone = source_share['instance']['availability_zone']
+            source_share_az = source_share['instance']['availability_zone']
+            if availability_zone is None:
+                availability_zone = source_share_az
+            elif (availability_zone != source_share_az
+                  and not CONF.use_scheduler_creating_share_from_snapshot):
+                LOG.error("The specified availability zone must be the same "
+                          "as parent share when you have the configuration "
+                          "option 'use_scheduler_creating_share_from_snapshot'"
+                          " set to False.")
+                msg = _("The specified availability zone must be the same "
+                        "as the parent share when creating from snapshot.")
+                raise exception.InvalidInput(reason=msg)
             if share_type is None:
                 # Grab the source share's share_type if no new share type
                 # has been provided.
@@ -327,19 +342,23 @@ class API(base.Base):
                         context, reservations, share_type_id=share_type_id)
 
         host = None
-        if snapshot and not CONF.use_scheduler_creating_share_from_snapshot:
-            # Shares from snapshots with restriction - source host only.
-            # It is common situation for different types of backends.
-            host = snapshot['share']['instance']['host']
-
-        elif share_group:
+        snapshot_host = None
+        if snapshot:
+            snapshot_host = snapshot['share']['instance']['host']
+            if not CONF.use_scheduler_creating_share_from_snapshot:
+                # Shares from snapshots with restriction - source host only.
+                # It is common situation for different types of backends.
+                host = snapshot['share']['instance']['host']
+
+        if share_group and host is None:
             host = share_group['host']
 
         self.create_instance(
             context, share, share_network_id=share_network_id, host=host,
             availability_zone=availability_zone, share_group=share_group,
             share_group_snapshot_member=share_group_snapshot_member,
-            share_type_id=share_type_id, availability_zones=availability_zones)
+            share_type_id=share_type_id, availability_zones=availability_zones,
+            snapshot_host=snapshot_host)
 
         # Retrieve the share with instance details
         share = self.db.share_get(context, share['id'])
@@ -415,14 +434,16 @@ class API(base.Base):
     def create_instance(self, context, share, share_network_id=None,
                         host=None, availability_zone=None,
                         share_group=None, share_group_snapshot_member=None,
-                        share_type_id=None, availability_zones=None):
+                        share_type_id=None, availability_zones=None,
+                        snapshot_host=None):
         request_spec, share_instance = (
             self.create_share_instance_and_get_request_spec(
                 context, share, availability_zone=availability_zone,
                 share_group=share_group, host=host,
                 share_network_id=share_network_id,
                 share_type_id=share_type_id,
-                availability_zones=availability_zones))
+                availability_zones=availability_zones,
+                snapshot_host=snapshot_host))
 
         if share_group_snapshot_member:
             # Inherit properties from the share_group_snapshot_member
@@ -461,7 +482,7 @@ class API(base.Base):
             self, context, share, availability_zone=None,
             share_group=None, host=None, share_network_id=None,
             share_type_id=None, cast_rules_to_readonly=False,
-            availability_zones=None):
+            availability_zones=None, snapshot_host=None):
 
         availability_zone_id = None
         if availability_zone:
@@ -528,6 +549,7 @@ class API(base.Base):
             'share_proto': share['share_proto'],
             'share_id': share['id'],
             'snapshot_id': share['snapshot_id'],
+            'snapshot_host': snapshot_host,
             'share_type': share_type,
             'share_group': share_group,
             'availability_zone_id': availability_zone_id,

manila/share/driver.py

@@ -24,6 +24,7 @@ import time
 from oslo_config import cfg
 from oslo_log import log
 
+from manila.common import constants
 from manila import exception
 from manila.i18n import _
 from manila import network
@@ -649,8 +650,46 @@ class ShareDriver(object):
         raise NotImplementedError()
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
-        """Is called to create share from snapshot."""
+                                   share_server=None, parent_share=None):
+        """Is called to create share from snapshot.
+
+        Creating a share from snapshot can take longer than a simple clone
+        operation if data copy is required from one host to another. For this
+        reason driver will be able complete this creation asynchronously, by
+        providing a 'creating_from_snapshot' status in the model update.
+
+        When answering asynchronously, drivers must implement the call
+        'get_share_status' in order to provide updates for shares with
+        'creating_from_snapshot' status.
+
+        It is expected that the driver returns a model update to the share
+        manager that contains: share status and a list of export_locations.
+        A list of 'export_locations' is mandatory only for share in 'available'
+        status.
+        The current supported status are 'available' and
+        'creating_from_snapshot'.
+
+        :param context: Current context
+        :param share: Share instance model with share data.
+        :param snapshot: Snapshot instance model .
+        :param share_server: Share server model or None.
+        :param parent_share: Share model from parent snapshot with share data
+            and share server model.
+
+        :returns: a dictionary of updates containing current share status and
+            its export_location (if available).
+
+            Example::
+
+                {
+                    'status': 'available',
+                    'export_locations': [{...}, {...}],
+                }
+
+        :raises: ShareBackendException.
+            A ShareBackendException in this method will set the instance to
+            'error' and the operation will end.
+        """
         raise NotImplementedError()
 
     def create_snapshot(self, context, snapshot, share_server=None):
@@ -1311,6 +1350,15 @@ class ShareDriver(object):
             share_server=None):
         """Create a share group from a share group snapshot.
 
+        When creating a share from snapshot operation takes longer than a
+        simple clone operation, drivers will be able to complete this creation
+        asynchronously, by providing a 'creating_from_snapshot' status in the
+        returned model update. The current supported status are 'available' and
+        'creating_from_snapshot'.
+
+        In order to provide updates for shares with 'creating_from_snapshot'
+        status, drivers must implement the call 'get_share_status'.
+
         :param context:
         :param share_group_dict: The share group details
             EXAMPLE:
@@ -1372,12 +1420,27 @@ class ShareDriver(object):
 
             share_update_list - a list of dictionaries containing dicts for
             every share created in the share group. Any share dicts should at a
-            minimum contain the 'id' key and 'export_locations'.
+            minimum contain the 'id' key and, for synchronous creation, the
+            'export_locations'. For asynchronous share creation this dict
+            must also contain the key 'status' with the value set to
+            'creating_from_snapshot'. The current supported status are
+            'available' and 'creating_from_snapshot'.
             Export locations should be in the same format as returned by
             a share_create. This list may be empty or None. EXAMPLE:
             .. code::
 
-                [{'id': 'uuid', 'export_locations': [{...}, {...}]}]
+                [
+                    {
+                    'id': 'uuid',
+                    'export_locations': [{...}, {...}],
+                    },
+                    {
+                    'id': 'uuid',
+                    'export_locations': [],
+                    'status': 'creating_from_snapshot',
+                    },
+                ]
+
         """
         # Ensure that the share group snapshot has members
         if not share_group_snapshot_dict['share_group_snapshot_members']:
@@ -1389,18 +1452,38 @@ class ShareDriver(object):
 
         LOG.debug('Creating share group from group snapshot %s.',
                   share_group_snapshot_dict['id'])
-
         for clone in clone_list:
             kwargs = {}
+            share_update_info = {}
             if self.driver_handles_share_servers:
                 kwargs['share_server'] = share_server
-            export_locations = (
+            model_update = (
                 self.create_share_from_snapshot(
                     context, clone['share'], clone['snapshot'], **kwargs))
-            share_update_list.append({
+            if isinstance(model_update, dict):
+                status = model_update.get('status')
+                # NOTE(dviroel): share status is mandatory when answering
+                # a model update. If not provided, won't be possible to
+                # determine if was successfully created.
+                if status is None:
+                    msg = _("Driver didn't provide a share status.")
+                    raise exception.InvalidShareInstance(reason=msg)
+                if status not in [constants.STATUS_AVAILABLE,
+                                  constants.STATUS_CREATING_FROM_SNAPSHOT]:
+                    msg = _('Driver returned an invalid status: %s') % status
+                    raise exception.InvalidShareInstance(reason=msg)
+                share_update_info.update({'status': status})
+                export_locations = model_update.get('export_locations', [])
+            else:
+                # NOTE(dviroel): the driver that doesn't implement the new
+                # model_update will return only the export locations
+                export_locations = model_update
+
+            share_update_info.update({
                 'id': clone['share']['id'],
                 'export_locations': export_locations,
             })
+            share_update_list.append(share_update_info)
         return None, share_update_list
 
     def delete_share_group(self, context, share_group_dict, share_server=None):
@@ -2742,3 +2825,31 @@ class ShareDriver(object):
             this share server.
         """
         raise NotImplementedError()
+
+    def get_share_status(self, share, share_server=None):
+        """Invoked periodically to get the current status of a given share.
+
+        Driver can use this method to update the status of a share that is
+        still pending from other operations.
+        This method is expected to be called in a periodic interval set by the
+        'periodic_interval' configuration in seconds.
+
+        :param share: share to get updated status from.
+        :param share_server: share server model or None.
+        :returns: a dictionary of updates with the current share status, that
+            must be 'available', 'creating_from_snapshot' or 'error', a list of
+            export locations, if available, and a progress field which
+            indicates the completion of the share creation operation.
+            EXAMPLE::
+
+                {
+                    'status': 'available',
+                    'export_locations': [{...}, {...}],
+                    'progress': '50%'
+                }
+
+        :raises: ShareBackendException.
+            A ShareBackendException in this method will set the instance status
+            to 'error'.
+        """
+        raise NotImplementedError()

manila/share/drivers/dell_emc/driver.py

@@ -107,7 +107,7 @@ class EMCShareDriver(driver.ShareDriver):
         return location
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Is called to create share from snapshot."""
         location = self.plugin.create_share_from_snapshot(
             context, share, snapshot, share_server)

manila/share/drivers/dell_emc/plugins/powermax/connection.py

@@ -214,7 +214,7 @@ class PowerMaxStorageConnection(driver.StorageConnection):
                    'share_name': share_name})
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Create a share from a snapshot - clone a snapshot."""
         share_name = share['id']
 

manila/share/drivers/dell_emc/plugins/unity/connection.py

@@ -223,7 +223,7 @@ class UnityStorageConnection(driver.StorageConnection):
         return locations
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Create a share from a snapshot - clone a snapshot."""
         share_name = share['id']
 

manila/share/drivers/dell_emc/plugins/vnx/connection.py

@@ -211,7 +211,7 @@ class VNXStorageConnection(driver.StorageConnection):
                    'share_name': share_name})
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Create a share from a snapshot - clone a snapshot."""
         share_name = share['id']
 

manila/share/drivers/generic.py

@@ -640,7 +640,7 @@ class GenericShareDriver(driver.ExecuteMixin, driver.ShareDriver):
 
     @ensure_server
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Is called to create share from snapshot."""
         return self._create_share(
             context, share,

manila/share/drivers/glusterfs/layout.py

@@ -235,7 +235,7 @@ class GlusterfsShareLayoutBase(object):
 
     @abc.abstractmethod
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Is called to create share from snapshot."""
 
     @abc.abstractmethod

manila/share/drivers/glusterfs/layout_directory.py

@@ -188,7 +188,7 @@ class GlusterfsDirectoryMappedLayout(layout.GlusterfsShareLayoutBase):
         pass
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         raise NotImplementedError
 
     def create_snapshot(self, context, snapshot, share_server=None):

manila/share/drivers/glusterfs/layout_volume.py

@@ -463,7 +463,7 @@ class GlusterfsVolumeMappedLayout(layout.GlusterfsShareLayoutBase):
         return backend_snapshot_name
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         old_gmgr = self._share_manager(snapshot['share_instance'])
 
         # Snapshot clone feature in GlusterFS server essential to support this

manila/share/drivers/hdfs/hdfs_native.py

@@ -227,7 +227,7 @@ class HDFSNativeShareDriver(driver.ExecuteMixin, driver.ShareDriver):
         return self._get_share_path(share)
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Creates a snapshot."""
         self._create_share(share)
         share_path = '/' + share['name']

manila/share/drivers/hitachi/hnas/driver.py

@@ -448,7 +448,7 @@ class HitachiHNASDriver(driver.ShareDriver):
                  {'id': snapshot['id']})
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         r"""Creates a new share from snapshot.
 
         :param context: The `context.RequestContext` object for the request

manila/share/drivers/hpe/hpe_3par_driver.py

@@ -501,7 +501,7 @@ class HPE3ParShareDriver(driver.ShareDriver):
         return self._hpe3par.build_export_locations(protocol, ips, path)
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Is called to create share from snapshot."""
 
         fpg, vfs, ips = self._get_pool_location(share, share_server)

manila/share/drivers/huawei/base.py

@@ -66,7 +66,7 @@ class HuaweiBase(object):
 
     @abc.abstractmethod
     def create_share_from_snapshot(self, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Create share from snapshot."""
 
     @abc.abstractmethod

manila/share/drivers/huawei/huawei_nas.py

@@ -119,7 +119,7 @@ class HuaweiNasDriver(driver.ShareDriver):
         self.plugin.extend_share(share, new_size, share_server)
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Create a share from snapshot."""
         LOG.debug("Create a share from snapshot %s.", snapshot['snapshot_id'])
         location = self.plugin.create_share_from_snapshot(share, snapshot)

manila/share/drivers/huawei/v3/connection.py

@@ -383,7 +383,7 @@ class V3StorageConnection(driver.HuaweiBase):
         return share
 
     def create_share_from_snapshot(self, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Create a share from snapshot."""
         share_fs_id = self.helper.get_fsid_by_name(snapshot['share_name'])
         if not share_fs_id:

manila/share/drivers/ibm/gpfs.py

@@ -493,7 +493,7 @@ class GPFSShareDriver(driver.ExecuteMixin, driver.GaneshaMixin,
         return location
 
     def create_share_from_snapshot(self, ctx, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Is called to create share from a snapshot."""
         share_path = self._get_share_path(share)
         self._create_share_from_snapshot(share, snapshot, share_path)

manila/share/drivers/infinidat/infinibox.py

@@ -403,7 +403,7 @@ class InfiniboxShareDriver(driver.ShareDriver):
 
     @infinisdk_to_manila_exceptions
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         name = self._make_share_name(share)
         infinidat_snapshot = self._get_infinidat_snapshot(snapshot)
         infinidat_new_share = infinidat_snapshot.create_snapshot(

manila/share/drivers/inspur/as13000/as13000_nas.py

@@ -343,7 +343,7 @@ class AS13000ShareDriver(driver.ShareDriver):
 
     @inspur_driver_debug_trace
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Create a share from snapshot."""
         pool, name, size, proto = self._get_share_instance_pnsp(share)
 

manila/share/drivers/lvm.py

@@ -242,7 +242,7 @@ class LVMShareDriver(LVMMixin, driver.ShareDriver):
         return location
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Is called to create share from snapshot."""
         self._allocate_container(share)
         snapshot_device_name = self._get_local_path(snapshot)

manila/share/drivers/maprfs/maprfs_native.py

@@ -206,7 +206,7 @@ class MapRFSNativeShareDriver(driver.ExecuteMixin, driver.ShareDriver):
             raise exception.ShareResourceNotFound(share_id=share['share_id'])
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Creates a share from snapshot."""
         metadata = self.api.get_share_metadata(context,
                                                {'id': share['share_id']})

manila/share/drivers/netapp/dataontap/cluster_mode/lib_base.py

@@ -485,7 +485,7 @@ class NetAppCmodeFileStorageLibrary(object):
 
     @na_utils.trace
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Creates new share from snapshot."""
         vserver, vserver_client = self._get_vserver(share_server=share_server)
         self._allocate_container_from_snapshot(

manila/share/drivers/nexenta/ns4/nexenta_nas.py

@@ -77,7 +77,7 @@ class NexentaNasDriver(driver.ShareDriver):
         return self.helper.create_filesystem(share)
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Is called to create share from snapshot."""
         LOG.debug('Creating share from snapshot %s.', snapshot['name'])
         return self.helper.create_share_from_snapshot(share, snapshot)

manila/share/drivers/nexenta/ns5/nexenta_nas.py

@@ -189,7 +189,7 @@ class NexentaNasDriver(driver.ShareDriver):
         return '%s:%s' % (self.nas_host, filesystem['mountPoint'])
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Is called to create share from snapshot."""
         snapshot_path = self._get_snapshot_path(snapshot)
         LOG.debug('Creating share from snapshot %s.', snapshot_path)

manila/share/drivers/qnap/qnap.py

@@ -519,7 +519,7 @@ class QnapShareDriver(driver.ShareDriver):
                  retries=5)
     @utils.synchronized('qnap-create_share_from_snapshot')
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Create a share from a snapshot."""
         LOG.debug('Entering create_share_from_snapshot. The source '
                   'snapshot=%(snap)s. The created share=%(share)s',

manila/share/drivers/tegile/tegile.py

@@ -281,7 +281,7 @@ class TegileShareDriver(driver.ShareDriver):
 
     @debugger
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Create a share from a snapshot - clone a snapshot."""
         pool, project, share_name = self._get_pool_project_share_name(share)
 

manila/share/drivers/veritas/veritas_isa.py

@@ -524,7 +524,7 @@ class ACCESSShareDriver(driver.ExecuteMixin, driver.ShareDriver):
                                   json.dumps(data2), 'DELETE')
 
     def create_share_from_snapshot(self, ctx, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """create share from a snapshot."""
         sharename = snapshot['share_name']
         va_sharename = self._get_va_share_name(sharename)

manila/share/drivers/zfsonlinux/driver.py

@@ -580,7 +580,7 @@ class ZFSonLinuxShareDriver(zfs_utils.ExecuteMixin, driver.ShareDriver):
 
     @ensure_share_server_not_provided
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Is called to create a share from snapshot."""
         dataset_name = self._get_dataset_name(share)
         ssh_cmd = '%(username)s@%(host)s' % {

manila/share/drivers/zfssa/zfssashare.py

@@ -242,7 +242,7 @@ class ZFSSAShareDriver(driver.ShareDriver):
                                    snapshot['id'])
 
     def create_share_from_snapshot(self, context, share, snapshot,
-                                   share_server=None):
+                                   share_server=None, parent_share=None):
         """Create a share from a snapshot - clone a snapshot."""
         lcfg = self.configuration
         LOG.debug("ZFSSAShareDriver.create_share_from_snapshot: clone=%s",

manila/share/manager.py

@@ -403,7 +403,7 @@ class ShareManager(manager.SchedulerDependentManager):
             self._ensure_share_instance_has_pool(ctxt, share_instance)
             share_instance = self.db.share_instance_get(
                 ctxt, share_instance['id'], with_share_data=True)
-            share_instance_dict = self._get_share_replica_dict(
+            share_instance_dict = self._get_share_instance_dict(
                 ctxt, share_instance)
             update_share_instances.append(share_instance_dict)
 
@@ -541,6 +541,7 @@ class ShareManager(manager.SchedulerDependentManager):
             self.db.share_instance_update(context, share_instance['id'],
                                           {'status': constants.STATUS_ERROR})
         parent_share_server = None
+        parent_share_same_dest = False
         if snapshot:
             parent_share_server_id = (
                 snapshot['share']['instance']['share_server_id'])
@@ -562,6 +563,8 @@ class ShareManager(manager.SchedulerDependentManager):
                 raise exception.InvalidShareServer(
                     share_server_id=parent_share_server
                 )
+            parent_share_same_dest = (snapshot['share']['instance']['host']
+                                      == share_instance['host'])
         share_network_subnet_id = None
         if share_network_id:
             share_network_subnet = (
@@ -578,7 +581,7 @@ class ShareManager(manager.SchedulerDependentManager):
                 parent_share_server['share_network_subnet_id'])
 
         def get_available_share_servers():
-            if parent_share_server:
+            if parent_share_server and parent_share_same_dest:
                 return [parent_share_server]
             else:
                 return (
@@ -929,9 +932,9 @@ class ShareManager(manager.SchedulerDependentManager):
                         "All snapshots must have '%(status)s' status to be "
                         "migrated by the driver along with share "
                         "%(share)s.") % {
-                            'share': share_ref['id'],
-                            'status': constants.STATUS_AVAILABLE
-                        }
+                        'share': share_ref['id'],
+                        'status': constants.STATUS_AVAILABLE
+                    }
                     raise exception.ShareMigrationFailed(reason=msg)