Browse Source

Add generic volume groups

This is the second patch that implements the generic-volume-group
bluerpint. It adds the groups table and introduces create/delete/
update/list/show APIs for groups.

It depends on the first patch which adds group types and group specs:
    https://review.openstack.org/#/c/320165/

Client side patch is here:
    https://review.openstack.org/#/c/322627/

Current microversion is 3.13. The following CLI's are supported:
cinder --os-volume-api-version 3.13 group-create --name my_group
    <group type uuid> <volume type uuid>
cinder --os-volume-api-version 3.13 group-list
cinder --os-volume-api-version 3.13 create --group-id <group uuid>
    --volume-type <volume type uuid> <size>
cinder --os-volume-api-version 3.13 group-update <group uuid>
    --name new_name  description new_description
    --add-volumes <uuid of volume to add>
    --remove-volumes <uuid of volume to remove>
cinder --os-volume-api-version 3.13 group-show <group uuid>
cinder --os-volume-api-version 3.13 group-delete
    --delete-volumes <group uuid>

APIImpact
DocImpact
Change-Id: I35157439071786872bc9976741c4ef75698f7cb7
Partial-Implements: blueprint generic-volume-group
changes/59/322459/66
xing-yang 5 years ago
parent
commit
8c74c74695
  1. 3
      cinder/api/openstack/api_version_request.py
  2. 4
      cinder/api/openstack/rest_api_version_history.rst
  3. 232
      cinder/api/v3/groups.py
  4. 12
      cinder/api/v3/router.py
  5. 72
      cinder/api/v3/views/groups.py
  6. 20
      cinder/api/v3/views/volumes.py
  7. 154
      cinder/api/v3/volumes.py
  8. 3
      cinder/common/config.py
  9. 66
      cinder/db/api.py
  10. 291
      cinder/db/sqlalchemy/api.py
  11. 97
      cinder/db/sqlalchemy/migrate_repo/versions/078_add_groups_and_group_volume_type_mapping_table.py
  12. 58
      cinder/db/sqlalchemy/models.py
  13. 9
      cinder/exception.py
  14. 27
      cinder/group/__init__.py
  15. 543
      cinder/group/api.py
  16. 1
      cinder/objects/__init__.py
  17. 2
      cinder/objects/base.py
  18. 18
      cinder/objects/fields.py
  19. 168
      cinder/objects/group.py
  20. 10
      cinder/objects/request_spec.py
  21. 33
      cinder/objects/volume.py
  22. 10
      cinder/objects/volume_type.py
  23. 28
      cinder/quota.py
  24. 19
      cinder/scheduler/driver.py
  25. 223
      cinder/scheduler/filter_scheduler.py
  26. 29
      cinder/scheduler/manager.py
  27. 24
      cinder/scheduler/rpcapi.py
  28. 806
      cinder/tests/unit/api/v3/test_groups.py
  29. 184
      cinder/tests/unit/api/v3/test_volumes.py
  30. 4
      cinder/tests/unit/fake_constants.py
  31. 0
      cinder/tests/unit/group/__init__.py
  32. 176
      cinder/tests/unit/group/test_groups.py
  33. 207
      cinder/tests/unit/objects/test_group.py
  34. 15
      cinder/tests/unit/objects/test_objects.py
  35. 3
      cinder/tests/unit/objects/test_volume.py
  36. 6
      cinder/tests/unit/policy.json
  37. 83
      cinder/tests/unit/scheduler/test_filter_scheduler.py
  38. 13
      cinder/tests/unit/scheduler/test_rpcapi.py
  39. 62
      cinder/tests/unit/test_migrations.py
  40. 84
      cinder/tests/unit/test_volume_rpcapi.py
  41. 35
      cinder/tests/unit/utils.py
  42. 51
      cinder/tests/unit/volume/flows/test_create_volume_flow.py
  43. 33
      cinder/volume/api.py
  44. 97
      cinder/volume/driver.py
  45. 41
      cinder/volume/flows/api/create_volume.py
  46. 394
      cinder/volume/manager.py
  47. 21
      cinder/volume/rpcapi.py
  48. 30
      cinder/volume/utils.py
  49. 6
      cinder/volume/volume_types.py
  50. 6
      etc/cinder/policy.json
  51. 4
      releasenotes/notes/generic-volume-groups-69f998ce44f42737.yaml
  52. 7
      tools/lintstack.py

3
cinder/api/openstack/api_version_request.py

@ -60,6 +60,7 @@ REST_API_VERSION_HISTORY = """
* 3.10 - Add group_id filter to list/detail volumes in _get_volumes.
* 3.11 - Add group types and group specs API.
* 3.12 - Add volumes summary API.
* 3.13 - Add generic volume groups API.
"""
@ -68,7 +69,7 @@ REST_API_VERSION_HISTORY = """
# minimum version of the API supported.
# Explicitly using /v1 or /v2 enpoints will still work
_MIN_API_VERSION = "3.0"
_MAX_API_VERSION = "3.12"
_MAX_API_VERSION = "3.13"
_LEGACY_API_VERSION1 = "1.0"
_LEGACY_API_VERSION2 = "2.0"

4
cinder/api/openstack/rest_api_version_history.rst

@ -178,3 +178,7 @@ user documentation.
3.12
----
Added volumes/summary API.
3.13
----
Added create/delete/update/list/show APIs for generic volume groups.

232
cinder/api/v3/groups.py

@ -0,0 +1,232 @@
# Copyright (c) 2016 EMC Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""The groups controller."""
from oslo_log import log as logging
from oslo_utils import strutils
import webob
from webob import exc
from cinder.api import common
from cinder.api.openstack import wsgi
from cinder.api.v3.views import groups as views_groups
from cinder import exception
from cinder import group as group_api
from cinder.i18n import _, _LI
LOG = logging.getLogger(__name__)
GROUP_API_VERSION = '3.13'
class GroupsController(wsgi.Controller):
"""The groups API controller for the OpenStack API."""
_view_builder_class = views_groups.ViewBuilder
def __init__(self):
self.group_api = group_api.API()
super(GroupsController, self).__init__()
@wsgi.Controller.api_version(GROUP_API_VERSION)
def show(self, req, id):
"""Return data about the given group."""
LOG.debug('show called for member %s', id)
context = req.environ['cinder.context']
# Not found exception will be handled at the wsgi level
group = self.group_api.get(
context,
group_id=id)
return self._view_builder.detail(req, group)
@wsgi.Controller.api_version(GROUP_API_VERSION)
@wsgi.action("delete")
def delete_group(self, req, id, body):
return self._delete(req, id, body)
def _delete(self, req, id, body):
"""Delete a group."""
LOG.debug('delete called for group %s', id)
context = req.environ['cinder.context']
del_vol = False
if body:
if not self.is_valid_body(body, 'delete'):
msg = _("Missing required element 'delete' in "
"request body.")
raise exc.HTTPBadRequest(explanation=msg)
grp_body = body['delete']
try:
del_vol = strutils.bool_from_string(
grp_body.get('delete-volumes', False),
strict=True)
except ValueError:
msg = (_("Invalid value '%s' for delete-volumes flag.")
% del_vol)
raise exc.HTTPBadRequest(explanation=msg)
LOG.info(_LI('Delete group with id: %s'), id,
context=context)
try:
group = self.group_api.get(context, id)
self.group_api.delete(context, group, del_vol)
except exception.GroupNotFound:
# Not found exception will be handled at the wsgi level
raise
except exception.InvalidGroup as error:
raise exc.HTTPBadRequest(explanation=error.msg)
return webob.Response(status_int=202)
@wsgi.Controller.api_version(GROUP_API_VERSION)
def index(self, req):
"""Returns a summary list of groups."""
return self._get_groups(req, is_detail=False)
@wsgi.Controller.api_version(GROUP_API_VERSION)
def detail(self, req):
"""Returns a detailed list of groups."""
return self._get_groups(req, is_detail=True)
def _get_groups(self, req, is_detail):
"""Returns a list of groups through view builder."""
context = req.environ['cinder.context']
filters = req.params.copy()
marker, limit, offset = common.get_pagination_params(filters)
sort_keys, sort_dirs = common.get_sort_params(filters)
groups = self.group_api.get_all(
context, filters=filters, marker=marker, limit=limit,
offset=offset, sort_keys=sort_keys, sort_dirs=sort_dirs)
if is_detail:
groups = self._view_builder.detail_list(
req, groups)
else:
groups = self._view_builder.summary_list(
req, groups)
return groups
@wsgi.Controller.api_version(GROUP_API_VERSION)
@wsgi.response(202)
def create(self, req, body):
"""Create a new group."""
LOG.debug('Creating new group %s', body)
self.assert_valid_body(body, 'group')
context = req.environ['cinder.context']
group = body['group']
self.validate_name_and_description(group)
name = group.get('name')
description = group.get('description')
group_type = group.get('group_type')
if not group_type:
msg = _("group_type must be provided to create "
"group %(name)s.") % {'name': name}
raise exc.HTTPBadRequest(explanation=msg)
volume_types = group.get('volume_types')
if not volume_types:
msg = _("volume_types must be provided to create "
"group %(name)s.") % {'name': name}
raise exc.HTTPBadRequest(explanation=msg)
availability_zone = group.get('availability_zone')
LOG.info(_LI("Creating group %(name)s."),
{'name': name},
context=context)
try:
new_group = self.group_api.create(
context, name, description, group_type, volume_types,
availability_zone=availability_zone)
except (exception.Invalid, exception.ObjectActionError) as error:
raise exc.HTTPBadRequest(explanation=error.msg)
except exception.NotFound:
# Not found exception will be handled at the wsgi level
raise
retval = self._view_builder.summary(req, new_group)
return retval
@wsgi.Controller.api_version(GROUP_API_VERSION)
def update(self, req, id, body):
"""Update the group.
Expected format of the input parameter 'body':
.. code-block:: json
{
"group":
{
"name": "my_group",
"description": "My group",
"add_volumes": "volume-uuid-1,volume-uuid-2,...",
"remove_volumes": "volume-uuid-8,volume-uuid-9,..."
}
}
"""
LOG.debug('Update called for group %s.', id)
if not body:
msg = _("Missing request body.")
raise exc.HTTPBadRequest(explanation=msg)
self.assert_valid_body(body, 'group')
context = req.environ['cinder.context']
group = body.get('group')
self.validate_name_and_description(group)
name = group.get('name')
description = group.get('description')
add_volumes = group.get('add_volumes')
remove_volumes = group.get('remove_volumes')
# Allow name or description to be changed to an empty string ''.
if (name is None and description is None and not add_volumes
and not remove_volumes):
msg = _("Name, description, add_volumes, and remove_volumes "
"can not be all empty in the request body.")
raise exc.HTTPBadRequest(explanation=msg)
LOG.info(_LI("Updating group %(id)s with name %(name)s "
"description: %(description)s add_volumes: "
"%(add_volumes)s remove_volumes: %(remove_volumes)s."),
{'id': id, 'name': name,
'description': description,
'add_volumes': add_volumes,
'remove_volumes': remove_volumes},
context=context)
try:
group = self.group_api.get(context, id)
self.group_api.update(
context, group, name, description,
add_volumes, remove_volumes)
except exception.GroupNotFound:
# Not found exception will be handled at the wsgi level
raise
except exception.InvalidGroup as error:
raise exc.HTTPBadRequest(explanation=error.msg)
return webob.Response(status_int=202)
def create_resource():
return wsgi.Resource(GroupsController())

12
cinder/api/v3/router.py

@ -31,6 +31,7 @@ from cinder.api.v3 import clusters
from cinder.api.v3 import consistencygroups
from cinder.api.v3 import group_specs
from cinder.api.v3 import group_types
from cinder.api.v3 import groups
from cinder.api.v3 import messages
from cinder.api.v3 import snapshot_manage
from cinder.api.v3 import volume_manage
@ -82,6 +83,17 @@ class APIRouter(cinder.api.openstack.APIRouter):
parent_resource=dict(member_name='group_type',
collection_name='group_types'))
self.resources['groups'] = groups.create_resource()
mapper.resource("group", "groups",
controller=self.resources['groups'],
collection={'detail': 'GET'},
member={'action': 'POST'})
mapper.connect("groups",
"/{project_id}/groups/{id}/action",
controller=self.resources["groups"],
action="action",
conditions={"action": ["POST"]})
self.resources['snapshots'] = snapshots.create_resource(ext_mgr)
mapper.resource("snapshot", "snapshots",
controller=self.resources['snapshots'],

72
cinder/api/v3/views/groups.py

@ -0,0 +1,72 @@
# Copyright (C) 2016 EMC Corporation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from cinder.api import common
class ViewBuilder(common.ViewBuilder):
"""Model group API responses as a python dictionary."""
_collection_name = "groups"
def __init__(self):
"""Initialize view builder."""
super(ViewBuilder, self).__init__()
def summary_list(self, request, groups):
"""Show a list of groups without many details."""
return self._list_view(self.summary, request, groups)
def detail_list(self, request, groups):
"""Detailed view of a list of groups ."""
return self._list_view(self.detail, request, groups)
def summary(self, request, group):
"""Generic, non-detailed view of a group."""
return {
'group': {
'id': group.id,
'name': group.name
}
}
def detail(self, request, group):
"""Detailed view of a single group."""
return {
'group': {
'id': group.id,
'status': group.status,
'availability_zone': group.availability_zone,
'created_at': group.created_at,
'name': group.name,
'description': group.description,
'group_type': group.group_type_id,
'volume_types': [v_type.id for v_type in group.volume_types],
}
}
def _list_view(self, func, request, groups):
"""Provide a view for a list of groups."""
groups_list = [
func(request, group)['group']
for group in groups]
grp_links = self._get_collection_links(request,
groups,
self._collection_name)
groups_dict = dict(groups=groups_list)
if grp_links:
groups_dict['group_links'] = grp_links
return groups_dict

20
cinder/api/v3/views/volumes.py

@ -1,3 +1,6 @@
# Copyright 2016 EMC Corporation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -10,9 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
from cinder.api.v2.views import volumes as views_v2
class ViewBuilder(object):
"""Model a server API response as a python dictionary."""
class ViewBuilder(views_v2.ViewBuilder):
"""Model a volumes API V3 response as a python dictionary."""
def quick_summary(self, volume_count, volume_size):
"""Number of volumes and size of volumes."""
@ -22,3 +27,14 @@ class ViewBuilder(object):
'total_size': volume_size
},
}
def detail(self, request, volume):
"""Detailed view of a single volume."""
volume_ref = super(ViewBuilder, self).detail(request, volume)
req_version = request.api_version_request
# Add group_id if min version is greater than or equal to 3.13.
if req_version.matches("3.13", None):
volume_ref['volume']['group_id'] = volume.get('group_id')
return volume_ref

154
cinder/api/v3/volumes.py

@ -13,11 +13,21 @@
"""The volumes V3 api."""
from oslo_log import log as logging
from oslo_utils import uuidutils
from webob import exc
from cinder.api import common
from cinder.api.openstack import wsgi
from cinder.api.v2 import volumes as volumes_v2
from cinder.api.v3.views import volumes as volume_views_v3
from cinder import exception
from cinder import group as group_api
from cinder.i18n import _, _LI
from cinder import utils
from cinder.volume import volume_types
LOG = logging.getLogger(__name__)
SUMMARY_BASE_MICRO_VERSION = '3.12'
@ -25,6 +35,12 @@ SUMMARY_BASE_MICRO_VERSION = '3.12'
class VolumeController(volumes_v2.VolumeController):
"""The Volumes API controller for the OpenStack API V3."""
_view_builder_class = volume_views_v3.ViewBuilder
def __init__(self, ext_mgr):
self.group_api = group_api.API()
super(VolumeController, self).__init__(ext_mgr)
def _get_volumes(self, req, is_detail):
"""Returns a list of volumes, transformed through view builder."""
@ -88,6 +104,144 @@ class VolumeController(volumes_v2.VolumeController):
volumes = self.volume_api.get_volume_summary(context, filters=filters)
return view_builder_v3.quick_summary(volumes[0], int(volumes[1]))
@wsgi.response(202)
def create(self, req, body):
"""Creates a new volume.
:param req: the request
:param body: the request body
:returns: dict -- the new volume dictionary
:raises: HTTPNotFound, HTTPBadRequest
"""
self.assert_valid_body(body, 'volume')
LOG.debug('Create volume request body: %s', body)
context = req.environ['cinder.context']
req_version = req.api_version_request
# Remove group_id from body if max version is less than 3.13.
if req_version.matches(None, "3.12"):
# NOTE(xyang): The group_id is from a group created with a
# group_type. So with this group_id, we've got a group_type
# for this volume. Also if group_id is passed in, that means
# we already know which backend is hosting the group and the
# volume will be created on the same backend as well. So it
# won't go through the scheduler again if a group_id is
# passed in.
try:
body.get('volume', {}).pop('group_id', None)
except AttributeError:
msg = (_("Invalid body provided for creating volume. "
"Request API version: %s.") % req_version)
raise exc.HTTPBadRequest(explanation=msg)
volume = body['volume']
kwargs = {}
self.validate_name_and_description(volume)
# NOTE(thingee): v2 API allows name instead of display_name
if 'name' in volume:
volume['display_name'] = volume.pop('name')
# NOTE(thingee): v2 API allows description instead of
# display_description
if 'description' in volume:
volume['display_description'] = volume.pop('description')
if 'image_id' in volume:
volume['imageRef'] = volume.pop('image_id')
req_volume_type = volume.get('volume_type', None)
if req_volume_type:
# Not found exception will be handled at the wsgi level
if not uuidutils.is_uuid_like(req_volume_type):
kwargs['volume_type'] = (
volume_types.get_volume_type_by_name(
context, req_volume_type))
else:
kwargs['volume_type'] = volume_types.get_volume_type(
context, req_volume_type)
kwargs['metadata'] = volume.get('metadata', None)
snapshot_id = volume.get('snapshot_id')
if snapshot_id is not None:
# Not found exception will be handled at the wsgi level
kwargs['snapshot'] = self.volume_api.get_snapshot(context,
snapshot_id)
else:
kwargs['snapshot'] = None
source_volid = volume.get('source_volid')
if source_volid is not None:
# Not found exception will be handled at the wsgi level
kwargs['source_volume'] = (
self.volume_api.get_volume(context,
source_volid))
else:
kwargs['source_volume'] = None
source_replica = volume.get('source_replica')
if source_replica is not None:
# Not found exception will be handled at the wsgi level
src_vol = self.volume_api.get_volume(context,
source_replica)
if src_vol['replication_status'] == 'disabled':
explanation = _('source volume id:%s is not'
' replicated') % source_replica
raise exc.HTTPBadRequest(explanation=explanation)
kwargs['source_replica'] = src_vol
else:
kwargs['source_replica'] = None
consistencygroup_id = volume.get('consistencygroup_id')
if consistencygroup_id is not None:
# Not found exception will be handled at the wsgi level
kwargs['consistencygroup'] = (
self.consistencygroup_api.get(context,
consistencygroup_id))
else:
kwargs['consistencygroup'] = None
# Get group_id if volume is in a group.
group_id = volume.get('group_id')
if group_id is not None:
try:
kwargs['group'] = self.group_api.get(context, group_id)
except exception.GroupNotFound as error:
raise exc.HTTPNotFound(explanation=error.msg)
size = volume.get('size', None)
if size is None and kwargs['snapshot'] is not None:
size = kwargs['snapshot']['volume_size']
elif size is None and kwargs['source_volume'] is not None:
size = kwargs['source_volume']['size']
elif size is None and kwargs['source_replica'] is not None:
size = kwargs['source_replica']['size']
LOG.info(_LI("Create volume of %s GB"), size)
if self.ext_mgr.is_loaded('os-image-create'):
image_ref = volume.get('imageRef')
if image_ref is not None:
image_uuid = self._image_uuid_from_ref(image_ref, context)
kwargs['image_id'] = image_uuid
kwargs['availability_zone'] = volume.get('availability_zone', None)
kwargs['scheduler_hints'] = volume.get('scheduler_hints', None)
multiattach = volume.get('multiattach', False)
kwargs['multiattach'] = multiattach
new_volume = self.volume_api.create(context,
size,
volume.get('display_name'),
volume.get('display_description'),
**kwargs)
retval = self._view_builder.detail(req, new_volume)
return retval
def create_resource(ext_mgr):
return wsgi.Resource(VolumeController(ext_mgr))

3
cinder/common/config.py

@ -190,6 +190,9 @@ global_opts = [
cfg.StrOpt('consistencygroup_api_class',
default='cinder.consistencygroup.api.API',
help='The full class name of the consistencygroup API class'),
cfg.StrOpt('group_api_class',
default='cinder.group.api.API',
help='The full class name of the group API class'),
cfg.StrOpt('os_privileged_user_name',
help='OpenStack privileged account username. Used for requests '
'to other services (such as Nova) that require an account '

66
cinder/db/api.py

@ -269,6 +269,12 @@ def volume_get_all_by_group(context, group_id, filters=None):
return IMPL.volume_get_all_by_group(context, group_id, filters=filters)
def volume_get_all_by_generic_group(context, group_id, filters=None):
"""Get all volumes belonging to a generic volume group."""
return IMPL.volume_get_all_by_generic_group(context, group_id,
filters=filters)
def volume_get_all_by_project(context, project_id, marker, limit,
sort_keys=None, sort_dirs=None, filters=None,
offset=None):
@ -299,6 +305,14 @@ def volume_update(context, volume_id, values):
return IMPL.volume_update(context, volume_id, values)
def volumes_update(context, values_list):
"""Set the given properties on a list of volumes and update them.
Raises NotFound if a volume does not exist.
"""
return IMPL.volumes_update(context, values_list)
def volume_include_in_cluster(context, cluster, partial_rename=True,
**filters):
"""Include all volumes matching the filters into a cluster.
@ -716,6 +730,11 @@ def group_type_access_remove(context, type_id, project_id):
return IMPL.group_type_access_remove(context, type_id, project_id)
def volume_type_get_all_by_group(context, group_id):
"""Get all volumes in a group."""
return IMPL.volume_type_get_all_by_group(context, group_id)
####################
@ -1281,6 +1300,53 @@ def consistencygroup_include_in_cluster(context, cluster, partial_rename=True,
###################
def group_get(context, group_id):
"""Get a group or raise if it does not exist."""
return IMPL.group_get(context, group_id)
def group_get_all(context, filters=None, marker=None, limit=None,
offset=None, sort_keys=None, sort_dirs=None):
"""Get all groups."""
return IMPL.group_get_all(context, filters=filters,
marker=marker, limit=limit,
offset=offset, sort_keys=sort_keys,
sort_dirs=sort_dirs)
def group_create(context, values):
"""Create a group from the values dictionary."""
return IMPL.group_create(context, values)
def group_get_all_by_project(context, project_id, filters=None,
marker=None, limit=None, offset=None,
sort_keys=None, sort_dirs=None):
"""Get all groups belonging to a project."""
return IMPL.group_get_all_by_project(context, project_id,
filters=filters,
marker=marker, limit=limit,
offset=offset,
sort_keys=sort_keys,
sort_dirs=sort_dirs)
def group_update(context, group_id, values):
"""Set the given properties on a group and update it.
Raises NotFound if group does not exist.
"""
return IMPL.group_update(context, group_id, values)
def group_destroy(context, group_id):
"""Destroy the group or raise if it does not exist."""
return IMPL.group_destroy(context, group_id)
###################
def cgsnapshot_get(context, cgsnapshot_id):
"""Get a cgsnapshot or raise if it does not exist."""
return IMPL.cgsnapshot_get(context, cgsnapshot_id)

291
cinder/db/sqlalchemy/api.py

@ -342,6 +342,15 @@ def _sync_consistencygroups(context, project_id, session,
return {key: groups}
def _sync_groups(context, project_id, session,
volume_type_id=None,
volume_type_name=None):
(_junk, groups) = _group_data_get_for_project(
context, project_id, session=session)
key = 'groups'
return {key: groups}
def _sync_backup_gigabytes(context, project_id, session, volume_type_id=None,
volume_type_name=None):
key = 'backup_gigabytes'
@ -356,7 +365,8 @@ QUOTA_SYNC_FUNCTIONS = {
'_sync_gigabytes': _sync_gigabytes,
'_sync_consistencygroups': _sync_consistencygroups,
'_sync_backups': _sync_backups,
'_sync_backup_gigabytes': _sync_backup_gigabytes
'_sync_backup_gigabytes': _sync_backup_gigabytes,
'_sync_groups': _sync_groups,
}
@ -1662,14 +1672,16 @@ def _volume_get_query(context, session=None, project_only=False,
options(joinedload('volume_admin_metadata')).\
options(joinedload('volume_type')).\
options(joinedload('volume_attachment')).\
options(joinedload('consistencygroup'))
options(joinedload('consistencygroup')).\
options(joinedload('group'))
else:
return model_query(context, models.Volume, session=session,
project_only=project_only).\
options(joinedload('volume_metadata')).\
options(joinedload('volume_type')).\
options(joinedload('volume_attachment')).\
options(joinedload('consistencygroup'))
options(joinedload('consistencygroup')).\
options(joinedload('group'))
@require_context
@ -1832,7 +1844,7 @@ def volume_get_all_by_group(context, group_id, filters=None):
"""Retrieves all volumes associated with the group_id.
:param context: context to query under
:param group_id: group ID for all volumes being retrieved
:param group_id: consistency group ID for all volumes being retrieved
:param filters: dictionary of filters; values that are in lists, tuples,
or sets cause an 'IN' operation, while exact matching
is used for other values, see _process_volume_filters
@ -1848,6 +1860,27 @@ def volume_get_all_by_group(context, group_id, filters=None):
return query.all()
@require_context
def volume_get_all_by_generic_group(context, group_id, filters=None):
"""Retrieves all volumes associated with the group_id.
:param context: context to query under
:param group_id: group ID for all volumes being retrieved
:param filters: dictionary of filters; values that are in lists, tuples,
or sets cause an 'IN' operation, while exact matching
is used for other values, see _process_volume_filters
function for more information
:returns: list of matching volumes
"""
query = _volume_get_query(context).filter_by(group_id=group_id)
if filters:
query = _process_volume_filters(query, filters)
# No volumes would match, return empty list
if query is None:
return []
return query.all()
@require_context
def volume_get_all_by_project(context, project_id, marker, limit,
sort_keys=None, sort_dirs=None, filters=None,
@ -2140,6 +2173,38 @@ def volume_update(context, volume_id, values):
return volume_ref
@handle_db_data_error
@require_context
def volumes_update(context, values_list):
session = get_session()
with session.begin():
volume_refs = []
for values in values_list:
volume_id = values['id']
values.pop('id')
metadata = values.get('metadata')
if metadata is not None:
_volume_user_metadata_update(context,
volume_id,
values.pop('metadata'),
delete=True,
session=session)
admin_metadata = values.get('admin_metadata')
if is_admin_context(context) and admin_metadata is not None:
_volume_admin_metadata_update(context,
volume_id,
values.pop('admin_metadata'),
delete=True,
session=session)
volume_ref = _volume_get(context, volume_id, session=session)
volume_ref.update(values)
volume_refs.append(volume_ref)
return volume_refs
@require_context
def volume_attachment_update(context, attachment_id, values):
session = get_session()
@ -3554,7 +3619,12 @@ def volume_type_destroy(context, id):
_volume_type_get(context, id, session)
results = model_query(context, models.Volume, session=session). \
filter_by(volume_type_id=id).all()
if results:
group_count = model_query(context,
models.GroupVolumeTypeMapping,
read_deleted="no",
session=session).\
filter_by(volume_type_id=id).count()
if results or group_count:
LOG.error(_LE('VolumeType %s deletion failed, '
'VolumeType in use.'), id)
raise exception.VolumeTypeInUse(volume_type_id=id)
@ -3618,7 +3688,8 @@ def volume_get_active_by_window(context,
query = (query.options(joinedload('volume_metadata')).
options(joinedload('volume_type')).
options(joinedload('volume_attachment')).
options(joinedload('consistencygroup')))
options(joinedload('consistencygroup')).
options(joinedload('group')))
if is_admin_context(context):
query = query.options(joinedload('volume_admin_metadata'))
@ -3650,6 +3721,29 @@ def group_type_access_get_all(context, type_id):
filter_by(group_type_id=group_type_id).all()
def _group_volume_type_mapping_query(context, session=None):
return model_query(context, models.GroupVolumeTypeMapping, session=session,
read_deleted="no")
@require_admin_context
def volume_type_get_all_by_group(context, group_id):
# Generic volume group
mappings = (_group_volume_type_mapping_query(context).
filter_by(group_id=group_id).all())
session = get_session()
with session.begin():
volume_type_ids = [mapping.volume_type_id for mapping in mappings]
query = (model_query(context,
models.VolumeTypes,
session=session,
read_deleted='no').
filter(models.VolumeTypes.id.in_(volume_type_ids)).
options(joinedload('extra_specs')).
all())
return query
@require_admin_context
def volume_type_access_add(context, type_id, project_id):
"""Add given tenant to the volume type access list."""
@ -5067,6 +5161,188 @@ def consistencygroup_include_in_cluster(context, cluster,
###############################
@require_admin_context
def _group_data_get_for_project(context, project_id,
session=None):
query = model_query(context,
func.count(models.Group.id),
read_deleted="no",
session=session).\
filter_by(project_id=project_id)
result = query.first()
return (0, result[0] or 0)
@require_context
def _group_get(context, group_id, session=None):
result = (model_query(context, models.Group, session=session,
project_only=True).
filter_by(id=group_id).
first())
if not result:
raise exception.GroupNotFound(group_id=group_id)
return result
@require_context
def group_get(context, group_id):
return _group_get(context, group_id)
def _groups_get_query(context, session=None, project_only=False):
return model_query(context, models.Group, session=session,
project_only=project_only)
def _process_groups_filters(query, filters):
if filters:
# Ensure that filters' keys exist on the model
if not is_valid_model_filters(models.Group, filters):
return
query = query.filter_by(**filters)
return query
def _group_get_all(context, filters=None, marker=None, limit=None,
offset=None, sort_keys=None, sort_dirs=None):
if filters and not is_valid_model_filters(models.Group,
filters):
return []
session = get_session()
with session.begin():
# Generate the paginate query
query = _generate_paginate_query(context, session, marker,
limit, sort_keys, sort_dirs, filters,
offset, models.Group)
return query.all()if query else []
@require_admin_context
def group_get_all(context, filters=None, marker=None, limit=None,
offset=None, sort_keys=None, sort_dirs=None):
"""Retrieves all groups.
If no sort parameters are specified then the returned groups are sorted
first by the 'created_at' key and then by the 'id' key in descending
order.
:param context: context to query under
:param marker: the last item of the previous page, used to determine the
next page of results to return
:param limit: maximum number of items to return
:param sort_keys: list of attributes by which results should be sorted,
paired with corresponding item in sort_dirs
:param sort_dirs: list of directions in which results should be sorted,
paired with corresponding item in sort_keys
:param filters: Filters for the query in the form of key/value.
:returns: list of matching groups
"""
return _group_get_all(context, filters, marker, limit, offset,
sort_keys, sort_dirs)
@require_context
def group_get_all_by_project(context, project_id, filters=None,
marker=None, limit=None, offset=None,
sort_keys=None, sort_dirs=None):
"""Retrieves all groups in a project.
If no sort parameters are specified then the returned groups are sorted
first by the 'created_at' key and then by the 'id' key in descending
order.
:param context: context to query under
:param marker: the last item of the previous page, used to determine the
next page of results to return
:param limit: maximum number of items to return
:param sort_keys: list of attributes by which results should be sorted,
paired with corresponding item in sort_dirs
:param sort_dirs: list of directions in which results should be sorted,
paired with corresponding item in sort_keys
:param filters: Filters for the query in the form of key/value.
:returns: list of matching groups
"""
authorize_project_context(context, project_id)
if not filters:
filters = {}
else:
filters = filters.copy()
filters['project_id'] = project_id
return _group_get_all(context, filters, marker, limit, offset,
sort_keys, sort_dirs)
@handle_db_data_error
@require_context
def group_create(context, values):
group = models.Group()
if not values.get('id'):
values['id'] = six.text_type(uuid.uuid4())
mappings = []
for item in values.get('volume_type_ids') or []:
mapping = models.GroupVolumeTypeMapping()
mapping['volume_type_id'] = item
mapping['group_id'] = values['id']
mappings.append(mapping)
values['volume_types'] = mappings
session = get_session()
with session.begin():
group.update(values)
session.add(group)
return _group_get(context, values['id'], session=session)
@handle_db_data_error
@require_context
def group_update(context, group_id, values):
session = get_session()
with session.begin():
result = (model_query(context, models.Group,
project_only=True).
filter_by(id=group_id).
first())
if not result:
raise exception.GroupNotFound(
_("No group with id %s") % group_id)
result.update(values)
result.save(session=session)
return result
@require_admin_context
def group_destroy(context, group_id):
session = get_session()
with session.begin():
(model_query(context, models.Group, session=session).
filter_by(id=group_id).
update({'status': fields.GroupStatus.DELETED,
'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at')}))
(session.query(models.GroupVolumeTypeMapping).
filter_by(group_id=group_id).
update({'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at')}))
###############################
@require_context
def _cgsnapshot_get(context, cgsnapshot_id, session=None):
result = model_query(context, models.Cgsnapshot, session=session,
@ -5436,6 +5712,9 @@ PAGINATION_HELPERS = {
_message_get),
models.GroupTypes: (_group_type_get_query, _process_group_types_filters,
_group_type_get_db_object),
models.Group: (_groups_get_query,
_process_groups_filters,
_group_get),
}

97
cinder/db/sqlalchemy/migrate_repo/versions/078_add_groups_and_group_volume_type_mapping_table.py

@ -0,0 +1,97 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from sqlalchemy import Boolean, Column, DateTime, Integer
from sqlalchemy import ForeignKey, MetaData, String, Table
# Default number of quota groups. We should not read from config file.
DEFAULT_QUOTA_GROUPS = 10
CLASS_NAME = 'default'
CREATED_AT = datetime.datetime.now() # noqa
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
# New table
groups = Table(
'groups',
meta,
Column('created_at', DateTime(timezone=False)),
Column('updated_at', DateTime(timezone=False)),
Column('deleted_at', DateTime(timezone=False)),
Column('deleted', Boolean),
Column('id', String(36), primary_key=True, nullable=False),
Column('user_id', String(length=255)),
Column('project_id', String(length=255)),
Column('cluster_name', String(255)),
Column('host', String(length=255)),
Column('availability_zone', String(length=255)),
Column('name', String(length=255)),
Column('description', String(length=255)),
Column('group_type_id', String(length=36)),
Column('status', String(length=255)),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
groups.create()
# Add column to volumes table
volumes = Table('volumes', meta, autoload=True)
group_id = Column('group_id', String(36),
ForeignKey('groups.id'))
volumes.create_column(group_id)
volumes.update().values(group_id=None).execute()
# New group_volume_type_mapping table
Table('volume_types', meta, autoload=True)
grp_vt_mapping = Table(
'group_volume_type_mapping', meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('deleted_at', DateTime),
Column('deleted', Boolean),
Column('id', Integer, primary_key=True, nullable=False),
Column('volume_type_id', String(36), ForeignKey('volume_types.id'),
nullable=False),
Column('group_id', String(36),
ForeignKey('groups.id'), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
grp_vt_mapping.create()
# Add group quota data into DB.
quota_classes = Table('quota_classes', meta, autoload=True)
rows = (quota_classes.count().
where(quota_classes.c.resource == 'groups').
execute().scalar())
# Do not add entries if there are already 'groups' entries.
if rows:
return
# Set groups
qci = quota_classes.insert()
qci.execute({'created_at': CREATED_AT,
'class_name': CLASS_NAME,
'resource': 'groups',
'hard_limit': DEFAULT_QUOTA_GROUPS,
'deleted': False, })

58
cinder/db/sqlalchemy/models.py

@ -171,6 +171,23 @@ class ConsistencyGroup(BASE, CinderBase):
source_cgid = Column(String(36))
class Group(BASE, CinderBase):
"""Represents a generic volume group."""
__tablename__ = 'groups'
id = Column(String(36), primary_key=True)
user_id = Column(String(255), nullable=False)
project_id = Column(String(255), nullable=False)
cluster_name = Column(String(255))
host = Column(String(255))
availability_zone = Column(String(255))
name = Column(String(255))
description = Column(String(255))
status = Column(String(255))
group_type_id = Column(String(36))
class Cgsnapshot(BASE, CinderBase):
"""Represents a cgsnapshot."""
__tablename__ = 'cgsnapshots'
@ -240,6 +257,7 @@ class Volume(BASE, CinderBase):
encryption_key_id = Column(String(36))
consistencygroup_id = Column(String(36))
group_id = Column(String(36))
bootable = Column(Boolean, default=False)
multiattach = Column(Boolean, default=False)
@ -256,6 +274,12 @@ class Volume(BASE, CinderBase):
foreign_keys=consistencygroup_id,
primaryjoin='Volume.consistencygroup_id == ConsistencyGroup.id')
group = relationship(
Group,
backref="volumes",
foreign_keys=group_id,
primaryjoin='Volume.group_id == Group.id')
class VolumeMetadata(BASE, CinderBase):
"""Represents a metadata key/value pair for a volume."""
@ -330,13 +354,33 @@ class GroupTypes(BASE, CinderBase):
name = Column(String(255))
description = Column(String(255))
is_public = Column(Boolean, default=True)
# TODO(xyang): Uncomment the following after groups table is added.
# groups = relationship(Group,
# backref=backref('group_type', uselist=False),
# foreign_keys=id,
# primaryjoin='and_('
# 'Group.group_type_id == GroupTypes.id, '
# 'GroupTypes.deleted == False)')
groups = relationship(Group,
backref=backref('group_type', uselist=False),
foreign_keys=id,
primaryjoin='and_('
'Group.group_type_id == GroupTypes.id, '
'GroupTypes.deleted == False)')
class GroupVolumeTypeMapping(BASE, CinderBase):
"""Represent mapping between groups and volume_types."""
__tablename__ = "group_volume_type_mapping"
id = Column(Integer, primary_key=True, nullable=False)
volume_type_id = Column(String(36),
ForeignKey('volume_types.id'),
nullable=False)
group_id = Column(String(36),
ForeignKey('groups.id'),
nullable=False)
group = relationship(
Group,
backref="volume_types",
foreign_keys=group_id,
primaryjoin='and_('
'GroupVolumeTypeMapping.group_id == Group.id,'
'GroupVolumeTypeMapping.deleted == False)'
)
class VolumeTypeProjects(BASE, CinderBase):

9
cinder/exception.py

@ -1044,6 +1044,15 @@ class InvalidConsistencyGroup(Invalid):
message = _("Invalid ConsistencyGroup: %(reason)s")
# Group
class GroupNotFound(NotFound):
message = _("Group %(group_id)s could not be found.")
class InvalidGroup(Invalid):
message = _("Invalid Group: %(reason)s")
# CgSnapshot
class CgSnapshotNotFound(NotFound):
message = _("CgSnapshot %(cgsnapshot_id)s could not be found.")

27
cinder/group/__init__.py

@ -0,0 +1,27 @@
# Copyright (C) 2016 EMC Corporation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Importing full names to not pollute the namespace and cause possible
# collisions with use of 'from cinder.transfer import <foo>' elsewhere.
from oslo_utils import importutils
from cinder.common import config
CONF = config.CONF
API = importutils.import_class(
CONF.group_api_class)

543
cinder/group/api.py

@ -0,0 +1,543 @@
# Copyright (C) 2016 EMC Corporation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Handles all requests relating to groups.
"""
import functools
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import excutils
from oslo_utils import timeutils
from cinder.db import base
from cinder import exception
from cinder.i18n import _, _LE, _LW
from cinder import objects
from cinder.objects import base as objects_base
from cinder.objects import fields as c_fields
import cinder.policy
from cinder import quota
from cinder.scheduler import rpcapi as scheduler_rpcapi
from cinder.volume import api as volume_api
from cinder.volume import rpcapi as volume_rpcapi
from cinder.volume import utils as vol_utils
from cinder.volume import volume_types
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
GROUP_QUOTAS = quota.GROUP_QUOTAS
VALID_REMOVE_VOL_FROM_GROUP_STATUS = (
'available',
'in-use',
'error',
'error_deleting')
VALID_ADD_VOL_TO_GROUP_STATUS = (
'available',
'in-use')
def wrap_check_policy(func):
"""Check policy corresponding to the wrapped methods prior to execution.
This decorator requires the first 3 args of the wrapped function
to be (self, context, group)
"""
@functools.wraps(func)
def wrapped(self, context, target_obj, *args, **kwargs):
check_policy(context, func.__name__, target_obj)
return func(self, context, target_obj, *args, **kwargs)
return wrapped
def check_policy(context, action, target_obj=None):
target = {
'project_id': context.project_id,
'user_id': context.user_id,
}
if isinstance(target_obj, objects_base.CinderObject):
# Turn object into dict so target.update can work
target.update(
target_obj.obj_to_primitive()['versioned_object.data'] or {})
else:
target.update(target_obj or {})
_action = 'group:%s' % action
cinder.policy.enforce(context, _action, target)
class API(base.Base):
"""API for interacting with the volume manager for groups."""
def __init__(self, db_driver=None):
self.scheduler_rpcapi = scheduler_rpcapi.SchedulerAPI()
self.volume_rpcapi = volume_rpcapi.VolumeAPI()
self.volume_api = volume_api.API()
super(API, self).__init__(db_driver)
def _extract_availability_zone(self, availability_zone):
raw_zones = self.volume_api.list_availability_zones(enable_cache=True)
availability_zones = set([az['name'] for az in raw_zones])
if CONF.storage_availability_zone:
availability_zones.add(CONF.storage_availability_zone)
if availability_zone is None:
if CONF.default_availability_zone:
availability_zone = CONF.default_availability_zone
else:
# For backwards compatibility use the storage_availability_zone
availability_zone = CONF.storage_availability_zone
if availability_zone not in availability_zones:
if CONF.allow_availability_zone_fallback:
original_az = availability_zone
availability_zone = (
CONF.default_availability_zone or
CONF.storage_availability_zone)
LOG.warning(_LW("Availability zone '%(s_az)s' "
"not found, falling back to "
"'%(s_fallback_az)s'."),
{'s_az': original_az,
's_fallback_az': availability_zone})
else:
msg = _("Availability zone '%(s_az)s' is invalid.")
msg = msg % {'s_az': availability_zone}
raise exception.InvalidInput(reason=msg)
return availability_zone
def create(self, context, name, description, group_type,
volume_types, availability_zone=None):
check_policy(context, 'create')
req_volume_types = []
# NOTE: Admin context is required to get extra_specs of volume_types.
req_volume_types = (self.db.volume_types_get_by_name_or_id(
context.elevated(), volume_types))
req_group_type = self.db.group_type_get(context, group_type)
availability_zone = self._extract_availability_zone(availability_zone)
kwargs = {'user_id': context.user_id,
'project_id': context.project_id,
'availability_zone': availability_zone,
'status': c_fields.GroupStatus.CREATING,
'name': name,
'description': description,
'volume_type_ids': volume_types,
'group_type_id': group_type}
group = None
try:
group = objects.Group(context=context, **kwargs)