openstack
/
manila
Code Issues Proposed changes

Add new quota for share replicas 

This patch adds new quotas for share replicas and replica sizes.
This quotas can be related to either tenants and users or tenants
and share types. Now, when creating a share replica, manila will
check if there are resources available for that specific request.

Partially-Implements: bp limit-share-replicas-per-share
Change-Id: I8ba7bc6f167c28d6c169b2187d0e1bda7cad3f69
This commit is contained in:
silvacarloss 2020-02-21 11:28:04 +00:00
parent 1edd0c39a6
commit dceced6d6e
24 changed files with 932 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
manila/api/openstack/api_version_request.py
View File

@ -143,13 +143,14 @@ REST_API_VERSION_HISTORY = """
* 2.52 - Added 'created_before' and 'created_since' field to list messages
filters, support querying user messages within the specified time
period.
* 2.53 - Added quota control to share replicas.
"""
# The minimum and maximum versions of the API supported
# The default api version request is defined to be the
# minimum version of the API supported.
_MIN_API_VERSION = "2.0"
_MAX_API_VERSION = "2.52"
_MAX_API_VERSION = "2.53"
DEFAULT_API_VERSION = _MIN_API_VERSION

4
manila/api/openstack/rest_api_version_history.rst
View File

@ -292,3 +292,7 @@ user documentation.
----
Added 'created_before' and 'created_since' field to list messages api,
support querying user messages within the specified time period.
2.53
----
Added quota control for share replicas and replica gigabytes.

2
manila/api/v1/limits.py
View File

@ -60,7 +60,7 @@ class LimitsController(wsgi.Controller):
rate_limits = req.environ.get("manila.limits", [])
builder = self._get_view_builder(req)
return builder.build(rate_limits, abs_limits)
return builder.build(req, rate_limits, abs_limits)
def _get_view_builder(self, req):
return limits_views.ViewBuilder()

27
manila/api/v2/quota_sets.py
View File

@ -85,13 +85,15 @@ class QuotaSetsMixin(object):
raise webob.exc.HTTPBadRequest(explanation=msg)
@staticmethod
def _ensure_share_group_related_args_are_absent(body):
def _ensure_specific_microversion_args_are_absent(body, keys,
microversion):
body = body.get('quota_set', body)
for key in ('share_groups', 'share_group_snapshots'):
for key in keys:
if body.get(key):
msg = _("'%(key)s' key is not supported by this microversion. "
"Use 2.40 or greater microversion to be able "
"to use '%(key)s' quotas.") % {"key": key}
msg = (_("'%(key)s' key is not supported by this "
"microversion. Use %(microversion)s or greater "
"microversion to be able to use '%(key)s' quotas.") %
{"key": key, "microversion": microversion})
raise webob.exc.HTTPBadRequest(explanation=msg)
def _get_quotas(self, context, project_id, user_id=None,
@ -148,8 +150,8 @@ class QuotaSetsMixin(object):
body = body.get('quota_set', {})
if share_type and body.get('share_groups',
body.get('share_group_snapshots')):
msg = _("Share type quotas handle only 'shares', 'gigabytes', "
"'snapshots' and 'snapshot_gigabytes' quotas.")
msg = _("Share type quotas cannot constrain share groups and "
"share group snapshots.")
raise webob.exc.HTTPBadRequest(explanation=msg)
try:
@ -282,7 +284,10 @@ class QuotaSetsControllerLegacy(QuotaSetsMixin, wsgi.Controller):
@wsgi.Controller.api_version('1.0', '2.6')
def update(self, req, id, body):
self._ensure_share_type_arg_is_absent(req)
self._ensure_share_group_related_args_are_absent(body)
self._ensure_specific_microversion_args_are_absent(
body, ['share_groups', 'share_group_snapshots'], "2.40")
self._ensure_specific_microversion_args_are_absent(
body, ['share_replicas', 'replica_gigabytes'], "2.53")
return self._update(req, id, body)
@wsgi.Controller.api_version('1.0', '2.6')
@ -319,7 +324,11 @@ class QuotaSetsController(QuotaSetsMixin, wsgi.Controller):
if req.api_version_request < api_version.APIVersionRequest("2.39"):
self._ensure_share_type_arg_is_absent(req)
elif req.api_version_request < api_version.APIVersionRequest("2.40"):
self._ensure_share_group_related_args_are_absent(body)
self._ensure_specific_microversion_args_are_absent(
body, ['share_groups', 'share_group_snapshots'], "2.40")
elif req.api_version_request < api_version.APIVersionRequest("2.53"):
self._ensure_specific_microversion_args_are_absent(
body, ['share_replicas', 'replica_gigabytes'], "2.53")
return self._update(req, id, body)
@wsgi.Controller.api_version('2.7')

25
manila/api/views/limits.py
View File

@ -15,15 +15,21 @@
import datetime
from manila.api import common
from manila import utils
class ViewBuilder(object):
class ViewBuilder(common.ViewBuilder):
"""OpenStack API base limits view builder."""
def build(self, rate_limits, absolute_limits):
_collection_name = "limits"
_detail_version_modifiers = [
"add_share_replica_quotas",
]
def build(self, request, rate_limits, absolute_limits):
rate_limits = self._build_rate_limits(rate_limits)
absolute_limits = self._build_absolute_limits(absolute_limits)
absolute_limits = self._build_absolute_limits(request, absolute_limits)
output = {
"limits": {
@ -34,7 +40,7 @@ class ViewBuilder(object):
return output
def _build_absolute_limits(self, absolute_limits):
def _build_absolute_limits(self, request, absolute_limits):
"""Builder for absolute limits.
absolute_limits should be given as a dict of limits.
@ -58,6 +64,8 @@ class ViewBuilder(object):
},
}
limits = {}
self.update_versioned_resource_dict(request, limit_names,
absolute_limits)
for mapping_key in limit_names.keys():
for k, v in absolute_limits.get(mapping_key, {}).items():
if k in limit_names.get(mapping_key, []) and v is not None:
@ -101,3 +109,12 @@ class ViewBuilder(object):
"unit": rate_limit["unit"],
"next-available": utils.isotime(at=next_avail),
}
@common.ViewBuilder.versioned_method("2.53")
def add_share_replica_quotas(self, request, limit_names, absolute_limits):
limit_names["limit"]["share_replicas"] = ["maxTotalShareReplicas"]
limit_names["limit"]["replica_gigabytes"] = (
["maxTotalReplicaGigabytes"])
limit_names["in_use"]["share_replicas"] = ["totalShareReplicasUsed"]
limit_names["in_use"]["replica_gigabytes"] = (
["totalReplicaGigabytesUsed"])

6
manila/api/views/quota_class_sets.py
View File

@ -21,6 +21,7 @@ class ViewBuilder(common.ViewBuilder):
_collection_name = "quota_class_set"
_detail_version_modifiers = [
"add_share_group_quotas",
"add_share_replica_quotas",
]
def detail_list(self, request, quota_class_set, quota_class=None):
@ -46,3 +47,8 @@ class ViewBuilder(common.ViewBuilder):
view['share_groups'] = share_groups
if share_group_snapshots is not None:
view['share_group_snapshots'] = share_group_snapshots
@common.ViewBuilder.versioned_method("2.53")
def add_share_replica_quotas(self, context, view, quota_class_set):
view['share_replicas'] = quota_class_set.get('share_replicas')
view['replica_gigabytes'] = quota_class_set.get('replica_gigabytes')

6
manila/api/views/quota_sets.py
View File

@ -21,6 +21,7 @@ class ViewBuilder(common.ViewBuilder):
_collection_name = "quota_set"
_detail_version_modifiers = [
"add_share_group_quotas",
"add_share_replica_quotas",
]
def detail_list(self, request, quota_set, project_id=None,
@ -53,3 +54,8 @@ class ViewBuilder(common.ViewBuilder):
view['share_groups'] = share_groups
if share_group_snapshots is not None:
view['share_group_snapshots'] = share_group_snapshots
@common.ViewBuilder.versioned_method("2.53")
def add_share_replica_quotas(self, context, view, quota_class_set):
view['share_replicas'] = quota_class_set.get('share_replicas')
view['replica_gigabytes'] = quota_class_set.get('replica_gigabytes')

6
manila/db/api.py
View File

@ -1305,9 +1305,11 @@ def share_replica_update(context, share_replica_id, values,
with_share_data=with_share_data)
def share_replica_delete(context, share_replica_id):
def share_replica_delete(context, share_replica_id,
need_to_update_usages=True):
"""Deletes a share replica."""
return IMPL.share_replica_delete(context, share_replica_id)
return IMPL.share_replica_delete(
context, share_replica_id, need_to_update_usages=need_to_update_usages)
def purge_deleted_records(context, age_in_days):

126
manila/db/sqlalchemy/api.py
View File

@ -362,6 +362,20 @@ def _sync_share_group_snapshots(context, project_id, user_id, session,
return {'share_group_snapshots': share_group_snapshots_count}
def _sync_share_replicas(context, project_id, user_id, session,
share_type_id=None):
share_replicas_count, _junk = share_replica_data_get_for_project(
context, project_id, user_id, session, share_type_id=share_type_id)
return {'share_replicas': share_replicas_count}
def _sync_replica_gigabytes(context, project_id, user_id, session,
share_type_id=None):
_junk, replica_gigs = share_replica_data_get_for_project(
context, project_id, user_id, session, share_type_id=share_type_id)
return {'replica_gigabytes': replica_gigs}
QUOTA_SYNC_FUNCTIONS = {
'_sync_shares': _sync_shares,
'_sync_snapshots': _sync_snapshots,
@ -370,6 +384,8 @@ QUOTA_SYNC_FUNCTIONS = {
'_sync_share_networks': _sync_share_networks,
'_sync_share_groups': _sync_share_groups,
'_sync_share_group_snapshots': _sync_share_group_snapshots,
'_sync_share_replicas': _sync_share_replicas,
'_sync_replica_gigabytes': _sync_replica_gigabytes,
}
@ -1472,6 +1488,55 @@ def share_instances_get_all(context, filters=None):
return query
@require_context
def _update_share_instance_usages(context, share, instance_ref,
is_replica=False):
deltas = {}
no_instances_remain = len(share.instances) == 0
share_usages_to_release = {"shares": -1, "gigabytes": -share['size']}
replica_usages_to_release = {"share_replicas": -1,
"replica_gigabytes": -share['size']}
if is_replica and no_instances_remain:
# A share that had a replication_type is being deleted, so there's
# need to update the share replica quotas and the share quotas
deltas.update(replica_usages_to_release)
deltas.update(share_usages_to_release)
elif is_replica:
# The user is deleting a share replica
deltas.update(replica_usages_to_release)
else:
# A share with no replication_type is being deleted
deltas.update(share_usages_to_release)
reservations = None
try:
# we give the user_id of the share, to update
# the quota usage for the user, who created the share
reservations = QUOTAS.reserve(
context,
project_id=share['project_id'],
user_id=share['user_id'],
share_type_id=instance_ref['share_type_id'],
**deltas)
QUOTAS.commit(
context, reservations, project_id=share['project_id'],
user_id=share['user_id'],
share_type_id=instance_ref['share_type_id'])
except Exception:
resource_name = (
'share replica' if is_replica else 'share')
resource_id = instance_ref['id'] if is_replica else share['id']
msg = (_("Failed to update usages deleting %(resource_name)s "
"'%(id)s'.") % {'id': resource_id,
"resource_name": resource_name})
LOG.exception(msg)
if reservations:
QUOTAS.rollback(
context, reservations,
share_type_id=instance_ref['share_type_id'])
@require_context
def share_instance_delete(context, instance_id, session=None,
need_to_update_usages=False):
@ -1482,6 +1547,7 @@ def share_instance_delete(context, instance_id, session=None,
share_export_locations_update(context, instance_id, [], delete=True)
instance_ref = share_instance_get(context, instance_id,
session=session)
is_replica = instance_ref['replica_state'] is not None
instance_ref.soft_delete(session=session, update_status=True)
share = share_get(context, instance_ref['share_id'], session=session)
if len(share.instances) == 0:
@ -1490,30 +1556,9 @@ def share_instance_delete(context, instance_id, session=None,
share_id=share['id']).soft_delete()
share.soft_delete(session=session)
if need_to_update_usages:
reservations = None
try:
# we give the user_id of the share, to update
# the quota usage for the user, who created the share
reservations = QUOTAS.reserve(
context,
project_id=share['project_id'],
shares=-1,
gigabytes=-share['size'],
user_id=share['user_id'],
share_type_id=instance_ref['share_type_id'])
QUOTAS.commit(
context, reservations, project_id=share['project_id'],
user_id=share['user_id'],
share_type_id=instance_ref['share_type_id'])
except Exception:
LOG.exception(
"Failed to update usages deleting share '%s'.",
share["id"])
if reservations:
QUOTAS.rollback(
context, reservations,
share_type_id=instance_ref['share_type_id'])
if need_to_update_usages:
_update_share_instance_usages(context, share, instance_ref,
is_replica=is_replica)
def _set_instances_share_data(context, instances, session):
@ -1733,11 +1778,13 @@ def share_replica_update(context, share_replica_id, values,
@require_context
def share_replica_delete(context, share_replica_id, session=None):
def share_replica_delete(context, share_replica_id, session=None,
need_to_update_usages=True):
"""Deletes a share replica."""
session = session or get_session()
share_instance_delete(context, share_replica_id, session=session)
share_instance_delete(context, share_replica_id, session=session,
need_to_update_usages=need_to_update_usages)
################
@ -2558,7 +2605,7 @@ def snapshot_data_get_for_project(context, project_id, user_id,
query = query.filter_by(user_id=user_id)
result = query.first()
return (result[0] or 0, result[1] or 0)
return result[0] or 0, result[1] or 0
@require_context
@ -4680,6 +4727,31 @@ def count_share_group_snapshots(context, project_id, user_id=None,
return query.first()[0]
@require_context
def share_replica_data_get_for_project(context, project_id, user_id=None,
session=None, share_type_id=None):
session = session or get_session()
query = model_query(
context, models.ShareInstance,
func.count(models.ShareInstance.id),
func.sum(models.Share.size),
read_deleted="no",
session=session).join(
models.Share,
models.ShareInstance.share_id == models.Share.id).filter(
models.Share.project_id == project_id).filter(
models.ShareInstance.replica_state.isnot(None))
if share_type_id:
query = query.filter(
models.ShareInstance.share_type_id == share_type_id)
elif user_id:
query = query.filter(models.Share.user_id == user_id)
result = query.first()
return result[0] or 0, result[1] or 0
@require_context
def count_share_group_snapshots_in_share_group(context, share_group_id,
session=None):

11
manila/exception.py
View File

@ -443,6 +443,17 @@ class ShareGroupSnapshotsLimitExceeded(QuotaError):
"Maximum number of allowed share-group-snapshots is exceeded.")
class ShareReplicasLimitExceeded(QuotaError):
message = _(
"Maximum number of allowed share-replicas is exceeded.")
class ShareReplicaSizeExceedsAvailableQuota(QuotaError):
message = _(
"Requested share replica exceeds allowed project/user or share type "
"gigabytes quota.")
class GlusterfsException(ManilaException):
message = _("Unknown Gluster exception.")

10
manila/quota.py
View File

@ -45,6 +45,12 @@ quota_opts = [
cfg.IntOpt('quota_share_networks',
default=10,
help='Number of share-networks allowed per project.'),
cfg.IntOpt('quota_share_replicas',
default=100,
help='Number of share-replicas allowed per project.'),
cfg.IntOpt('quota_replica_gigabytes',
default=1000,
help='Number of replica gigabytes allowed per project.'),
cfg.IntOpt('quota_share_groups',
default=50,
@ -1059,6 +1065,10 @@ resources = [
'quota_share_groups'),
ReservableResource('share_group_snapshots', '_sync_share_group_snapshots',
'quota_share_group_snapshots'),
ReservableResource('share_replicas', '_sync_share_replicas',
'quota_share_replicas'),
ReservableResource('replica_gigabytes', '_sync_replica_gigabytes',
'quota_replica_gigabytes'),
]

227
manila/share/api.py
View File

@ -76,6 +76,78 @@ class API(base.Base):
compatible_azs.append(az['name'])
return compatible_azs
def _check_if_share_quotas_exceeded(self, context, quota_exception,
share_size, operation='create'):
overs = quota_exception.kwargs['overs']
usages = quota_exception.kwargs['usages']
quotas = quota_exception.kwargs['quotas']
def _consumed(name):
return (usages[name]['reserved'] + usages[name]['in_use'])
if 'gigabytes' in overs:
LOG.warning("Quota exceeded for %(s_pid)s, "
"tried to %(operation)s "
"%(s_size)sG share (%(d_consumed)dG of "
"%(d_quota)dG already consumed).", {
's_pid': context.project_id,
's_size': share_size,
'd_consumed': _consumed('gigabytes'),
'd_quota': quotas['gigabytes'],
'operation': operation})
raise exception.ShareSizeExceedsAvailableQuota()
elif 'shares' in overs:
LOG.warning("Quota exceeded for %(s_pid)s, "
"tried to %(operation)s "
"share (%(d_consumed)d shares "
"already consumed).", {
's_pid': context.project_id,
'd_consumed': _consumed('shares'),
'operation': operation})
raise exception.ShareLimitExceeded(allowed=quotas['shares'])
def _check_if_replica_quotas_exceeded(self, context, quota_exception,
replica_size,
resource_type='share_replica'):
overs = quota_exception.kwargs['overs']
usages = quota_exception.kwargs['usages']
quotas = quota_exception.kwargs['quotas']
def _consumed(name):
return (usages[name]['reserved'] + usages[name]['in_use'])
if 'share_replicas' in overs:
LOG.warning("Quota exceeded for %(s_pid)s, "
"unable to create share-replica (%(d_consumed)d "
"of %(d_quota)d already consumed).", {
's_pid': context.project_id,
'd_consumed': _consumed('share_replicas'),
'd_quota': quotas['share_replicas']})
exception_kwargs = {}
if resource_type != 'share_replica':
msg = _("Failed while creating a share with replication "
"support. Maximum number of allowed share-replicas "
"is exceeded.")
exception_kwargs['message'] = msg
raise exception.ShareReplicasLimitExceeded(**exception_kwargs)
elif 'replica_gigabytes' in overs:
LOG.warning("Quota exceeded for %(s_pid)s, "
"unable to create a share replica size of "
"%(s_size)sG (%(d_consumed)dG of "
"%(d_quota)dG already consumed).", {
's_pid': context.project_id,
's_size': replica_size,
'd_consumed': _consumed('replica_gigabytes'),
'd_quota': quotas['replica_gigabytes']})
exception_kwargs = {}
if resource_type != 'share_replica':
msg = _("Failed while creating a share with replication "
"support. Requested share replica exceeds allowed "
"project/user or share type gigabytes quota.")
exception_kwargs['message'] = msg
raise exception.ShareReplicaSizeExceedsAvailableQuota(
**exception_kwargs)
def create(self, context, share_proto, size, name, description,
snapshot_id=None, availability_zone=None, metadata=None,
share_network_id=None, share_type=None, is_public=False,
@ -146,37 +218,23 @@ class API(base.Base):
supported=CONF.enabled_share_protocols))
raise exception.InvalidInput(reason=msg)
deltas = {'shares': 1, 'gigabytes': size}
share_type_attributes = self.get_share_attributes_from_share_type(
share_type)
share_type_supports_replication = share_type_attributes.get(
'replication_type', None)
if share_type_supports_replication:
deltas.update(
{'share_replicas': 1, 'replica_gigabytes': size})
try:
reservations = QUOTAS.reserve(
context, shares=1, gigabytes=size,
share_type_id=share_type_id,
)
context, share_type_id=share_type_id, **deltas)
except exception.OverQuota as e:
overs = e.kwargs['overs']
usages = e.kwargs['usages']
quotas = e.kwargs['quotas']
def _consumed(name):
return (usages[name]['reserved'] + usages[name]['in_use'])
if 'gigabytes' in overs:
LOG.warning("Quota exceeded for %(s_pid)s, "
"tried to create "
"%(s_size)sG share (%(d_consumed)dG of "
"%(d_quota)dG already consumed).", {
's_pid': context.project_id,
's_size': size,
'd_consumed': _consumed('gigabytes'),
'd_quota': quotas['gigabytes']})
raise exception.ShareSizeExceedsAvailableQuota()
elif 'shares' in overs:
LOG.warning("Quota exceeded for %(s_pid)s, "
"tried to create "
"share (%(d_consumed)d shares "
"already consumed).", {
's_pid': context.project_id,
'd_consumed': _consumed('shares')})
raise exception.ShareLimitExceeded(allowed=quotas['shares'])
self._check_if_share_quotas_exceeded(context, e, size)
if share_type_supports_replication:
self._check_if_replica_quotas_exceeded(context, e, size,
resource_type='share')
share_group = None
if share_group_id:
@ -231,7 +289,7 @@ class API(base.Base):
'is_public': is_public,
'share_group_id': share_group_id,
}
options.update(self.get_share_attributes_from_share_type(share_type))
options.update(share_type_attributes)
if share_group_snapshot_member:
options['source_share_group_snapshot_member_id'] = (
@ -513,6 +571,14 @@ class API(base.Base):
'az': availability_zone}
raise exception.InvalidShare(message=msg % payload)
try:
reservations = QUOTAS.reserve(
context, share_replicas=1, replica_gigabytes=share['size'],
share_type_id=share_type['id']
)
except exception.OverQuota as e:
self._check_if_replica_quotas_exceeded(context, e, share['size'])
if share_network_id:
if availability_zone:
try:
@ -552,14 +618,29 @@ class API(base.Base):
cast_rules_to_readonly = True
else:
cast_rules_to_readonly = False
request_spec, share_replica = (
self.create_share_instance_and_get_request_spec(
context, share, availability_zone=availability_zone,
share_network_id=share_network_id,
share_type_id=share['instance']['share_type_id'],
cast_rules_to_readonly=cast_rules_to_readonly,
availability_zones=type_azs)
)
try:
request_spec, share_replica = (
self.create_share_instance_and_get_request_spec(
context, share, availability_zone=availability_zone,
share_network_id=share_network_id,
share_type_id=share['instance']['share_type_id'],
cast_rules_to_readonly=cast_rules_to_readonly,
availability_zones=type_azs)
)
QUOTAS.commit(
context, reservations, project_id=share['project_id'],
share_type_id=share_type['id'],
)
except Exception:
with excutils.save_and_reraise_exception():
try:
self.db.share_replica_delete(
context, share_replica['id'],
need_to_update_usages=False)
finally:
QUOTAS.rollback(
context, reservations, share_type_id=share_type['id'])
all_replicas = self.db.share_replicas_get_all_by_share(
context, share['id'])
@ -1964,34 +2045,64 @@ class API(base.Base):
'size': share['size']})
raise exception.InvalidInput(reason=msg)
replicas = self.db.share_replicas_get_all_by_share(
context, share['id'])
supports_replication = len(replicas) > 0
deltas = {
'project_id': share['project_id'],
'gigabytes': size_increase,
'user_id': share['user_id'],
'share_type_id': share['instance']['share_type_id']
}
# NOTE(carloss): If the share type supports replication, we must get
# all the replicas that pertain to the share and calculate the final
# size (size to increase * amount of replicas), since all the replicas
# are going to be extended when the driver sync them.
if supports_replication:
replica_gigs_to_increase = len(replicas) * size_increase
deltas.update({'replica_gigabytes': replica_gigs_to_increase})
try:
# we give the user_id of the share, to update the quota usage
# for the user, who created the share, because on share delete
# only this quota will be decreased
reservations = QUOTAS.reserve(
context,
project_id=share['project_id'],
gigabytes=size_increase,
user_id=share['user_id'],
share_type_id=share['instance']['share_type_id'])
reservations = QUOTAS.reserve(context, **deltas)
except exception.OverQuota as exc:
usages = exc.kwargs['usages']
quotas = exc.kwargs['quotas']
# Check if the exceeded quota was 'gigabytes'
self._check_if_share_quotas_exceeded(context, exc, share['size'],
operation='extend')
# NOTE(carloss): Check if the exceeded quota is
# 'replica_gigabytes'. If so the failure could be caused due to
# lack of quotas to extend the share's replicas, then the
# '_check_if_replica_quotas_exceeded' method can't be reused here
# since the error message must be different from the default one.
if supports_replication:
overs = exc.kwargs['overs']
usages = exc.kwargs['usages']
quotas = exc.kwargs['quotas']
def _consumed(name):
return usages[name]['reserved'] + usages[name]['in_use']
def _consumed(name):
return (usages[name]['reserved'] + usages[name]['in_use'])
msg = ("Quota exceeded for %(s_pid)s, tried to extend share "
"by %(s_size)sG, (%(d_consumed)dG of %(d_quota)dG "
"already consumed).")
LOG.error(msg, {'s_pid': context.project_id,
's_size': size_increase,
'd_consumed': _consumed('gigabytes'),
'd_quota': quotas['gigabytes']})
raise exception.ShareSizeExceedsAvailableQuota(
requested=size_increase,
consumed=_consumed('gigabytes'),
quota=quotas['gigabytes'])
if 'replica_gigabytes' in overs:
LOG.warning("Replica gigabytes quota exceeded "
"for %(s_pid)s, tried to extend "
"%(s_size)sG share (%(d_consumed)dG of "
"%(d_quota)dG already consumed).", {
's_pid': context.project_id,
's_size': share['size'],
'd_consumed': _consumed(
'replica_gigabytes'),
'd_quota': quotas['replica_gigabytes']})
msg = _("Failed while extending a share with replication "
"support. There is no available quota to extend "
"the share and its %(count)d replicas. Maximum "
"number of allowed replica_gigabytes is "
"exceeded.") % {'count': len(replicas)}
raise exception.ShareReplicaSizeExceedsAvailableQuota(
message=msg)
self.update(context, share, {'status': constants.STATUS_EXTENDING})
self.share_rpcapi.extend_share(context, share, new_size, reservations)

73
manila/share/manager.py
View File

@ -2405,6 +2405,11 @@ class ShareManager(manager.SchedulerDependentManager):
context = context.elevated()
share_ref = self.db.share_get(context, share_id)
share_instance = self._get_share_instance(context, share_ref)
share_type_extra_specs = self._get_extra_specs_from_share_type(
context, share_instance['share_type_id'])
share_type_supports_replication = share_type_extra_specs.get(
'replication_type', None)
project_id = share_ref['project_id']
try:
@ -2430,14 +2435,19 @@ class ShareManager(manager.SchedulerDependentManager):
msg = _("Driver cannot calculate share size.")
raise exception.InvalidShare(reason=msg)
reservations = QUOTAS.reserve(
context,
project_id=project_id,
user_id=context.user_id,
shares=1,
gigabytes=share_update['size'],
share_type_id=share_instance['share_type_id'],
)
deltas = {
'project_id': project_id,
'user_id': context.user_id,
'shares': 1,
'gigabytes': share_update['size'],
'share_type_id': share_instance['share_type_id'],
}
if share_type_supports_replication:
deltas.update({'share_replicas': 1,
'replica_gigabytes': share_update['size']})
reservations = QUOTAS.reserve(context, **deltas)
QUOTAS.commit(
context, reservations, project_id=project_id,
share_type_id=share_instance['share_type_id'],
@ -2570,6 +2580,9 @@ class ShareManager(manager.SchedulerDependentManager):
share_instance = self._get_share_instance(context, share_ref)
share_server = None
project_id = share_ref['project_id']
replicas = self.db.share_replicas_get_all_by_share(
context, share_id)
supports_replication = len(replicas) > 0
def share_manage_set_error_status(msg, exception):
status = {'status': constants.STATUS_UNMANAGE_ERROR}
@ -2590,14 +2603,20 @@ class ShareManager(manager.SchedulerDependentManager):
("Share can not be unmanaged: %s."), e)
return
deltas = {
'project_id': project_id,
'shares': -1,
'gigabytes': -share_ref['size'],
'share_type_id': share_instance['share_type_id'],
}
# NOTE(carloss): while unmanaging a share, a share will not contain
# replicas other than the active one. So there is no need to
# recalculate the amount of share replicas to be deallocated.
if supports_replication:
deltas.update({'share_replicas': -1,
'replica_gigabytes': -share_ref['size']})
try:
reservations = QUOTAS.reserve(
context,
project_id=project_id,
shares=-1,
gigabytes=-share_ref['size'],
share_type_id=share_instance['share_type_id'],
)
reservations = QUOTAS.reserve(context, **deltas)
QUOTAS.commit(
context, reservations, project_id=project_id,
share_type_id=share_instance['share_type_id'],
@ -3885,6 +3904,9 @@ class ShareManager(manager.SchedulerDependentManager):
project_id = share['project_id']
user_id = share['user_id']
new_size = int(new_size)
replicas = self.db.share_replicas_get_all_by_share(
context, share['id'])
supports_replication = len(replicas) > 0
self._notify_about_share_usage(context, share,
share_instance, "shrink.start")
@ -3903,13 +3925,20 @@ class ShareManager(manager.SchedulerDependentManager):
# we give the user_id of the share, to update the quota usage
# for the user, who created the share, because on share delete
# only this quota will be decreased
reservations = QUOTAS.reserve(
context,
project_id=project_id,
user_id=user_id,
share_type_id=share_instance['share_type_id'],
gigabytes=-size_decrease,
)
deltas = {
'project_id': project_id,
'user_id': user_id,
'share_type_id': share_instance['share_type_id'],
'gigabytes': -size_decrease,
}
# NOTE(carloss): if the share supports replication we need
# to query all its replicas and calculate the final size to
# deallocate (amount of replicas * size to decrease).
if supports_replication:
replica_gigs_to_deallocate = len(replicas) * size_decrease
deltas.update(
{'replica_gigabytes': -replica_gigs_to_deallocate})
reservations = QUOTAS.reserve(context, **deltas)
except Exception as e:
error_occurred(
e, ("Failed to update quota on share shrinking."))

65
manila/tests/api/v1/test_limits.py
View File

@ -16,6 +16,7 @@
"""
Tests dealing with HTTP rate-limiting.
"""
import ddt
from oslo_serialization import jsonutils
import six
@ -23,10 +24,12 @@ from six import moves
from six.moves import http_client
import webob
from manila.api.openstack import api_version_request as api_version
from manila.api.v1 import limits
from manila.api import views
import manila.context
from manila import test
from manila.tests.api import fakes
TEST_LIMITS = [
@ -36,6 +39,7 @@ TEST_LIMITS = [
limits.Limit("PUT", "*", "", 10, limits.PER_MINUTE),
limits.Limit("PUT", "/shares", "^/shares", 5, limits.PER_MINUTE),
]
SHARE_REPLICAS_LIMIT_MICROVERSION = "2.53"
class BaseLimitTestSuite(test.TestCase):
@ -64,24 +68,20 @@ class BaseLimitTestSuite(test.TestCase):
return self.time
@ddt.ddt
class LimitsControllerTest(BaseLimitTestSuite):
"""Tests for `limits.LimitsController` class."""
def setUp(self):
"""Run before each test."""
super(LimitsControllerTest, self).setUp()
self.controller = limits.create_resource()
self.controller = limits.LimitsController()
def _get_index_request(self, accept_header="application/json"):
def _get_index_request(self, accept_header="application/json",
microversion=api_version.DEFAULT_API_VERSION):
"""Helper to set routing arguments."""
request = webob.Request.blank("/")
request = fakes.HTTPRequest.blank('/limit', version=microversion)
request.accept = accept_header
request.environ["wsgiorg.routing_args"] = (None, {
"action": "index",
"controller": "",
})
context = manila.context.RequestContext('testuser', 'testproject')
request.environ["manila.context"] = context
return request
def _populate_limits(self, request):
@ -98,19 +98,20 @@ class LimitsControllerTest(BaseLimitTestSuite):
def test_empty_index_json(self):
"""Test getting empty limit details in JSON."""
request = self._get_index_request()
response = request.get_response(self.controller)
response = self.controller.index(request)
expected = {
"limits": {
"rate": [],
"absolute": {},
},
}
body = jsonutils.loads(response.body)
self.assertEqual(expected, body)
self.assertEqual(expected, response)
def test_index_json(self):
@ddt.data(api_version.DEFAULT_API_VERSION,
SHARE_REPLICAS_LIMIT_MICROVERSION)
def test_index_json(self, microversion):
"""Test getting limit details in JSON."""
request = self._get_index_request()
request = self._get_index_request(microversion=microversion)
request = self._populate_limits(request)
self.absolute_limits = {
'limit': {
@ -128,7 +129,15 @@ class LimitsControllerTest(BaseLimitTestSuite):
'share_networks': 7,
},
}
response = request.get_response(self.controller)
if microversion == SHARE_REPLICAS_LIMIT_MICROVERSION:
self.absolute_limits['limit']['share_replicas'] = 20
self.absolute_limits['limit']['replica_gigabytes'] = 20
self.absolute_limits['in_use']['share_replicas'] = 3
self.absolute_limits['in_use']['replica_gigabytes'] = 3
response = self.controller.index(request)
expected = {
"limits": {
"rate": [
@ -181,8 +190,13 @@ class LimitsControllerTest(BaseLimitTestSuite):
},
},
}
body = jsonutils.loads(response.body)
self.assertEqual(expected, body)
if microversion == SHARE_REPLICAS_LIMIT_MICROVERSION:
expected['limits']['absolute']["maxTotalShareReplicas"] = 20
expected['limits']['absolute']["totalShareReplicasUsed"] = 3
expected['limits']['absolute']["maxTotalReplicaGigabytes"] = 20
expected['limits']['absolute']["totalReplicaGigabytesUsed"] = 3
# body = jsonutils.loads(response.body)
self.assertEqual(expected, response)
def _populate_limits_diff_regex(self, request):
"""Put limit info into a request."""
@ -197,7 +211,7 @@ class LimitsControllerTest(BaseLimitTestSuite):
"""Test getting limit details in JSON."""
request = self._get_index_request()
request = self._populate_limits_diff_regex(request)
response = request.get_response(self.controller)
response = self.controller.index(request)
expected = {
"limits": {
"rate": [
@ -232,14 +246,12 @@ class LimitsControllerTest(BaseLimitTestSuite):
"absolute": {},
},
}
body = jsonutils.loads(response.body)
self.assertEqual(expected, body)
self.assertEqual(expected, response)
def _test_index_absolute_limits_json(self, expected):
request = self._get_index_request()
response = request.get_response(self.controller)
body = jsonutils.loads(response.body)
self.assertEqual(expected, body['limits']['absolute'])
response = self.controller.index(request)
self.assertEqual(expected, response['limits']['absolute'])
def test_index_ignores_extra_absolute_limits_json(self):
self.absolute_limits = {
@ -783,6 +795,7 @@ class LimitsViewBuilderTest(test.TestCase):
}
def test_build_limits(self):
request = fakes.HTTPRequest.blank('/')
tdate = "2011-07-21T18:17:06Z"
expected_limits = {
"limits": {
@ -817,15 +830,17 @@ class LimitsViewBuilderTest(test.TestCase):
}
}
output = self.view_builder.build(self.rate_limits,
output = self.view_builder.build(request,
self.rate_limits,
self.absolute_limits)
self.assertDictMatch(expected_limits, output)
def test_build_limits_empty_limits(self):
request = fakes.HTTPRequest.blank('/')
expected_limits = {"limits": {"rate": [], "absolute": {}}}
abs_limits = {}
rate_limits = []
output = self.view_builder.build(rate_limits, abs_limits)
output = self.view_builder.build(request, rate_limits, abs_limits)
self.assertDictMatch(expected_limits, output)

17
manila/tests/api/v2/test_quota_class_sets.py
View File

@ -26,6 +26,7 @@ from oslo_config import cfg
import webob.exc
import webob.response
from manila.api.openstack import api_version_request as api_version
from manila.api.v2 import quota_class_sets
from manila import context
from manila import exception
@ -60,6 +61,7 @@ class QuotaSetsControllerTest(test.TestCase):
('os-', '1.0', quota_class_sets.QuotaClassSetsControllerLegacy),
('os-', '2.6', quota_class_sets.QuotaClassSetsControllerLegacy),
('', '2.7', quota_class_sets.QuotaClassSetsController),
('', '2.53', quota_class_sets.QuotaClassSetsController),
)
@ddt.unpack
def test_show_quota(self, url, version, controller):
@ -86,6 +88,13 @@ class QuotaSetsControllerTest(test.TestCase):
for k, v in quotas.items():
CONF.set_default('quota_' + k, v)
if req.api_version_request >= api_version.APIVersionRequest("2.40"):
expected['quota_class_set']['share_groups'] = 50
expected['quota_class_set']['share_group_snapshots'] = 50
if req.api_version_request >= api_version.APIVersionRequest("2.53"):
expected['quota_class_set']['share_replicas'] = 100
expected['quota_class_set']['replica_gigabytes'] = 1000
result = controller().show(req, self.class_name)
self.assertEqual(expected, result)
@ -109,6 +118,7 @@ class QuotaSetsControllerTest(test.TestCase):
('os-', '1.0', quota_class_sets.QuotaClassSetsControllerLegacy),
('os-', '2.6', quota_class_sets.QuotaClassSetsControllerLegacy),
('', '2.7', quota_class_sets.QuotaClassSetsController),
('', '2.53', quota_class_sets.QuotaClassSetsController),
)
@ddt.unpack
def test_update_quota(self, url, version, controller):
@ -132,6 +142,13 @@ class QuotaSetsControllerTest(test.TestCase):
}
}
if req.api_version_request >= api_version.APIVersionRequest("2.40"):
expected['quota_class_set']['share_groups'] = 50
expected['quota_class_set']['share_group_snapshots'] = 50
if req.api_version_request >= api_version.APIVersionRequest("2.53"):
expected['quota_class_set']['share_replicas'] = 100
expected['quota_class_set']['replica_gigabytes'] = 1000
update_result = controller().update(
req, self.class_name, body=body)

39
manila/tests/api/v2/test_quota_sets.py
View File

@ -34,6 +34,8 @@ from manila.tests.api import fakes
from manila import utils
CONF = cfg.CONF
sg_quota_keys = ['share_groups', 'share_group_snapshots']
replica_quota_keys = ['share_replicas']
def _get_request(is_admin, user_in_url):
@ -269,27 +271,38 @@ class QuotaSetsControllerTest(test.TestCase):
self.assertIsNone(result)
@ddt.data(
{},
{"quota_set": {}},
{"quota_set": {"foo": "bar"}},
{"foo": "bar"},
({}, sg_quota_keys, '2.40'),
({"quota_set": {}}, sg_quota_keys, '2.40'),
({"quota_set": {"foo": "bar"}}, sg_quota_keys, '2.40'),
({"foo": "bar"}, replica_quota_keys, '2.53'),
({"quota_set": {"foo": "bar"}}, replica_quota_keys, '2.53'),
)
def test__ensure_share_group_related_args_are_absent_success(self, body):
result = self.controller._ensure_share_group_related_args_are_absent(
body)
@ddt.unpack
def test__ensure_specific_microversion_args_are_absent_success(
self, body, keys, microversion):
result = self.controller._ensure_specific_microversion_args_are_absent(
body, keys, microversion)
self.assertIsNone(result)
@ddt.data(
{"share_groups": 5},
{"share_group_snapshots": 6},
{"quota_set": {"share_groups": 7}},
{"quota_set": {"share_group_snapshots": 8}},
({"share_groups": 5}, sg_quota_keys, '2.40'),
({"share_group_snapshots": 6}, sg_quota_keys, '2.40'),
({"quota_set": {"share_groups": 7}}, sg_quota_keys, '2.40'),
({"quota_set": {"share_group_snapshots": 8}}, sg_quota_keys, '2.40'),
({"quota_set": {"share_replicas": 9}}, replica_quota_keys, '2.53'),
({"quota_set": {"share_replicas": 10}}, replica_quota_keys, '2.53'),
)
def test__ensure_share_group_related_args_are_absent_error(self, body):
@ddt.unpack
def test__ensure_specific_microversion_args_are_absent_error(
self, body, keys, microversion):
self.assertRaises(
webob.exc.HTTPBadRequest,
self.controller._ensure_share_group_related_args_are_absent, body)
self.controller._ensure_specific_microversion_args_are_absent,
body,
keys,
microversion
)
@ddt.data(_get_request(True, True), _get_request(True, False))
def test__ensure_share_type_arg_is_absent(self, req):

5
manila/tests/api/v2/test_share_replicas.py
View File

@ -657,7 +657,7 @@ class ShareReplicasApiTest(test.TestCase):
def test_force_delete_missing_replica(self):
replica, req = self._create_replica_get_req()
share_replicas.db.share_replica_delete(
self.admin_context, replica['id'])
self.admin_context, replica['id'], need_to_update_usages=False)
self._force_delete(self.admin_context, req, valid_code=404)
@ -665,10 +665,9 @@ class ShareReplicasApiTest(test.TestCase):
replica, req = self._create_replica_get_req()
share_replicas.db.share_replica_delete(
self.admin_context, replica['id'])
self.admin_context, replica['id'], need_to_update_usages=False)
share_api_call = self.mock_object(self.controller.share_api,
'update_share_replica')
body = {'resync': {}}
req.body = six.b(jsonutils.dumps(body))
req.environ['manila.context'] = self.admin_context

11
manila/tests/api/views/test_quota_class_sets.py
View File

@ -34,6 +34,7 @@ class ViewBuilderTestCase(test.TestCase):
@ddt.data(
("fake_quota_class", "2.40"), (None, "2.40"),
("fake_quota_class", "2.39"), (None, "2.39"),
("fake_quota_class", "2.53"), (None, "2.53"),
)
@ddt.unpack
def test_detail_list_with_share_type(self, quota_class, microversion):
@ -64,6 +65,16 @@ class ViewBuilderTestCase(test.TestCase):
"share_group_snapshots"] = quota_class_set[
"share_group_snapshots"]
if req.api_version_request >= api_version.APIVersionRequest("2.53"):
fake_share_replicas_value = 46
fake_replica_gigabytes_value = 100
expected[self.builder._collection_name]["share_replicas"] = (
fake_share_replicas_value)
expected[self.builder._collection_name][
"replica_gigabytes"] = fake_replica_gigabytes_value
quota_class_set['share_replicas'] = fake_share_replicas_value
quota_class_set['replica_gigabytes'] = fake_replica_gigabytes_value
result = self.builder.detail_list(
req, quota_class_set, quota_class=quota_class)

13
manila/tests/api/views/test_quota_sets.py
View File

@ -40,6 +40,9 @@ class ViewBuilderTestCase(test.TestCase):
(None, 'fake_share_type_id', "2.39"),
('fake_project_id', None, "2.39"),
(None, None, "2.39"),
(None, 'fake_share_type_id', "2.53"),
('fake_project_id', None, "2.53"),
(None, None, "2.53"),
)
@ddt.unpack
def test_detail_list_with_share_type(self, project_id, share_type,
@ -73,6 +76,16 @@ class ViewBuilderTestCase(test.TestCase):
"share_group_snapshots"] = quota_set[
"share_group_snapshots"]
if req.api_version_request >= api_version.APIVersionRequest("2.53"):
fake_share_replicas_value = 46
fake_replica_gigabytes_value = 100
expected[self.builder._collection_name]["share_replicas"] = (
fake_share_replicas_value)
expected[self.builder._collection_name][
"replica_gigabytes"] = fake_replica_gigabytes_value
quota_set['share_replicas'] = fake_share_replicas_value
quota_set['replica_gigabytes'] = fake_replica_gigabytes_value
result = self.builder.detail_list(
req, quota_set, project_id=project_id, share_type=share_type)

99
manila/tests/db/sqlalchemy/test_api.py
View File

@ -827,6 +827,9 @@ class ShareDatabaseAPITestCase(test.TestCase):
def test_share_replica_delete(self):
share = db_utils.create_share()
share = db_api.share_get(self.ctxt, share['id'])
self.mock_object(quota.QUOTAS, 'reserve',
mock.Mock(return_value='reservation'))
self.mock_object(quota.QUOTAS, 'commit')
replica = db_utils.create_share_replica(
share_id=share['id'], replica_state=constants.REPLICA_STATE_ACTIVE)
@ -837,6 +840,55 @@ class ShareDatabaseAPITestCase(test.TestCase):
self.assertEqual(
[], db_api.share_replicas_get_all_by_share(self.ctxt, share['id']))
share_type_id = share['instances'][0].get('share_type_id', None)
quota.QUOTAS.reserve.assert_called_once_with(
self.ctxt, project_id=share['project_id'],
user_id=share['user_id'], share_type_id=share_type_id,
share_replicas=-1, replica_gigabytes=share['size'])
quota.QUOTAS.commit.assert_called_once_with(
self.ctxt, 'reservation', project_id=share['project_id'],
user_id=share['user_id'], share_type_id=share_type_id)
@ddt.data(
(True, {"share_replicas": -1, "replica_gigabytes": 0}, 'active'),
(False, {"shares": -1, "gigabytes": 0}, None),
(False, {"shares": -1, "gigabytes": 0,
"share_replicas": -1, "replica_gigabytes": 0}, 'active')
)
@ddt.unpack
def test_share_instance_delete_quota_error(self, is_replica, deltas,
replica_state):
share = db_utils.create_share(replica_state=replica_state)
share = db_api.share_get(self.ctxt, share['id'])
instance_id = share['instances'][0]['id']
if is_replica:
replica = db_utils.create_share_replica(
share_id=share['id'],
replica_state=constants.REPLICA_STATE_ACTIVE)
instance_id = replica['id']
reservation = 'fake'
share_type_id = share['instances'][0]['share_type_id']
self.mock_object(quota.QUOTAS, 'reserve',
mock.Mock(return_value=reservation))
self.mock_object(quota.QUOTAS, 'commit', mock.Mock(
side_effect=exception.QuotaError('fake')))
self.mock_object(quota.QUOTAS, 'rollback')
# NOTE(silvacarlose): not calling with assertRaises since the
# _update_share_instance_usages method is not raising an exception
db_api.share_instance_delete(
self.ctxt, instance_id, session=None, need_to_update_usages=True)
quota.QUOTAS.reserve.assert_called_once_with(
self.ctxt, project_id=share['project_id'],
user_id=share['user_id'], share_type_id=share_type_id, **deltas)
quota.QUOTAS.commit.assert_called_once_with(
self.ctxt, reservation, project_id=share['project_id'],
user_id=share['user_id'], share_type_id=share_type_id)
quota.QUOTAS.rollback.assert_called_once_with(
self.ctxt, reservation, share_type_id=share_type_id)
def test_share_instance_access_copy(self):
share = db_utils.create_share()
@ -3488,6 +3540,53 @@ class ShareTypeAPITestCase(test.TestCase):
for q_reservation in q_reservations:
self.assertIsNone(q_reservation['share_type_id'])
@ddt.data(
(None, None, 5),
('fake2', None, 2),
(None, 'fake', 3),
)
@ddt.unpack
def test_share_replica_data_get_for_project(
self, user_id, share_type_id, expected_result):
kwargs = {}
if share_type_id:
kwargs.update({'id': share_type_id})
share_type_1 = db_utils.create_share_type(**kwargs)
share_type_2 = db_utils.create_share_type()
share_1 = db_utils.create_share(size=1, user_id='fake',
share_type_id=share_type_1['id'])
share_2 = db_utils.create_share(size=1, user_id='fake2',
share_type_id=share_type_2['id'])
project_id = share_1['project_id']
db_utils.create_share_replica(
replica_state=constants.REPLICA_STATE_ACTIVE,
share_id=share_1['id'], share_type_id=share_type_1['id'])
db_utils.create_share_replica(
replica_state=constants.REPLICA_STATE_IN_SYNC,
share_id=share_1['id'], share_type_id=share_type_1['id'])
db_utils.create_share_replica(
replica_state=constants.REPLICA_STATE_IN_SYNC,
share_id=share_1['id'], share_type_id=share_type_1['id'])
db_utils.create_share_replica(
replica_state=constants.REPLICA_STATE_ACTIVE,
share_id=share_2['id'], share_type_id=share_type_2['id'])
db_utils.create_share_replica(
replica_state=constants.REPLICA_STATE_IN_SYNC,
share_id=share_2['id'], share_type_id=share_type_2['id'])
kwargs = {}
if user_id:
kwargs.update({'user_id': user_id})
if share_type_id:
kwargs.update({'share_type_id': share_type_id})
total_amount, total_size = db_api.share_replica_data_get_for_project(
self.ctxt, project_id, **kwargs)
self.assertEqual(expected_result, total_amount)
self.assertEqual(expected_result, total_size)
def test_share_type_get_by_name_or_id_found_by_id(self):
share_type = db_utils.create_share_type()

180
manila/tests/share/test_api.py
View File

@ -2168,6 +2168,34 @@ class ShareAPITestCase(test.TestCase):
quota.QUOTAS.commit.assert_called_once_with(
self.context, 'reservation', share_type_id=share_type['id'])
def test_create_share_share_type_contains_replication_type(self):
extra_specs = {'replication_type': constants.REPLICATION_TYPE_READABLE}
share_type = db_utils.create_share_type(extra_specs=extra_specs)
share_type = db_api.share_type_get(self.context, share_type['id'])
share, share_data = self._setup_create_mocks(
share_type_id=share_type['id'])
az = share_data.pop('availability_zone')
self.mock_object(quota.QUOTAS, 'reserve',
mock.Mock(return_value='reservation'))
self.mock_object(quota.QUOTAS, 'commit')
self.api.create(
self.context,
share_data['share_proto'],
share_data['size'],
share_data['display_name'],
share_data['display_description'],
availability_zone=az,
share_type=share_type
)
quota.QUOTAS.reserve.assert_called_once_with(
self.context, share_type_id=share_type['id'],
gigabytes=1, shares=1, share_replicas=1, replica_gigabytes=1)
quota.QUOTAS.commit.assert_called_once_with(
self.context, 'reservation', share_type_id=share_type['id']
)
def test_create_from_snapshot_with_different_share_type(self):
snapshot, share, share_data, request_spec = (
self._setup_create_from_snapshot_mocks()
@ -2709,19 +2737,52 @@ class ShareAPITestCase(test.TestCase):
self.assertRaises(exception.InvalidInput,
self.api.extend, self.context, share, new_size)
def test_extend_quota_error(self):
def _setup_extend_mocks(self, supports_replication):
replica_list = []
if supports_replication:
replica_list.append({'id': 'fake_replica_id'})
replica_list.append({'id': 'fake_replica_id_2'})
self.mock_object(db_api, 'share_replicas_get_all_by_share',
mock.Mock(return_value=replica_list))
@ddt.data(
(False, 'gigabytes', exception.ShareSizeExceedsAvailableQuota),
(True, 'replica_gigabytes',
exception.ShareReplicaSizeExceedsAvailableQuota)
)
@ddt.unpack
def test_extend_quota_error(self, supports_replication, quota_key,
expected_exception):
self._setup_extend_mocks(supports_replication)
share = db_utils.create_share(status=constants.STATUS_AVAILABLE,
size=100)
new_size = 123
usages = {'gigabytes': {'reserved': 11, 'in_use': 12}}
quotas = {'gigabytes': 13}
exc = exception.OverQuota(usages=usages, quotas=quotas, overs=new_size)
replica_amount = len(
db_api.share_replicas_get_all_by_share.return_value)
value_to_be_extended = new_size - share['size']
usages = {quota_key: {'reserved': 11, 'in_use': 12}}
quotas = {quota_key: 13}
overs = {quota_key: new_size}
exc = exception.OverQuota(usages=usages, quotas=quotas, overs=overs)
expected_deltas = {
'project_id': share['project_id'],
'gigabytes': value_to_be_extended,
'user_id': share['user_id'],
'share_type_id': share['instance']['share_type_id']
}
if supports_replication:
expected_deltas.update(
{'replica_gigabytes': value_to_be_extended * replica_amount})
self.mock_object(quota.QUOTAS, 'reserve', mock.Mock(side_effect=exc))
self.assertRaises(exception.ShareSizeExceedsAvailableQuota,
self.assertRaises(expected_exception,
self.api.extend, self.context, share, new_size)
quota.QUOTAS.reserve.assert_called_once_with(
mock.ANY, **expected_deltas
)
def test_extend_quota_user(self):
self._setup_extend_mocks(False)
share = db_utils.create_share(status=constants.STATUS_AVAILABLE,
size=100)
diff_user_context = context.RequestContext(
@ -2743,12 +2804,28 @@ class ShareAPITestCase(test.TestCase):
user_id=share['user_id']
)
def test_extend_valid(self):
@ddt.data(True, False)
def test_extend_valid(self, supports_replication):
self._setup_extend_mocks(supports_replication)
share = db_utils.create_share(status=constants.STATUS_AVAILABLE,
size=100)
new_size = 123
size_increase = int(new_size) - share['size']
replica_amount = len(
db_api.share_replicas_get_all_by_share.return_value)
expected_deltas = {
'project_id': share['project_id'],
'gigabytes': size_increase,
'user_id': share['user_id'],
'share_type_id': share['instance']['share_type_id']
}
if supports_replication:
new_replica_size = size_increase * replica_amount
expected_deltas.update({'replica_gigabytes': new_replica_size})
self.mock_object(self.api, 'update')
self.mock_object(self.api.share_rpcapi, 'extend_share')
self.mock_object(quota.QUOTAS, 'reserve')
self.api.extend(self.context, share, new_size)
@ -2757,6 +2834,8 @@ class ShareAPITestCase(test.TestCase):
self.api.share_rpcapi.extend_share.assert_called_once_with(
self.context, share, new_size, mock.ANY
)
quota.QUOTAS.reserve.assert_called_once_with(
self.context, **expected_deltas)
def test_shrink_invalid_status(self):
invalid_status = 'fake'
@ -3836,6 +3915,95 @@ class ShareAPITestCase(test.TestCase):
self.assertTrue(mock_rpcapi_update_share_replica_call.called)
self.assertIsNone(retval)
@ddt.data({'overs': {'replica_gigabytes': 'fake'},
'expected_exception':
exception.ShareReplicaSizeExceedsAvailableQuota},
{'overs': {'share_replicas': 'fake'},
'expected_exception': exception.ShareReplicasLimitExceeded})
@ddt.unpack
def test_create_share_replica_over_quota(self, overs, expected_exception):
request_spec = fakes.fake_replica_request_spec()
replica = request_spec['share_instance_properties']
share = db_utils.create_share(replication_type='dr',
id=replica['share_id'])
share_type = db_utils.create_share_type()
share_type = db_api.share_type_get(self.context, share_type['id'])
usages = {'replica_gigabytes': {'reserved': 5, 'in_use': 5},
'share_replicas': {'reserved': 5, 'in_use': 5}}
quotas = {'share_replicas': 5, 'replica_gigabytes': 5}
exc = exception.OverQuota(overs=overs, usages=usages, quotas=quotas)
self.mock_object(quota.QUOTAS, 'reserve', mock.Mock(side_effect=exc))
self.mock_object(db_api, 'share_replicas_get_available_active_replica',
mock.Mock(return_value={'host': 'fake_ar_host'}))
self.mock_object(share_types, 'get_share_type',
mock.Mock(return_value=share_type))
self.assertRaises(
expected_exception,
self.api.create_share_replica,
self.context,
share
)
quota.QUOTAS.reserve.assert_called_once_with(
self.context, share_type_id=share_type['id'],
share_replicas=1, replica_gigabytes=share['size'])
(db_api.share_replicas_get_available_active_replica
.assert_called_once_with(self.context, share['id']))
share_types.get_share_type.assert_called_once_with(
self.context, share['instance']['share_type_id'])
def test_create_share_replica_error_on_quota_commit(self):
request_spec = fakes.fake_replica_request_spec()
replica = request_spec['share_instance_properties']
share_type = db_utils.create_share_type()
fake_replica = fakes.fake_replica(id=replica['id'])
share = db_utils.create_share(replication_type='dr',
id=fake_replica['share_id'],
share_type_id=share_type['id'])
share_network_id = None
share_type = db_api.share_type_get(self.context, share_type['id'])
expected_azs = share_type['extra_specs'].get('availability_zones', '')
expected_azs = expected_azs.split(',') if expected_azs else []
reservation = 'fake'
self.mock_object(quota.QUOTAS, 'reserve',
mock.Mock(return_value=reservation))
self.mock_object(quota.QUOTAS, 'commit',
mock.Mock(side_effect=exception.QuotaError('fake')))
self.mock_object(db_api, 'share_replica_delete')
self.mock_object(quota.QUOTAS, 'rollback')
self.mock_object(db_api, 'share_replicas_get_available_active_replica',
mock.Mock(return_value={'host': 'fake_ar_host'}))
self.mock_object(share_types, 'get_share_type',
mock.Mock(return_value=share_type))
self.mock_object(
share_api.API, 'create_share_instance_and_get_request_spec',
mock.Mock(return_value=(request_spec, fake_replica)))
self.assertRaises(
exception.QuotaError,
self.api.create_share_replica,
self.context,
share
)
db_api.share_replica_delete.assert_called_once_with(
self.context, replica['id'], need_to_update_usages=False)
quota.QUOTAS.rollback.assert_called_once_with(
self.context, reservation,
share_type_id=share['instance']['share_type_id'])
(db_api.share_replicas_get_available_active_replica.
assert_called_once_with(self.context, share['id']))
share_types.get_share_type.assert_called_once_with(
self.context, share['instance']['share_type_id'])
(share_api.API.create_share_instance_and_get_request_spec.
assert_called_once_with(self.context, share, availability_zone=None,
share_network_id=share_network_id,
share_type_id=share_type['id'],
availability_zones=expected_azs,
cast_rules_to_readonly=False))
def test_migration_complete(self):
instance1 = db_utils.create_share_instance(

151
manila/tests/share/test_manager.py
View File

@ -44,6 +44,7 @@ from manila.share import share_types
from manila import test
from manila.tests.api import fakes as test_fakes
from manila.tests import db_utils
from manila.tests import fake_notifier
from manila.tests import fake_share as fakes
from manila.tests import fake_utils
from manila.tests import utils as test_utils
@ -2510,6 +2511,9 @@ class ShareManagerTestCase(test.TestCase):
self.mock_object(share_types,
'get_share_type_extra_specs',
mock.Mock(return_value='False'))
self.mock_object(
self.share_manager, '_get_extra_specs_from_share_type',
mock.Mock(return_value={}))
self.mock_object(self.share_manager.db, 'share_update', mock.Mock())
share = db_utils.create_share()
share_id = share['id']
@ -2526,6 +2530,9 @@ class ShareManagerTestCase(test.TestCase):
self.share_manager.db.share_update.assert_called_once_with(
utils.IsAMatcher(context.RequestContext), share_id,
{'status': constants.STATUS_MANAGE_ERROR, 'size': 1})
(self.share_manager._get_extra_specs_from_share_type.
assert_called_once_with(
mock.ANY, share['instance']['share_type_id']))
def test_manage_share_invalid_size(self):
self.mock_object(self.share_manager, 'driver')
@ -2537,6 +2544,9 @@ class ShareManagerTestCase(test.TestCase):
"manage_existing",
mock.Mock(return_value=None))
self.mock_object(self.share_manager.db, 'share_update', mock.Mock())
self.mock_object(
self.share_manager, '_get_extra_specs_from_share_type',
mock.Mock(return_value={}))
share = db_utils.create_share()
share_id = share['id']
driver_options = {'fake': 'fake'}
@ -2551,6 +2561,9 @@ class ShareManagerTestCase(test.TestCase):
self.share_manager.db.share_update.assert_called_once_with(
utils.IsAMatcher(context.RequestContext), share_id,
{'status': constants.STATUS_MANAGE_ERROR, 'size': 1})
(self.share_manager._get_extra_specs_from_share_type.
assert_called_once_with(
mock.ANY, share['instance']['share_type_id']))
def test_manage_share_quota_error(self):
self.mock_object(self.share_manager, 'driver')
@ -2564,6 +2577,9 @@ class ShareManagerTestCase(test.TestCase):
self.mock_object(quota.QUOTAS, 'reserve',
mock.Mock(side_effect=exception.QuotaError))
self.mock_object(self.share_manager.db, 'share_update', mock.Mock())
self.mock_object(
self.share_manager, '_get_extra_specs_from_share_type',
mock.Mock(return_value={}))
share = db_utils.create_share()
share_id = share['id']
driver_options = {'fake': 'fake'}
@ -2578,6 +2594,9 @@ class ShareManagerTestCase(test.TestCase):
self.share_manager.db.share_update.assert_called_once_with(
mock.ANY, share_id,
{'status': constants.STATUS_MANAGE_ERROR, 'size': 1})
(self.share_manager._get_extra_specs_from_share_type.
assert_called_once_with(
mock.ANY, share['instance']['share_type_id']))
def test_manage_share_incompatible_dhss(self):
self.mock_object(self.share_manager, 'driver')
@ -2586,9 +2605,15 @@ class ShareManagerTestCase(test.TestCase):
self.mock_object(share_types,
'get_share_type_extra_specs',
mock.Mock(return_value="True"))
self.mock_object(
self.share_manager, '_get_extra_specs_from_share_type',
mock.Mock(return_value={}))
self.assertRaises(
exception.InvalidShare, self.share_manager.manage_share,
self.context, share['id'], {})
(self.share_manager._get_extra_specs_from_share_type.
assert_called_once_with(
mock.ANY, share['instance']['share_type_id']))
@ddt.data({'dhss': True,
'driver_data': {'size': 1, 'replication_type': None}},
@ -2604,6 +2629,9 @@ class ShareManagerTestCase(test.TestCase):
self.mock_object(self.share_manager, 'driver')
self.share_manager.driver.driver_handles_share_servers = dhss
replication_type = driver_data.pop('replication_type')
extra_specs = {}
if replication_type is not None:
extra_specs.update({'replication_type': replication_type})
export_locations = driver_data.get('export_locations')
self.mock_object(self.share_manager.db, 'share_update', mock.Mock())
self.mock_object(quota.QUOTAS, 'reserve', mock.Mock())
@ -2615,6 +2643,10 @@ class ShareManagerTestCase(test.TestCase):
self.mock_object(share_types,
'get_share_type_extra_specs',
mock.Mock(return_value=six.text_type(dhss)))
self.mock_object(
self.share_manager, '_get_extra_specs_from_share_type',
mock.Mock(return_value=extra_specs))
if dhss:
mock_manage = self.mock_object(
self.share_manager.driver,
@ -2628,6 +2660,16 @@ class ShareManagerTestCase(test.TestCase):
share = db_utils.create_share(replication_type=replication_type)
share_id = share['id']
driver_options = {'fake': 'fake'}
expected_deltas = {
'project_id': share['project_id'],
'user_id': self.context.user_id,
'shares': 1,
'gigabytes': driver_data['size'],
'share_type_id': share['instance']['share_type_id'],
}
if replication_type:
expected_deltas.update({'share_replicas': 1,
'replica_gigabytes': driver_data['size']})
self.share_manager.manage_share(self.context, share_id, driver_options)
@ -2651,6 +2693,11 @@ class ShareManagerTestCase(test.TestCase):
self.share_manager.db.share_update.assert_called_once_with(
utils.IsAMatcher(context.RequestContext),
share_id, valid_share_data)
quota.QUOTAS.reserve.assert_called_once_with(
mock.ANY, **expected_deltas)
(self.share_manager._get_extra_specs_from_share_type.
assert_called_once_with(
mock.ANY, share['instance']['share_type_id']))
def test_update_quota_usages_new(self):
self.mock_object(self.share_manager.db, 'quota_usage_get',
@ -2688,9 +2735,12 @@ class ShareManagerTestCase(test.TestCase):
mock.ANY, project_id, mock.ANY, resource_name, usage)
def _setup_unmanage_mocks(self, mock_driver=True, mock_unmanage=None,
dhss=False):
dhss=False, supports_replication=False):
if mock_driver:
self.mock_object(self.share_manager, 'driver')
replicas_list = []
if supports_replication:
replicas_list.append({'id': 'fake_id'})
if mock_unmanage:
if dhss:
@ -2703,6 +2753,9 @@ class ShareManagerTestCase(test.TestCase):
self.mock_object(self.share_manager.db, 'share_update')
self.mock_object(self.share_manager.db, 'share_instance_delete')
self.mock_object(
self.share_manager.db, 'share_replicas_get_all_by_share',
mock.Mock(return_value=replicas_list))
def test_unmanage_share_invalid_share(self):
self.mock_object(self.share_manager, 'driver')
@ -2715,17 +2768,31 @@ class ShareManagerTestCase(test.TestCase):
self.share_manager.db.share_update.assert_called_once_with(
mock.ANY, share['id'], {'status': constants.STATUS_UNMANAGE_ERROR})
(self.share_manager.db.share_replicas_get_all_by_share.
assert_called_once_with(mock.ANY, share['id']))
def test_unmanage_share_valid_share(self):
@ddt.data(True, False)
def test_unmanage_share_valid_share(self, supports_replication):
self.mock_object(self.share_manager, 'driver')
self.share_manager.driver.driver_handles_share_servers = False
self._setup_unmanage_mocks(mock_driver=False,
mock_unmanage=mock.Mock())
self._setup_unmanage_mocks(
mock_driver=False, mock_unmanage=mock.Mock(),
supports_replication=supports_replication)
self.mock_object(quota.QUOTAS, 'reserve')
share = db_utils.create_share()
share_id = share['id']
share_instance_id = share.instance['id']
self.mock_object(self.share_manager.db, 'share_instance_get',
mock.Mock(return_value=share.instance))
reservation_params = {
'project_id': share['project_id'],
'shares': -1,
'gigabytes': -share['size'],
'share_type_id': share['instance']['share_type_id'],
}
if supports_replication:
reservation_params.update(
{'share_replicas': -1, 'replica_gigabytes': -share['size']})
self.share_manager.unmanage_share(self.context, share_id)
@ -2733,13 +2800,19 @@ class ShareManagerTestCase(test.TestCase):
assert_called_once_with(share.instance))
self.share_manager.db.share_instance_delete.assert_called_once_with(
mock.ANY, share_instance_id)
quota.QUOTAS.reserve.assert_called_once_with(
mock.ANY, **reservation_params)
(self.share_manager.db.share_replicas_get_all_by_share.
assert_called_once_with(mock.ANY, share['id']))
def test_unmanage_share_valid_share_with_share_server(self):
@ddt.data(True, False)
def test_unmanage_share_valid_share_with_share_server(
self, supports_replication):
self.mock_object(self.share_manager, 'driver')
self.share_manager.driver.driver_handles_share_servers = True
self._setup_unmanage_mocks(mock_driver=False,
mock_unmanage=mock.Mock(),
dhss=True)
self._setup_unmanage_mocks(
mock_driver=False, mock_unmanage=mock.Mock(), dhss=True,
supports_replication=supports_replication)
server = db_utils.create_share_server(id='fake_server_id')
share = db_utils.create_share(share_server_id='fake_server_id')
self.mock_object(self.share_manager.db, 'share_server_update')
@ -2747,6 +2820,16 @@ class ShareManagerTestCase(test.TestCase):
mock.Mock(return_value=server))
self.mock_object(self.share_manager.db, 'share_instance_get',
mock.Mock(return_value=share.instance))
self.mock_object(quota.QUOTAS, 'reserve')
reservation_params = {
'project_id': share['project_id'],
'shares': -1,
'gigabytes': -share['size'],
'share_type_id': share['instance']['share_type_id'],
}
if supports_replication:
reservation_params.update(
{'share_replicas': -1, 'replica_gigabytes': -share['size']})
share_id = share['id']
share_instance_id = share.instance['id']
@ -2759,6 +2842,10 @@ class ShareManagerTestCase(test.TestCase):
mock.ANY, share_instance_id)
self.share_manager.db.share_server_update.assert_called_once_with(
mock.ANY, server['id'], {'is_auto_deletable': False})
quota.QUOTAS.reserve.assert_called_once_with(
mock.ANY, **reservation_params)
(self.share_manager.db.share_replicas_get_all_by_share
.assert_called_once_with(mock.ANY, share['id']))
def test_unmanage_share_valid_share_with_quota_error(self):
self.mock_object(self.share_manager, 'driver')
@ -2775,6 +2862,8 @@ class ShareManagerTestCase(test.TestCase):
self.share_manager.driver.unmanage.assert_called_once_with(mock.ANY)
self.share_manager.db.share_instance_delete.assert_called_once_with(
mock.ANY, share_instance_id)
(self.share_manager.db.share_replicas_get_all_by_share.
assert_called_once_with(mock.ANY, share['id']))
def test_unmanage_share_remove_access_rules_error(self):
self.mock_object(self.share_manager, 'driver')
@ -2794,6 +2883,8 @@ class ShareManagerTestCase(test.TestCase):
self.share_manager.db.share_update.assert_called_once_with(
mock.ANY, share['id'], {'status': constants.STATUS_UNMANAGE_ERROR})
(self.share_manager.db.share_replicas_get_all_by_share.
assert_called_once_with(mock.ANY, share['id']))
def test_unmanage_share_valid_share_remove_access_rules(self):
self.mock_object(self.share_manager, 'driver')
@ -2816,6 +2907,8 @@ class ShareManagerTestCase(test.TestCase):
)
smanager.db.share_instance_delete.assert_called_once_with(
mock.ANY, share_instance_id)
(self.share_manager.db.share_replicas_get_all_by_share.
assert_called_once_with(mock.ANY, share['id']))
def test_delete_share_instance_share_server_not_found(self):
share_net = db_utils.create_share_network()
@ -3506,7 +3599,10 @@ class ShareManagerTestCase(test.TestCase):
(['INFO', 'share.extend.start'],
['INFO', 'share.extend.end']))
def test_shrink_share_quota_error(self):
@ddt.data((True, [{'id': 'fake'}]), (False, []))
@ddt.unpack
def test_shrink_share_quota_error(self, supports_replication,
replicas_list):
size = 5
new_size = 1
share = db_utils.create_share(size=size)
@ -3515,6 +3611,13 @@ class ShareManagerTestCase(test.TestCase):
self.mock_object(self.share_manager.db, 'share_update')
self.mock_object(quota.QUOTAS, 'reserve',
mock.Mock(side_effect=Exception('fake')))
self.mock_object(
self.share_manager.db, 'share_replicas_get_all_by_share',
mock.Mock(return_value=replicas_list))
deltas = {}
if supports_replication:
deltas.update({'replica_gigabytes': new_size - size})
self.assertRaises(
exception.ShareShrinkingError,
@ -3525,9 +3628,12 @@ class ShareManagerTestCase(test.TestCase):
project_id=six.text_type(share['project_id']),
user_id=six.text_type(share['user_id']),
share_type_id=None,
gigabytes=new_size - size
gigabytes=new_size - size,
**deltas
)
self.assertTrue(self.share_manager.db.share_update.called)
(self.share_manager.db.share_replicas_get_all_by_share
.assert_called_once_with(mock.ANY, share['id']))
@ddt.data({'exc': exception.InvalidShare('fake'),
'status': constants.STATUS_SHRINKING_ERROR},
@ -3570,8 +3676,8 @@ class ShareManagerTestCase(test.TestCase):
)
self.assertTrue(self.share_manager.db.share_get.called)
@mock.patch('manila.tests.fake_notifier.FakeNotifier._notify')
def test_shrink_share(self, mock_notify):
@ddt.data(True, False)
def test_shrink_share(self, supports_replication):
share = db_utils.create_share()
share_id = share['id']
new_size = 123
@ -3581,6 +3687,11 @@ class ShareManagerTestCase(test.TestCase):
}
fake_share_server = 'fake'
size_decrease = int(share['size']) - new_size
mock_notify = self.mock_object(fake_notifier.FakeNotifier, '_notify')
replicas_list = []
if supports_replication:
replicas_list.append(share)
replicas_list.append({'name': 'fake_replica'})
mock_notify.assert_not_called()
@ -3595,6 +3706,17 @@ class ShareManagerTestCase(test.TestCase):
self.mock_object(manager.driver, 'shrink_share')
self.mock_object(manager, '_get_share_server',
mock.Mock(return_value=fake_share_server))
self.mock_object(manager.db, 'share_replicas_get_all_by_share',
mock.Mock(return_value=replicas_list))
reservation_params = {
'gigabytes': -size_decrease,
'project_id': share['project_id'],
'share_type_id': None,
'user_id': share['user_id'],
}
if supports_replication:
reservation_params.update(
{'replica_gigabytes': -size_decrease * 2})
self.share_manager.shrink_share(self.context, share_id, new_size)
@ -3605,8 +3727,7 @@ class ShareManagerTestCase(test.TestCase):
)
quota.QUOTAS.reserve.assert_called_once_with(
mock.ANY, gigabytes=-size_decrease, project_id=share['project_id'],
share_type_id=None, user_id=share['user_id'],
mock.ANY, **reservation_params,
)
quota.QUOTAS.commit.assert_called_once_with(
mock.ANY, mock.ANY, project_id=share['project_id'],
@ -3619,6 +3740,8 @@ class ShareManagerTestCase(test.TestCase):
self.assert_notify_called(mock_notify,
(['INFO', 'share.shrink.start'],
['INFO', 'share.shrink.end']))
(self.share_manager.db.share_replicas_get_all_by_share.
assert_called_once_with(mock.ANY, share['id']))
def test_report_driver_status_driver_handles_ss_false(self):
fake_stats = {'field': 'val'}

5
manila/tests/test_quota.py
View File

@ -711,6 +711,7 @@ class QuotaEngineTestCase(test.TestCase):
def test_current_common_resources(self):
self.assertEqual(
['gigabytes', 'share_group_snapshots', 'share_groups',
'share_networks', 'shares', 'snapshot_gigabytes', 'snapshots'],
['gigabytes', 'replica_gigabytes', 'share_group_snapshots',
'share_groups', 'share_networks', 'share_replicas', 'shares',
'snapshot_gigabytes', 'snapshots'],
quota.QUOTAS.resources)

8
releasenotes/notes/add-tenant-quota-for-share-replicas-and-replicas-size-565ffca315afb6f0.yaml Normal file
View File

@ -0,0 +1,8 @@
---
features:
- |
Added quotas for amount of share replicas and share replica gigabytes.
upgrade:
- |
Two new config options are available for setting default quotas for share
replicas: `quota_share_replicas` and `quota_replica_gigabytes`.