Merge "Add config option to set per_share_size_limit."

manila/api/openstack/api_version_request.py

@@ -164,13 +164,14 @@ REST_API_VERSION_HISTORY = """
              provisioning:min_share_size extra specs,
              which can add minimum and maximum share size restrictions
              on a per share-type granularity.
+    * 2.62 - Added quota control to per share size.
 """
 
 # The minimum and maximum versions of the API supported
 # The default api version request is defined to be the
 # minimum version of the API supported.
 _MIN_API_VERSION = "2.0"
-_MAX_API_VERSION = "2.61"
+_MAX_API_VERSION = "2.62"
 DEFAULT_API_VERSION = _MIN_API_VERSION
 
 

manila/api/openstack/rest_api_version_history.rst

@@ -337,3 +337,7 @@ user documentation.
   Ability to add minimum and maximum share size restrictions which
   can be set on a per share-type granularity. Added new extra specs
   'provisioning:max_share_size' and 'provisioning:min_share_size'.
+
+2.62
+----
+  Added quota control to per share size.

manila/api/v2/quota_sets.py

@@ -329,6 +329,9 @@ class QuotaSetsController(QuotaSetsMixin, wsgi.Controller):
         elif req.api_version_request < api_version.APIVersionRequest("2.53"):
             self._ensure_specific_microversion_args_are_absent(
                 body, ['share_replicas', 'replica_gigabytes'], "2.53")
+        elif req.api_version_request < api_version.APIVersionRequest("2.62"):
+            self._ensure_specific_microversion_args_are_absent(
+                body, ['per_share_gigabytes'], "2.62")
         return self._update(req, id, body)
 
     @wsgi.Controller.api_version('2.7')

manila/api/views/quota_class_sets.py

@@ -22,6 +22,7 @@ class ViewBuilder(common.ViewBuilder):
     _detail_version_modifiers = [
         "add_share_group_quotas",
         "add_share_replica_quotas",
+        "add_per_share_gigabytes_quotas",
     ]
 
     def detail_list(self, request, quota_class_set, quota_class=None):
@@ -52,3 +53,8 @@ class ViewBuilder(common.ViewBuilder):
     def add_share_replica_quotas(self, context, view, quota_class_set):
         view['share_replicas'] = quota_class_set.get('share_replicas')
         view['replica_gigabytes'] = quota_class_set.get('replica_gigabytes')
+
+    @common.ViewBuilder.versioned_method("2.62")
+    def add_per_share_gigabytes_quotas(self, context, view, quota_class_set):
+        view['per_share_gigabytes'] = quota_class_set.get(
+            'per_share_gigabytes')

manila/api/views/quota_sets.py

@@ -22,6 +22,7 @@ class ViewBuilder(common.ViewBuilder):
     _detail_version_modifiers = [
         "add_share_group_quotas",
         "add_share_replica_quotas",
+        "add_per_share_gigabytes_quotas",
     ]
 
     def detail_list(self, request, quota_set, project_id=None,
@@ -59,3 +60,7 @@ class ViewBuilder(common.ViewBuilder):
     def add_share_replica_quotas(self, context, view, quota_class_set):
         view['share_replicas'] = quota_class_set.get('share_replicas')
         view['replica_gigabytes'] = quota_class_set.get('replica_gigabytes')
+
+    @common.ViewBuilder.versioned_method("2.62")
+    def add_per_share_gigabytes_quotas(self, context, view, quota_set):
+        view['per_share_gigabytes'] = quota_set.get('per_share_gigabytes')

manila/db/migrations/alembic/versions/0c23aec99b74_add_per_share_gigabytes_quota_class.py

@@ -0,0 +1,61 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""add_per_share_gigabytes_quota_class
+
+Revision ID: 0c23aec99b74
+Revises: 5aa813ae673d
+Create Date: 2021-01-03 10:01:57.276225
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0c23aec99b74'
+down_revision = '5aa813ae673d'
+
+from alembic import op
+from manila.db.migrations import utils
+from oslo_log import log
+from oslo_utils import timeutils
+from sqlalchemy import MetaData
+
+LOG = log.getLogger(__name__)
+
+
+def upgrade():
+    meta = MetaData()
+    meta.bind = op.get_bind()
+    connection = op.get_bind().connect()
+    quota_classes_table = utils.load_table('quota_classes', connection)
+
+    try:
+        op.bulk_insert
+        (quota_classes_table,
+         [{'created_at': timeutils.utcnow(),
+           'class_name': 'default',
+           'resource': 'per_share_gigabytes',
+           'hard_limit': -1,
+           'deleted': False, }])
+    except Exception:
+        LOG.error("Default per_share_gigabytes row not inserted "
+                  "into the quota_classes.")
+        raise
+
+
+def downgrade():
+    """Don't delete the 'default' entries at downgrade time.
+
+    We don't know if the user had default entries when we started.
+    If they did, we wouldn't want to remove them.  So, the safest
+    thing to do is just leave the 'default' entries at downgrade time.
+    """
+    pass

manila/exception.py

@@ -424,6 +424,12 @@ class SnapshotSizeExceedsAvailableQuota(QuotaError):
         "gigabytes quota.")
 
 
+class ShareSizeExceedsLimit(QuotaError):
+    message = _(
+        "Requested share size %(size)d is larger than "
+        "maximum allowed limit %(limit)d.")
+
+
 class ShareLimitExceeded(QuotaError):
     message = _(
         "Maximum number of shares allowed (%(allowed)d) either per "

manila/quota.py

@@ -39,6 +39,9 @@ quota_opts = [
     cfg.IntOpt('quota_gigabytes',
                default=1000,
                help='Number of share gigabytes allowed per project.'),
+    cfg.IntOpt('quota_per_share_gigabytes',
+               default=-1,
+               help='Max size allowed per share, in gigabytes.'),
     cfg.IntOpt('quota_snapshot_gigabytes',
                default=1000,
                help='Number of snapshot gigabytes allowed per project.'),
@@ -383,6 +386,50 @@ class DbQuotaDriver(object):
 
         return {k: v['limit'] for k, v in quotas.items()}
 
+    def limit_check(self, context, resources, values, project_id=None):
+        """Check simple quota limits.
+
+        For limits--those quotas for which there is no usage
+        synchronization function--this method checks that a set of
+        proposed values are permitted by the limit restriction.
+
+        This method will raise a QuotaResourceUnknown exception if a
+        given resource is unknown or if it is not a simple limit
+        resource.
+
+        If any of the proposed values is over the defined quota, an
+        OverQuota exception will be raised with the sorted list of the
+        resources which are too high.  Otherwise, the method returns
+        nothing.
+
+        :param context: The request context, for access checks.
+        :param resources: A dictionary of the registered resources.
+        :param values: A dictionary of the values to check against the
+                       quota.
+        :param project_id: Specify the project_id if current context
+                           is admin and admin wants to impact on
+                           common user's tenant.
+        """
+
+        # Ensure no value is less than zero
+        unders = [key for key, val in values.items() if val < 0]
+        if unders:
+            raise exception.InvalidQuotaValue(unders=sorted(unders))
+
+        # If project_id is None, then we use the project_id in context
+        if project_id is None:
+            project_id = context.project_id
+
+        quotas = self._get_quotas(context, resources, values.keys(),
+                                  has_sync=False, project_id=project_id)
+        # Check the quotas and construct a list of the resources that
+        # would be put over limit by the desired values
+        overs = [key for key, val in values.items()
+                 if quotas[key] >= 0 and quotas[key] < val]
+        if overs:
+            raise exception.OverQuota(overs=sorted(overs), quotas=quotas,
+                                      usages={})
+
     def reserve(self, context, resources, deltas, expire=None,
                 project_id=None, user_id=None, share_type_id=None,
                 overquota_allowed=False):
@@ -657,7 +704,8 @@ class ReservableResource(BaseResource):
         """
 
         super(ReservableResource, self).__init__(name, flag=flag)
-        self.sync = sync
+        if sync:
+            self.sync = sync
 
 
 class AbsoluteResource(BaseResource):
@@ -876,6 +924,34 @@ class QuotaEngine(object):
 
         return res.count(context, *args, **kwargs)
 
+    def limit_check(self, context, project_id=None, **values):
+        """Check simple quota limits.
+
+        For limits--those quotas for which there is no usage
+        synchronization function--this method checks that a set of
+        proposed values are permitted by the limit restriction.  The
+        values to check are given as keyword arguments, where the key
+        identifies the specific quota limit to check, and the value is
+        the proposed value.
+
+        This method will raise a QuotaResourceUnknown exception if a
+        given resource is unknown or if it is not a simple limit
+        resource.
+
+        If any of the proposed values is over the defined quota, an
+        OverQuota exception will be raised with the sorted list of the
+        resources which are too high.  Otherwise, the method returns
+        nothing.
+
+        :param context: The request context, for access checks.
+        :param project_id: Specify the project_id if current context
+                           is admin and admin wants to impact on
+                           common user's tenant.
+        """
+
+        return self._driver.limit_check(context, self._resources, values,
+                                        project_id=project_id)
+
     def reserve(self, context, expire=None, project_id=None, user_id=None,
                 share_type_id=None, overquota_allowed=False, **deltas):
         """Check quotas and reserve resources.
@@ -1059,6 +1135,8 @@ resources = [
     ReservableResource('shares', '_sync_shares', 'quota_shares'),
     ReservableResource('snapshots', '_sync_snapshots', 'quota_snapshots'),
     ReservableResource('gigabytes', '_sync_gigabytes', 'quota_gigabytes'),
+    ReservableResource('per_share_gigabytes', None,
+                       'quota_per_share_gigabytes'),
     ReservableResource('snapshot_gigabytes', '_sync_snapshot_gigabytes',
                        'quota_snapshot_gigabytes'),
     ReservableResource('share_networks', '_sync_share_networks',

manila/share/api.py

@@ -236,6 +236,8 @@ class API(base.Base):
                          supported=CONF.enabled_share_protocols))
             raise exception.InvalidInput(reason=msg)
 
+        self._check_is_share_size_within_per_share_quota_limit(context, size)
+
         deltas = {'shares': 1, 'gigabytes': size}
         share_type_attributes = self.get_share_attributes_from_share_type(
             share_type)
@@ -2020,6 +2022,17 @@ class API(base.Base):
             }
             raise exception.ShareBusyException(reason=msg)
 
+    def _check_is_share_size_within_per_share_quota_limit(self, context, size):
+        """Raises an exception if share size above per share quota limit."""
+        try:
+            values = {'per_share_gigabytes': size}
+            QUOTAS.limit_check(context, project_id=context.project_id,
+                               **values)
+        except exception.OverQuota as e:
+            quotas = e.kwargs['quotas']
+            raise exception.ShareSizeExceedsLimit(
+                size=size, limit=quotas['per_share_gigabytes'])
+
     def _check_metadata_properties(self, metadata=None):
         if not metadata:
             metadata = {}
@@ -2098,6 +2111,9 @@ class API(base.Base):
                                                     'size': share['size']})
             raise exception.InvalidInput(reason=msg)
 
+        self._check_is_share_size_within_per_share_quota_limit(context,
+                                                               new_size)
+
         # ensure we pass the share_type provisioning filter on size
         try:
             share_type = share_types.get_share_type(

manila/share/manager.py

@@ -2674,6 +2674,16 @@ class ShareManager(manager.SchedulerDependentManager):
                 share_types.provision_filter_on_size(context,
                                                      share_type,
                                                      share_update.get('size'))
+                try:
+                    values = {'per_share_gigabytes': share_update.get('size')}
+                    QUOTAS.limit_check(context, project_id=context.project_id,
+                                       **values)
+                except exception.OverQuota as e:
+                    quotas = e.kwargs['quotas']
+                    LOG.warning("Requested share size %(size)d is larger than "
+                                "maximum allowed limit %(limit)d.",
+                                {'size': share_update.get('size'),
+                                 'limit': quotas['per_share_gigabytes']})
 
             deltas = {
                 'project_id': project_id,

manila/tests/api/v2/test_quota_class_sets.py

@@ -62,6 +62,7 @@ class QuotaSetsControllerTest(test.TestCase):
         ('os-', '2.6', quota_class_sets.QuotaClassSetsControllerLegacy),
         ('', '2.7', quota_class_sets.QuotaClassSetsController),
         ('', '2.53', quota_class_sets.QuotaClassSetsController),
+        ('', '2.62', quota_class_sets.QuotaClassSetsController),
     )
     @ddt.unpack
     def test_show_quota(self, url, version, controller):
@@ -94,6 +95,8 @@ class QuotaSetsControllerTest(test.TestCase):
         if req.api_version_request >= api_version.APIVersionRequest("2.53"):
             expected['quota_class_set']['share_replicas'] = 100
             expected['quota_class_set']['replica_gigabytes'] = 1000
+        if req.api_version_request >= api_version.APIVersionRequest("2.62"):
+            expected['quota_class_set']['per_share_gigabytes'] = -1
 
         result = controller().show(req, self.class_name)
 
@@ -119,6 +122,7 @@ class QuotaSetsControllerTest(test.TestCase):
         ('os-', '2.6', quota_class_sets.QuotaClassSetsControllerLegacy),
         ('', '2.7', quota_class_sets.QuotaClassSetsController),
         ('', '2.53', quota_class_sets.QuotaClassSetsController),
+        ('', '2.62', quota_class_sets.QuotaClassSetsController),
     )
     @ddt.unpack
     def test_update_quota(self, url, version, controller):
@@ -148,6 +152,8 @@ class QuotaSetsControllerTest(test.TestCase):
         if req.api_version_request >= api_version.APIVersionRequest("2.53"):
             expected['quota_class_set']['share_replicas'] = 100
             expected['quota_class_set']['replica_gigabytes'] = 1000
+        if req.api_version_request >= api_version.APIVersionRequest("2.62"):
+            expected['quota_class_set']['per_share_gigabytes'] = -1
 
         update_result = controller().update(
             req, self.class_name, body=body)

manila/tests/api/v2/test_quota_sets.py

@@ -37,6 +37,7 @@ from manila import utils
 CONF = cfg.CONF
 sg_quota_keys = ['share_groups', 'share_group_snapshots']
 replica_quota_keys = ['share_replicas']
+per_share_size_quota_keys = ['per_share_gigabytes']
 
 
 def _get_request(is_admin, user_in_url):
@@ -172,7 +173,6 @@ class QuotaSetsControllerTest(test.TestCase):
                 'reserved': 0,
             },
         }}
-
         for k, v in quotas.items():
             CONF.set_default('quota_' + k, v)
 
@@ -277,6 +277,7 @@ class QuotaSetsControllerTest(test.TestCase):
         ({"quota_set": {"foo": "bar"}}, sg_quota_keys, '2.40'),
         ({"foo": "bar"}, replica_quota_keys, '2.53'),
         ({"quota_set": {"foo": "bar"}}, replica_quota_keys, '2.53'),
+        ({"quota_set": {"foo": "bar"}}, per_share_size_quota_keys, '2.62'),
     )
     @ddt.unpack
     def test__ensure_specific_microversion_args_are_absent_success(
@@ -293,6 +294,8 @@ class QuotaSetsControllerTest(test.TestCase):
         ({"quota_set": {"share_group_snapshots": 8}}, sg_quota_keys, '2.40'),
         ({"quota_set": {"share_replicas": 9}}, replica_quota_keys, '2.53'),
         ({"quota_set": {"share_replicas": 10}}, replica_quota_keys, '2.53'),
+        ({"quota_set": {"per_share_gigabytes": 10}},
+            per_share_size_quota_keys, '2.62'),
     )
     @ddt.unpack
     def test__ensure_specific_microversion_args_are_absent_error(
@@ -351,7 +354,6 @@ class QuotaSetsControllerTest(test.TestCase):
                 },
             }
         }
-
         for k, v in quotas.items():
             CONF.set_default('quota_' + k, v)
 

manila/tests/api/views/test_quota_class_sets.py

@@ -35,6 +35,7 @@ class ViewBuilderTestCase(test.TestCase):
         ("fake_quota_class", "2.40"), (None, "2.40"),
         ("fake_quota_class", "2.39"), (None, "2.39"),
         ("fake_quota_class", "2.53"), (None, "2.53"),
+        ("fake_quota_class", "2.62"), (None, "2.62"),
     )
     @ddt.unpack
     def test_detail_list_with_share_type(self, quota_class, microversion):
@@ -75,6 +76,12 @@ class ViewBuilderTestCase(test.TestCase):
             quota_class_set['share_replicas'] = fake_share_replicas_value
             quota_class_set['replica_gigabytes'] = fake_replica_gigabytes_value
 
+        if req.api_version_request >= api_version.APIVersionRequest("2.62"):
+            fake_per_share_gigabytes = 10
+            expected[self.builder._collection_name][
+                "per_share_gigabytes"] = fake_per_share_gigabytes
+            quota_class_set['per_share_gigabytes'] = fake_per_share_gigabytes
+
         result = self.builder.detail_list(
             req, quota_class_set, quota_class=quota_class)
 

manila/tests/api/views/test_quota_sets.py

@@ -43,6 +43,9 @@ class ViewBuilderTestCase(test.TestCase):
         (None, 'fake_share_type_id', "2.53"),
         ('fake_project_id', None, "2.53"),
         (None, None, "2.53"),
+        (None, 'fake_share_type_id', "2.62"),
+        ('fake_project_id', None, "2.62"),
+        (None, None, "2.62"),
     )
     @ddt.unpack
     def test_detail_list_with_share_type(self, project_id, share_type,
@@ -86,6 +89,12 @@ class ViewBuilderTestCase(test.TestCase):
             quota_set['share_replicas'] = fake_share_replicas_value
             quota_set['replica_gigabytes'] = fake_replica_gigabytes_value
 
+        if req.api_version_request >= api_version.APIVersionRequest("2.62"):
+            fake_per_share_gigabytes = 10
+            expected[self.builder._collection_name]["per_share_gigabytes"] = (
+                fake_per_share_gigabytes)
+            quota_set['per_share_gigabytes'] = fake_per_share_gigabytes
+
         result = self.builder.detail_list(
             req, quota_set, project_id=project_id, share_type=share_type)
 

manila/tests/share/test_api.py

@@ -889,6 +889,31 @@ class ShareAPITestCase(test.TestCase):
             self.context, share_type_id=None,
             shares=1, gigabytes=share_data['size'])
 
+    @ddt.data({'overs': {'per_share_gigabytes': 'fake'},
+               'expected_exception': exception.ShareSizeExceedsLimit})
+    @ddt.unpack
+    def test_create_share_over_per_share_quota(self, overs,
+                                               expected_exception):
+        share, share_data = self._setup_create_mocks()
+
+        quota.CONF.set_default("quota_per_share_gigabytes", 5)
+        share_data['size'] = 20
+
+        usages = {'per_share_gigabytes': {'reserved': 0, 'in_use': 0}}
+        quotas = {'per_share_gigabytes': 10}
+        exc = exception.OverQuota(overs=overs, usages=usages, quotas=quotas)
+        self.mock_object(quota.QUOTAS, 'reserve', mock.Mock(side_effect=exc))
+
+        self.assertRaises(
+            expected_exception,
+            self.api.create,
+            self.context,
+            share_data['share_proto'],
+            share_data['size'],
+            share_data['display_name'],
+            share_data['display_description']
+        )
+
     @ddt.data(exception.QuotaError, exception.InvalidShare)
     def test_create_share_error_on_quota_commit(self, expected_exception):
         share, share_data = self._setup_create_mocks()
@@ -2823,6 +2848,14 @@ class ShareAPITestCase(test.TestCase):
         self.assertRaises(exception.InvalidInput,
                           self.api.extend, self.context, share, new_size)
 
+    def test_extend_share_over_per_share_quota(self):
+        quota.CONF.set_default("quota_per_share_gigabytes", 5)
+        share = db_utils.create_share(status=constants.STATUS_AVAILABLE,
+                                      size=4)
+        new_size = 6
+        self.assertRaises(exception.ShareSizeExceedsLimit,
+                          self.api.extend, self.context, share, new_size)
+
     def test_extend_with_share_type_size_limit(self):
         share = db_utils.create_share(status=constants.STATUS_AVAILABLE,
                                       size=3)

manila/tests/test_exception.py

@@ -602,3 +602,10 @@ class ManilaExceptionResponseCode413(test.TestCase):
         # verify response code for exception.PortLimitExceeded
         e = exception.PortLimitExceeded()
         self.assertEqual(413, e.code)
+
+    def test_per_share_limit_exceeded(self):
+        # verify response code for exception.ShareSizeExceedsLimit
+        size = 779  # amount of share size
+        limit = 775  # amount of allowed share size limit
+        e = exception.ShareSizeExceedsLimit(size=size, limit=limit)
+        self.assertEqual(413, e.code)

manila/tests/test_quota.py

@@ -714,7 +714,7 @@ class QuotaEngineTestCase(test.TestCase):
 
     def test_current_common_resources(self):
         self.assertEqual(
-            ['gigabytes', 'replica_gigabytes', 'share_group_snapshots',
+            ['gigabytes', 'per_share_gigabytes', 'replica_gigabytes',
-             'share_groups', 'share_networks', 'share_replicas', 'shares',
+             'share_group_snapshots', 'share_groups', 'share_networks',
-             'snapshot_gigabytes', 'snapshots'],
+             'share_replicas', 'shares', 'snapshot_gigabytes', 'snapshots'],
             quota.QUOTAS.resources)

releasenotes/notes/add-per-share-gigabytes-quotas-f495eb0b27378660.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    'quota_per_share_gigabytes' config option allows admin to set per share
+    size limit for a project. The default value is -1["No Limit"] always
+    unless changed in manila.conf by admin.