Fix manila OverQuota issue while managing shares

Fixes the OverQuota issue while managing shares. Now, when a share
manage request is received and there is no available quota to this
resource, manila will allow the quotas to be exceeded and the
administrator will need to adjust it.

Change-Id: If6edfa79965f1a0e6959b436c53a714217d23f7d
Closes-Bug: #1863298

manila/db/api.py

@@ -250,12 +250,13 @@ def quota_usage_update(context, project_id, user_id, resource,
 
 def quota_reserve(context, resources, quotas, user_quotas, share_type_quotas,
                   deltas, expire, until_refresh, max_age,
-                  project_id=None, user_id=None, share_type_id=None):
+                  project_id=None, user_id=None, share_type_id=None,
+                  overquota_allowed=False):
     """Check quotas and create appropriate reservations."""
     return IMPL.quota_reserve(
         context, resources, quotas, user_quotas, share_type_quotas, deltas,
         expire, until_refresh, max_age, project_id=project_id, user_id=user_id,
-        share_type_id=share_type_id)
+        share_type_id=share_type_id, overquota_allowed=overquota_allowed)
 
 
 def reservation_commit(context, reservations, project_id=None, user_id=None,

manila/db/sqlalchemy/api.py

@@ -933,16 +933,19 @@ def _get_project_quota_usages(context, session, project_id):
 @require_context
 def quota_reserve(context, resources, project_quotas, user_quotas,
                   share_type_quotas, deltas, expire, until_refresh,
-                  max_age, project_id=None, user_id=None, share_type_id=None):
+                  max_age, project_id=None, user_id=None, share_type_id=None,
+                  overquota_allowed=False):
     user_reservations = _quota_reserve(
         context, resources, project_quotas, user_quotas,
-        deltas, expire, until_refresh, max_age, project_id, user_id=user_id)
+        deltas, expire, until_refresh, max_age, project_id, user_id=user_id,
+        overquota_allowed=overquota_allowed)
     if share_type_id:
         try:
             st_reservations = _quota_reserve(
                 context, resources, project_quotas, share_type_quotas,
                 deltas, expire, until_refresh, max_age, project_id,
-                share_type_id=share_type_id)
+                share_type_id=share_type_id,
+                overquota_allowed=overquota_allowed)
         except exception.OverQuota:
             with excutils.save_and_reraise_exception():
                 # rollback previous reservations
@@ -956,7 +959,8 @@ def quota_reserve(context, resources, project_quotas, user_quotas,
 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 def _quota_reserve(context, resources, project_quotas, user_or_st_quotas,
                    deltas, expire, until_refresh,
-                   max_age, project_id=None, user_id=None, share_type_id=None):
+                   max_age, project_id=None, user_id=None, share_type_id=None,
+                   overquota_allowed=False):
     elevated = context.elevated()
     session = get_session()
     with session.begin():
@@ -1104,6 +1108,21 @@ def _quota_reserve(context, resources, project_quotas, user_or_st_quotas,
                   user_or_st_quotas[res] < delta +
                   user_or_st_usages[res].total)]
 
+        # NOTE(carloss): If OverQuota is allowed, there is no problem to exceed
+        # the quotas, so we reset the overs list and LOG it.
+        if overs and overquota_allowed:
+            msg = _("The service has identified one or more exceeded "
+                    "quotas. Please check the quotas for project "
+                    "%(project_id)s, user %(user_id)s and share type "
+                    "%(share_type_id)s, and adjust them if "
+                    "necessary.") % {
+                "project_id": project_id,
+                "user_id": user_id,
+                "share_type_id": share_type_id
+            }
+            LOG.warning(msg)
+            overs = []
+
         # NOTE(Vek): The quota check needs to be in the transaction,
         #            but the transaction doesn't fail just because
         #            we're over quota, so the OverQuota raise is

manila/quota.py

@@ -384,7 +384,8 @@ class DbQuotaDriver(object):
         return {k: v['limit'] for k, v in quotas.items()}
 
     def reserve(self, context, resources, deltas, expire=None,
-                project_id=None, user_id=None, share_type_id=None):
+                project_id=None, user_id=None, share_type_id=None,
+                overquota_allowed=False):
         """Check quotas and reserve resources.
 
         For counting quotas--those quotas for which there is a usage
@@ -465,7 +466,7 @@ class DbQuotaDriver(object):
             context, resources, quotas, user_quotas, share_type_quotas,
             deltas, expire, CONF.until_refresh, CONF.max_age,
             project_id=project_id, user_id=user_id,
-            share_type_id=share_type_id)
+            share_type_id=share_type_id, overquota_allowed=overquota_allowed)
 
     def commit(self, context, reservations, project_id=None, user_id=None,
                share_type_id=None):
@@ -876,7 +877,7 @@ class QuotaEngine(object):
         return res.count(context, *args, **kwargs)
 
     def reserve(self, context, expire=None, project_id=None, user_id=None,
-                share_type_id=None, **deltas):
+                share_type_id=None, overquota_allowed=False, **deltas):
         """Check quotas and reserve resources.
 
         For counting quotas--those quotas for which there is a usage
@@ -917,6 +918,7 @@ class QuotaEngine(object):
             project_id=project_id,
             user_id=user_id,
             share_type_id=share_type_id,
+            overquota_allowed=overquota_allowed
         )
 
         LOG.debug("Created reservations %s", reservations)

manila/share/manager.py

@@ -2604,7 +2604,14 @@ class ShareManager(manager.SchedulerDependentManager):
                 deltas.update({'share_replicas': 1,
                                'replica_gigabytes': share_update['size']})
 
-            reservations = QUOTAS.reserve(context, **deltas)
+            # NOTE(carloss): Allowing OverQuota to do not compromise this
+            # operation. If this hit OverQuota error while managing a share,
+            # the admin would need to reset the state of the share and
+            # delete or force delete the share (bug 1863298). Allowing
+            # OverQuota makes this operation work properly and the admin will
+            # need to adjust quotas afterwards.
+            reservations = QUOTAS.reserve(context, overquota_allowed=True,
+                                          **deltas)
             QUOTAS.commit(
                 context, reservations, project_id=project_id,
                 share_type_id=share_instance['share_type_id'],

manila/tests/share/test_manager.py

@@ -2845,6 +2845,7 @@ class ShareManagerTestCase(test.TestCase):
             'shares': 1,
             'gigabytes': driver_data['size'],
             'share_type_id': share['instance']['share_type_id'],
+            'overquota_allowed': True
         }
         if replication_type:
             expected_deltas.update({'share_replicas': 1,

manila/tests/test_quota.py

@@ -354,6 +354,7 @@ class DbQuotaDriverTestCase(test.TestCase):
             'project_id': self.ctxt.project_id,
             'user_id': self.ctxt.user_id,
             'share_type_id': None,
+            'overquota_allowed': False
         }
         expected_kwargs.update(kwargs)
         st_quotas = st_quotas if kwargs.get('share_type_id') else {}
@@ -625,7 +626,8 @@ class QuotaEngineTestCase(test.TestCase):
         self.driver.reserve.assert_called_once_with(
             self.ctxt, self.engine._resources, {'delta1': 1, 'delta2': 2},
             expire='fake_expire', project_id=self.project_id,
-            user_id=self.user_id, share_type_id=self.share_type_id)
+            user_id=self.user_id, share_type_id=self.share_type_id,
+            overquota_allowed=False)
 
     @ddt.data(Exception('FakeException'), [None])
     def test_commit(self, side_effect):

releasenotes/notes/bug-1863298-fix-manage-overquota-issue-37031a593b66f8ba.yaml

@@ -0,0 +1,10 @@
+---
+fixes:
+  - |
+    Fixed an issue while bringing shares under Manila management. Now, when
+    a share is being managed and there is no available quota to complete
+    this operation, the service will allow the quotas to be exceeded and the
+    operation will be completed. The administrator will need to adjust the
+    quotas after. Please see
+    `Launchpad bug <https://bugs.launchpad.net/manila/+bug/1863298>`_ for
+    more details.