Fix ensure_nova_quota_limits

Change-Id: I8c79c3b35fc24acef74d12ac70076ccc9d4c4919

tobiko/openstack/nova/_quota_set.py

@@ -13,8 +13,11 @@
 #    under the License.
 from __future__ import absolute_import
 
+import typing
+
 from oslo_log import log
 
+import tobiko
 from tobiko.openstack import keystone
 from tobiko.openstack.nova import _client
 
@@ -47,7 +50,12 @@ def set_nova_quota_set(project: keystone.ProjectType = None,
 def ensure_nova_quota_limits(project: keystone.ProjectType = None,
                              user: keystone.UserType = None,
                              client: _client.NovaClientType = None,
-                             **required: int):
+                             retry_timeout: tobiko.Seconds = None,
+                             retry_interval: tobiko.Seconds = None,
+                             **required_quotas: int):
+    if not required_quotas:
+        return
+
     client = _client.nova_client(client)
     project = keystone.get_project_id(project=project,
                                       session=client.client.session)
@@ -55,41 +63,66 @@ def ensure_nova_quota_limits(project: keystone.ProjectType = None,
     if user:
         # Must increase project limits before user ones
         ensure_nova_quota_limits(project=project, client=client,
-                                 **required)
+                                 **required_quotas)
 
+    for attempt in tobiko.retry(timeout=retry_timeout,
+                                interval=retry_interval,
+                                default_timeout=60.,
+                                default_interval=3.):
+        actual_limits, expected_limits = get_nova_quota_limits_increase(
+            project=project, user=user, client=client,
+            extra_increase=10//attempt.number, **required_quotas)
+        if expected_limits:
+            if attempt.is_last:
+                raise EnsureNovaQuotaLimitsError(
+                    project=project,
+                    actual_limits=actual_limits,
+                    expected_limits=expected_limits)
+            LOG.info(f"Increase Nova quota limit(s) (project={project}, "
+                     f"user={user}): {actual_limits} -> {expected_limits}...")
+            try:
+                set_nova_quota_set(project=project, user=user, client=client,
+                                   **expected_limits)
+            except Exception:
+                if attempt.is_last:
+                    raise
+                LOG.exception("Error increasing Nova quota set limits: "
+                              f"{expected_limits}")
+        else:
+            LOG.debug(f"Required Nova quota limits are OK: {required_quotas}")
+            break
+    else:
+        raise RuntimeError("Broken retry loop")
+
+
+class EnsureNovaQuotaLimitsError(tobiko.TobikoException):
+    message = ("Neutron quota limits lower than "
+               "expected (project={project}): "
+               "{actual_limits} != {expected_limits}")
+
+
+def get_nova_quota_limits_increase(
+        project: keystone.ProjectType = None,
+        user: keystone.UserType = None,
+        client: _client.NovaClientType = None,
+        extra_increase=0,
+        **required_quotas: int) \
+        -> typing.Tuple[typing.Dict[str, int],
+                        typing.Dict[str, int]]:
     quota_set = get_nova_quota_set(project=project, user=user,
                                    client=client, detail=True)
-    actual_limits = {}
-    increment_limits = {}
-    for name, needed in required.items():
+    LOG.debug("Got Nova quota set:\n"
+              f"{quota_set}")
+    actual_limits: typing.Dict[str, int] = {}
+    expected_limits: typing.Dict[str, int] = {}
+    for name, needed in required_quotas.items():
         quota = getattr(quota_set, name)
-        limit: int = quota['limit']
-        if limit > 0:
-            in_use: int = max(0, quota['in_use']) + max(0, quota['reserved'])
-            required_limit = in_use + needed
+        limit: int = int(quota['limit'])
+        if limit >= 0:
+            in_use = max(0, int(quota['in_use']))
+            reserved = max(0, int(quota['reserved']))
+            required_limit = in_use + reserved + needed
             if required_limit >= limit:
                 actual_limits[name] = limit
-                increment_limits[name] = required_limit + 5
-
-    if increment_limits:
-        LOG.info(f"Increment Nova quota limit(s) (project={project}, "
-                 f"user={user}): {actual_limits} -> {increment_limits}...")
-        try:
-            set_nova_quota_set(project=project, user=user, client=client,
-                               **increment_limits)
-        except Exception:
-            LOG.exception("Unable to ensure nova quota set limits: "
-                          f"{increment_limits}")
-        quota_set = get_nova_quota_set(project=project, user=user,
-                                       client=client, detail=True)
-        new_limits = {name: getattr(quota_set, name)['limit']
-                      for name in increment_limits.keys()}
-
-        if new_limits == actual_limits:
-            LOG.error(f"Nova quota limit(s) not changed (project={project}, "
-                      f"user={user}")
-        else:
-            LOG.info(f"Nova quota limit(s) changed (project={project}, "
-                     f"user={user}): {actual_limits} -> {new_limits}...")
-
-    return quota_set
+                expected_limits[name] = required_limit + extra_increase
+    return actual_limits, expected_limits