diff --git a/tobiko/openstack/nova/_quota_set.py b/tobiko/openstack/nova/_quota_set.py index 9b69a2f6a..61efbe7eb 100644 --- a/tobiko/openstack/nova/_quota_set.py +++ b/tobiko/openstack/nova/_quota_set.py @@ -13,8 +13,11 @@ # under the License. from __future__ import absolute_import +import typing + from oslo_log import log +import tobiko from tobiko.openstack import keystone from tobiko.openstack.nova import _client @@ -47,7 +50,12 @@ def set_nova_quota_set(project: keystone.ProjectType = None, def ensure_nova_quota_limits(project: keystone.ProjectType = None, user: keystone.UserType = None, client: _client.NovaClientType = None, - **required: int): + retry_timeout: tobiko.Seconds = None, + retry_interval: tobiko.Seconds = None, + **required_quotas: int): + if not required_quotas: + return + client = _client.nova_client(client) project = keystone.get_project_id(project=project, session=client.client.session) @@ -55,41 +63,66 @@ def ensure_nova_quota_limits(project: keystone.ProjectType = None, if user: # Must increase project limits before user ones ensure_nova_quota_limits(project=project, client=client, - **required) + **required_quotas) + for attempt in tobiko.retry(timeout=retry_timeout, + interval=retry_interval, + default_timeout=60., + default_interval=3.): + actual_limits, expected_limits = get_nova_quota_limits_increase( + project=project, user=user, client=client, + extra_increase=10//attempt.number, **required_quotas) + if expected_limits: + if attempt.is_last: + raise EnsureNovaQuotaLimitsError( + project=project, + actual_limits=actual_limits, + expected_limits=expected_limits) + LOG.info(f"Increase Nova quota limit(s) (project={project}, " + f"user={user}): {actual_limits} -> {expected_limits}...") + try: + set_nova_quota_set(project=project, user=user, client=client, + **expected_limits) + except Exception: + if attempt.is_last: + raise + LOG.exception("Error increasing Nova quota set limits: " + f"{expected_limits}") + else: + LOG.debug(f"Required Nova quota limits are OK: {required_quotas}") + break + else: + raise RuntimeError("Broken retry loop") + + +class EnsureNovaQuotaLimitsError(tobiko.TobikoException): + message = ("Neutron quota limits lower than " + "expected (project={project}): " + "{actual_limits} != {expected_limits}") + + +def get_nova_quota_limits_increase( + project: keystone.ProjectType = None, + user: keystone.UserType = None, + client: _client.NovaClientType = None, + extra_increase=0, + **required_quotas: int) \ + -> typing.Tuple[typing.Dict[str, int], + typing.Dict[str, int]]: quota_set = get_nova_quota_set(project=project, user=user, client=client, detail=True) - actual_limits = {} - increment_limits = {} - for name, needed in required.items(): + LOG.debug("Got Nova quota set:\n" + f"{quota_set}") + actual_limits: typing.Dict[str, int] = {} + expected_limits: typing.Dict[str, int] = {} + for name, needed in required_quotas.items(): quota = getattr(quota_set, name) - limit: int = quota['limit'] - if limit > 0: - in_use: int = max(0, quota['in_use']) + max(0, quota['reserved']) - required_limit = in_use + needed + limit: int = int(quota['limit']) + if limit >= 0: + in_use = max(0, int(quota['in_use'])) + reserved = max(0, int(quota['reserved'])) + required_limit = in_use + reserved + needed if required_limit >= limit: actual_limits[name] = limit - increment_limits[name] = required_limit + 5 - - if increment_limits: - LOG.info(f"Increment Nova quota limit(s) (project={project}, " - f"user={user}): {actual_limits} -> {increment_limits}...") - try: - set_nova_quota_set(project=project, user=user, client=client, - **increment_limits) - except Exception: - LOG.exception("Unable to ensure nova quota set limits: " - f"{increment_limits}") - quota_set = get_nova_quota_set(project=project, user=user, - client=client, detail=True) - new_limits = {name: getattr(quota_set, name)['limit'] - for name in increment_limits.keys()} - - if new_limits == actual_limits: - LOG.error(f"Nova quota limit(s) not changed (project={project}, " - f"user={user}") - else: - LOG.info(f"Nova quota limit(s) changed (project={project}, " - f"user={user}): {actual_limits} -> {new_limits}...") - - return quota_set + expected_limits[name] = required_limit + extra_increase + return actual_limits, expected_limits