Fix ensure_nova_quota_limits

Change-Id: I8c79c3b35fc24acef74d12ac70076ccc9d4c4919
This commit is contained in:
Federico Ressi 2021-07-19 15:04:10 +02:00
parent 30c39cfdb2
commit 2fe5481f65
1 changed files with 66 additions and 33 deletions

View File

@ -13,8 +13,11 @@
# under the License.
from __future__ import absolute_import
import typing
from oslo_log import log
import tobiko
from tobiko.openstack import keystone
from tobiko.openstack.nova import _client
@ -47,7 +50,12 @@ def set_nova_quota_set(project: keystone.ProjectType = None,
def ensure_nova_quota_limits(project: keystone.ProjectType = None,
user: keystone.UserType = None,
client: _client.NovaClientType = None,
**required: int):
retry_timeout: tobiko.Seconds = None,
retry_interval: tobiko.Seconds = None,
**required_quotas: int):
if not required_quotas:
return
client = _client.nova_client(client)
project = keystone.get_project_id(project=project,
session=client.client.session)
@ -55,41 +63,66 @@ def ensure_nova_quota_limits(project: keystone.ProjectType = None,
if user:
# Must increase project limits before user ones
ensure_nova_quota_limits(project=project, client=client,
**required)
**required_quotas)
for attempt in tobiko.retry(timeout=retry_timeout,
interval=retry_interval,
default_timeout=60.,
default_interval=3.):
actual_limits, expected_limits = get_nova_quota_limits_increase(
project=project, user=user, client=client,
extra_increase=10//attempt.number, **required_quotas)
if expected_limits:
if attempt.is_last:
raise EnsureNovaQuotaLimitsError(
project=project,
actual_limits=actual_limits,
expected_limits=expected_limits)
LOG.info(f"Increase Nova quota limit(s) (project={project}, "
f"user={user}): {actual_limits} -> {expected_limits}...")
try:
set_nova_quota_set(project=project, user=user, client=client,
**expected_limits)
except Exception:
if attempt.is_last:
raise
LOG.exception("Error increasing Nova quota set limits: "
f"{expected_limits}")
else:
LOG.debug(f"Required Nova quota limits are OK: {required_quotas}")
break
else:
raise RuntimeError("Broken retry loop")
class EnsureNovaQuotaLimitsError(tobiko.TobikoException):
message = ("Neutron quota limits lower than "
"expected (project={project}): "
"{actual_limits} != {expected_limits}")
def get_nova_quota_limits_increase(
project: keystone.ProjectType = None,
user: keystone.UserType = None,
client: _client.NovaClientType = None,
extra_increase=0,
**required_quotas: int) \
-> typing.Tuple[typing.Dict[str, int],
typing.Dict[str, int]]:
quota_set = get_nova_quota_set(project=project, user=user,
client=client, detail=True)
actual_limits = {}
increment_limits = {}
for name, needed in required.items():
LOG.debug("Got Nova quota set:\n"
f"{quota_set}")
actual_limits: typing.Dict[str, int] = {}
expected_limits: typing.Dict[str, int] = {}
for name, needed in required_quotas.items():
quota = getattr(quota_set, name)
limit: int = quota['limit']
if limit > 0:
in_use: int = max(0, quota['in_use']) + max(0, quota['reserved'])
required_limit = in_use + needed
limit: int = int(quota['limit'])
if limit >= 0:
in_use = max(0, int(quota['in_use']))
reserved = max(0, int(quota['reserved']))
required_limit = in_use + reserved + needed
if required_limit >= limit:
actual_limits[name] = limit
increment_limits[name] = required_limit + 5
if increment_limits:
LOG.info(f"Increment Nova quota limit(s) (project={project}, "
f"user={user}): {actual_limits} -> {increment_limits}...")
try:
set_nova_quota_set(project=project, user=user, client=client,
**increment_limits)
except Exception:
LOG.exception("Unable to ensure nova quota set limits: "
f"{increment_limits}")
quota_set = get_nova_quota_set(project=project, user=user,
client=client, detail=True)
new_limits = {name: getattr(quota_set, name)['limit']
for name in increment_limits.keys()}
if new_limits == actual_limits:
LOG.error(f"Nova quota limit(s) not changed (project={project}, "
f"user={user}")
else:
LOG.info(f"Nova quota limit(s) changed (project={project}, "
f"user={user}): {actual_limits} -> {new_limits}...")
return quota_set
expected_limits[name] = required_limit + extra_increase
return actual_limits, expected_limits