Fix ensure_nova_quota_limits
Change-Id: I8c79c3b35fc24acef74d12ac70076ccc9d4c4919
This commit is contained in:
parent
30c39cfdb2
commit
2fe5481f65
|
@ -13,8 +13,11 @@
|
||||||
# under the License.
|
# under the License.
|
||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
|
|
||||||
|
import typing
|
||||||
|
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
|
|
||||||
|
import tobiko
|
||||||
from tobiko.openstack import keystone
|
from tobiko.openstack import keystone
|
||||||
from tobiko.openstack.nova import _client
|
from tobiko.openstack.nova import _client
|
||||||
|
|
||||||
|
@ -47,7 +50,12 @@ def set_nova_quota_set(project: keystone.ProjectType = None,
|
||||||
def ensure_nova_quota_limits(project: keystone.ProjectType = None,
|
def ensure_nova_quota_limits(project: keystone.ProjectType = None,
|
||||||
user: keystone.UserType = None,
|
user: keystone.UserType = None,
|
||||||
client: _client.NovaClientType = None,
|
client: _client.NovaClientType = None,
|
||||||
**required: int):
|
retry_timeout: tobiko.Seconds = None,
|
||||||
|
retry_interval: tobiko.Seconds = None,
|
||||||
|
**required_quotas: int):
|
||||||
|
if not required_quotas:
|
||||||
|
return
|
||||||
|
|
||||||
client = _client.nova_client(client)
|
client = _client.nova_client(client)
|
||||||
project = keystone.get_project_id(project=project,
|
project = keystone.get_project_id(project=project,
|
||||||
session=client.client.session)
|
session=client.client.session)
|
||||||
|
@ -55,41 +63,66 @@ def ensure_nova_quota_limits(project: keystone.ProjectType = None,
|
||||||
if user:
|
if user:
|
||||||
# Must increase project limits before user ones
|
# Must increase project limits before user ones
|
||||||
ensure_nova_quota_limits(project=project, client=client,
|
ensure_nova_quota_limits(project=project, client=client,
|
||||||
**required)
|
**required_quotas)
|
||||||
|
|
||||||
|
for attempt in tobiko.retry(timeout=retry_timeout,
|
||||||
|
interval=retry_interval,
|
||||||
|
default_timeout=60.,
|
||||||
|
default_interval=3.):
|
||||||
|
actual_limits, expected_limits = get_nova_quota_limits_increase(
|
||||||
|
project=project, user=user, client=client,
|
||||||
|
extra_increase=10//attempt.number, **required_quotas)
|
||||||
|
if expected_limits:
|
||||||
|
if attempt.is_last:
|
||||||
|
raise EnsureNovaQuotaLimitsError(
|
||||||
|
project=project,
|
||||||
|
actual_limits=actual_limits,
|
||||||
|
expected_limits=expected_limits)
|
||||||
|
LOG.info(f"Increase Nova quota limit(s) (project={project}, "
|
||||||
|
f"user={user}): {actual_limits} -> {expected_limits}...")
|
||||||
|
try:
|
||||||
|
set_nova_quota_set(project=project, user=user, client=client,
|
||||||
|
**expected_limits)
|
||||||
|
except Exception:
|
||||||
|
if attempt.is_last:
|
||||||
|
raise
|
||||||
|
LOG.exception("Error increasing Nova quota set limits: "
|
||||||
|
f"{expected_limits}")
|
||||||
|
else:
|
||||||
|
LOG.debug(f"Required Nova quota limits are OK: {required_quotas}")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
raise RuntimeError("Broken retry loop")
|
||||||
|
|
||||||
|
|
||||||
|
class EnsureNovaQuotaLimitsError(tobiko.TobikoException):
|
||||||
|
message = ("Neutron quota limits lower than "
|
||||||
|
"expected (project={project}): "
|
||||||
|
"{actual_limits} != {expected_limits}")
|
||||||
|
|
||||||
|
|
||||||
|
def get_nova_quota_limits_increase(
|
||||||
|
project: keystone.ProjectType = None,
|
||||||
|
user: keystone.UserType = None,
|
||||||
|
client: _client.NovaClientType = None,
|
||||||
|
extra_increase=0,
|
||||||
|
**required_quotas: int) \
|
||||||
|
-> typing.Tuple[typing.Dict[str, int],
|
||||||
|
typing.Dict[str, int]]:
|
||||||
quota_set = get_nova_quota_set(project=project, user=user,
|
quota_set = get_nova_quota_set(project=project, user=user,
|
||||||
client=client, detail=True)
|
client=client, detail=True)
|
||||||
actual_limits = {}
|
LOG.debug("Got Nova quota set:\n"
|
||||||
increment_limits = {}
|
f"{quota_set}")
|
||||||
for name, needed in required.items():
|
actual_limits: typing.Dict[str, int] = {}
|
||||||
|
expected_limits: typing.Dict[str, int] = {}
|
||||||
|
for name, needed in required_quotas.items():
|
||||||
quota = getattr(quota_set, name)
|
quota = getattr(quota_set, name)
|
||||||
limit: int = quota['limit']
|
limit: int = int(quota['limit'])
|
||||||
if limit > 0:
|
if limit >= 0:
|
||||||
in_use: int = max(0, quota['in_use']) + max(0, quota['reserved'])
|
in_use = max(0, int(quota['in_use']))
|
||||||
required_limit = in_use + needed
|
reserved = max(0, int(quota['reserved']))
|
||||||
|
required_limit = in_use + reserved + needed
|
||||||
if required_limit >= limit:
|
if required_limit >= limit:
|
||||||
actual_limits[name] = limit
|
actual_limits[name] = limit
|
||||||
increment_limits[name] = required_limit + 5
|
expected_limits[name] = required_limit + extra_increase
|
||||||
|
return actual_limits, expected_limits
|
||||||
if increment_limits:
|
|
||||||
LOG.info(f"Increment Nova quota limit(s) (project={project}, "
|
|
||||||
f"user={user}): {actual_limits} -> {increment_limits}...")
|
|
||||||
try:
|
|
||||||
set_nova_quota_set(project=project, user=user, client=client,
|
|
||||||
**increment_limits)
|
|
||||||
except Exception:
|
|
||||||
LOG.exception("Unable to ensure nova quota set limits: "
|
|
||||||
f"{increment_limits}")
|
|
||||||
quota_set = get_nova_quota_set(project=project, user=user,
|
|
||||||
client=client, detail=True)
|
|
||||||
new_limits = {name: getattr(quota_set, name)['limit']
|
|
||||||
for name in increment_limits.keys()}
|
|
||||||
|
|
||||||
if new_limits == actual_limits:
|
|
||||||
LOG.error(f"Nova quota limit(s) not changed (project={project}, "
|
|
||||||
f"user={user}")
|
|
||||||
else:
|
|
||||||
LOG.info(f"Nova quota limit(s) changed (project={project}, "
|
|
||||||
f"user={user}): {actual_limits} -> {new_limits}...")
|
|
||||||
|
|
||||||
return quota_set
|
|
||||||
|
|
Loading…
Reference in New Issue