diff --git a/tobiko/openstack/base/__init__.py b/tobiko/openstack/base/__init__.py new file mode 100644 index 000000000..0c45898c9 --- /dev/null +++ b/tobiko/openstack/base/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/tobiko/openstack/stacks/_fixture.py b/tobiko/openstack/base/_fixture.py similarity index 58% rename from tobiko/openstack/stacks/_fixture.py rename to tobiko/openstack/base/_fixture.py index 5a4c5d3da..7fca33cdb 100644 --- a/tobiko/openstack/stacks/_fixture.py +++ b/tobiko/openstack/base/_fixture.py @@ -14,18 +14,94 @@ from __future__ import absolute_import import abc +import collections import typing from oslo_log import log import tobiko from tobiko import config +from tobiko.openstack import keystone +from tobiko.openstack.neutron import _quota_set as neutron_quota +from tobiko.openstack.nova import _quota_set as nova_quota LOG = log.getLogger(__name__) -class ResourceFixture(tobiko.SharedFixture, abc.ABC): +class InvalidFixtureError(tobiko.TobikoException): + message = "invalid fixture {name!r}" + + +class BaseResourceFixture(tobiko.SharedFixture): + """Base class for fixtures both types: those which uses heat stacks and + those which are not. + """ + client: keystone.KeystoneClient = None + project: typing.Optional[str] = None + user: typing.Optional[str] = None + + def setup_fixture(self): + self.setup_client() + self.setup_project() + self.setup_user() + + def setup_project(self): + if self.project is None: + self.project = keystone.get_project_id(session=self.session) + + def setup_user(self): + if self.user is None: + self.user = keystone.get_user_id(session=self.session) + + @property + def session(self): + return self.setup_client().session + + def setup_client(self) -> keystone.KeystoneClient: + client = self.client + # NOTE(slaweq): it seems that due to bug + # https://github.com/python/mypy/issues/11673 + # in mypy this line is causing arg-type error so lets + # ignore it for now + if not isinstance( + client, keystone.KeystoneClient): # type: ignore[arg-type] + self.client = client = keystone.keystone_client(self.client) + return client + + def ensure_quota_limits(self): + """Ensures quota limits before creating a new stack + """ + try: + self.ensure_neutron_quota_limits() + self.ensure_nova_quota_limits() + except (nova_quota.EnsureNovaQuotaLimitsError, + neutron_quota.EnsureNeutronQuotaLimitsError) as ex: + raise InvalidFixtureError(name=self.fixture_name) from ex + + def ensure_neutron_quota_limits(self): + required_quota_set = self.neutron_required_quota_set + if required_quota_set: + neutron_quota.ensure_neutron_quota_limits(project=self.project, + **required_quota_set) + + def ensure_nova_quota_limits(self): + required_quota_set = self.nova_required_quota_set + if required_quota_set: + nova_quota.ensure_nova_quota_limits(project=self.project, + user=self.user, + **required_quota_set) + + @property + def neutron_required_quota_set(self) -> typing.Dict[str, int]: + return collections.defaultdict(int) + + @property + def nova_required_quota_set(self) -> typing.Dict[str, int]: + return collections.defaultdict(int) + + +class ResourceFixture(BaseResourceFixture, abc.ABC): """Base class for fixtures intended to manage Openstack resources not created using Heat, but with openstacksdk or other component clients (such as neutronclient, novaclient, manilaclient, etc). @@ -41,11 +117,11 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC): Child classes must define any other attributes required by the resource_create, resource_delete and resource_find methods. Examples: prefixes and default_prefixlen are needed for subnet_pools; description and - rules are needed for secutiry_groups; etc. + rules are needed for security_groups; etc. Child classes must define the resource_create, resource_delete and resource_find methods. In case of resource_create and resource_find, they - should return an object with the type defined for self._resouce. + should return an object with the type defined for self._resource. Child classes may optionally implement simple properties to access to resource_id and resource using a more representative name (these properties @@ -84,6 +160,7 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC): pass def setup_fixture(self): + super().setup_fixture() self.name = self.fixture_name if config.get_bool_env('TOBIKO_PREVENT_CREATE'): LOG.debug("%r should have been already created: %r", @@ -98,6 +175,9 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC): tobiko.addme_to_shared_resource(__name__, self.name) def try_create_resource(self): + # Ensure quota limits are OK just in time before start creating + # a new stack + self.ensure_quota_limits() if not self.resource: self._resource = self.resource_create() diff --git a/tobiko/openstack/heat/__init__.py b/tobiko/openstack/heat/__init__.py index ee19d0e81..4b9c90c1f 100644 --- a/tobiko/openstack/heat/__init__.py +++ b/tobiko/openstack/heat/__init__.py @@ -41,7 +41,6 @@ HeatStackNotFound = _stack.HeatStackNotFound heat_stack_parameters = _stack.heat_stack_parameters find_stack = _stack.find_stack list_stacks = _stack.list_stacks -InvalidStackError = _stack.InvalidStackError INIT_IN_PROGRESS = _stack.INIT_IN_PROGRESS INIT_COMPLETE = _stack.INIT_COMPLETE CREATE_IN_PROGRESS = _stack.CREATE_IN_PROGRESS diff --git a/tobiko/openstack/heat/_stack.py b/tobiko/openstack/heat/_stack.py index e7c47a507..ca38e88e9 100644 --- a/tobiko/openstack/heat/_stack.py +++ b/tobiko/openstack/heat/_stack.py @@ -13,7 +13,6 @@ # under the License. from __future__ import absolute_import -import collections from collections import abc import random import time @@ -28,8 +27,7 @@ from tobiko import config from tobiko.openstack.heat import _client from tobiko.openstack.heat import _template from tobiko.openstack import keystone -from tobiko.openstack import neutron -from tobiko.openstack import nova +from tobiko.openstack.base import _fixture as base_fixture LOG = log.getLogger(__name__) @@ -110,7 +108,7 @@ def find_stack(client: _client.HeatClientType = None, @keystone.skip_unless_has_keystone_credentials() -class HeatStackFixture(tobiko.SharedFixture): +class HeatStackFixture(base_fixture.BaseResourceFixture): """Manages Heat stacks.""" client: _client.HeatClientType = None @@ -124,8 +122,6 @@ class HeatStackFixture(tobiko.SharedFixture): stack: typing.Optional[StackType] = None stack_name: typing.Optional[str] = None parameters: typing.Optional['HeatStackParametersFixture'] = None - project: typing.Optional[str] = None - user: typing.Optional[str] = None output_needs_stack_complete: bool = True def __init__( @@ -150,12 +146,10 @@ class HeatStackFixture(tobiko.SharedFixture): self.wait_interval = wait_interval def setup_fixture(self): + super().setup_fixture() self.setup_stack_name() self.setup_template() self.setup_parameters() - self.setup_client() - self.setup_project() - self.setup_user() self.setup_stack() def setup_template(self): @@ -180,14 +174,6 @@ class HeatStackFixture(tobiko.SharedFixture): def session(self): return self.setup_client().http_client.session - def setup_project(self): - if self.project is None: - self.project = keystone.get_project_id(session=self.session) - - def setup_user(self): - if self.user is None: - self.user = keystone.get_user_id(session=self.session) - def setup_stack(self) -> stacks.Stack: stack = self.create_stack() tobiko.addme_to_shared_resource(__name__, stack.stack_name) @@ -207,7 +193,7 @@ class HeatStackFixture(tobiko.SharedFixture): try: stack = self.try_create_stack() break - except InvalidStackError: + except base_fixture.InvalidFixtureError: LOG.exception(f"Error creating stack '{self.stack_name}'", exc_info=1) if attempt.is_last: @@ -290,7 +276,7 @@ class HeatStackFixture(tobiko.SharedFixture): f"id='{stack_id}'.") try: stack = self.validate_created_stack() - except InvalidStackError as ex: + except base_fixture.InvalidFixtureError as ex: LOG.debug(f'Deleting invalid stack (name={self.stack_name}, "' f'"id={stack_id}): {ex}') # the stack shelf counter does not need to be decreased here, @@ -517,37 +503,6 @@ class HeatStackFixture(tobiko.SharedFixture): message = "Object {!r} has no attribute {!r}".format(self, name) raise AttributeError(message) - def ensure_quota_limits(self): - """Ensures quota limits before creating a new stack - """ - try: - self.ensure_neutron_quota_limits() - self.ensure_nova_quota_limits() - except (nova.EnsureNovaQuotaLimitsError, - neutron.EnsureNeutronQuotaLimitsError) as ex: - raise InvalidStackError(name=self.stack_name) from ex - - def ensure_neutron_quota_limits(self): - required_quota_set = self.neutron_required_quota_set - if required_quota_set: - neutron.ensure_neutron_quota_limits(project=self.project, - **required_quota_set) - - def ensure_nova_quota_limits(self): - required_quota_set = self.nova_required_quota_set - if required_quota_set: - nova.ensure_nova_quota_limits(project=self.project, - user=self.user, - **required_quota_set) - - @property - def neutron_required_quota_set(self) -> typing.Dict[str, int]: - return collections.defaultdict(int) - - @property - def nova_required_quota_set(self) -> typing.Dict[str, int]: - return collections.defaultdict(int) - class HeatStackKeyError(tobiko.TobikoException): message = "key {key!r} not found in stack {name!r}" @@ -717,11 +672,7 @@ class HeatStackNotFound(HeatStackError): message = "stack {name!r} not found" -class InvalidStackError(HeatStackError): - message = "invalid stack {name!r}" - - -class InvalidHeatStackStatus(InvalidStackError): +class InvalidHeatStackStatus(base_fixture.InvalidFixtureError): message = ("stack {name!r} status {observed!r} not in {expected!r}\n" "{status_reason!s}") diff --git a/tobiko/openstack/stacks/_neutron.py b/tobiko/openstack/stacks/_neutron.py index 4f762b775..a226803b0 100644 --- a/tobiko/openstack/stacks/_neutron.py +++ b/tobiko/openstack/stacks/_neutron.py @@ -26,8 +26,8 @@ import tobiko from tobiko import config from tobiko.openstack import heat from tobiko.openstack import neutron +from tobiko.openstack.base import _fixture as base_fixture from tobiko.openstack.stacks import _hot -from tobiko.openstack.stacks import _fixture from tobiko.shell import ip from tobiko.shell import sh from tobiko.shell import ssh @@ -284,7 +284,7 @@ class RouterNoSnatStackFixture(RouterStackFixture): @neutron.skip_if_missing_networking_extensions('subnet_allocation') -class SubnetPoolFixture(_fixture.ResourceFixture): +class SubnetPoolFixture(base_fixture.ResourceFixture): """Neutron Subnet Pool Fixture. A subnet pool is a dependency of network fixtures with either IPv4 or @@ -583,7 +583,7 @@ class SecurityGroupsFixture(heat.HeatStackFixture): @neutron.skip_if_missing_networking_extensions('stateful-security-group') -class StatelessSecurityGroupFixture(_fixture.ResourceFixture): +class StatelessSecurityGroupFixture(base_fixture.ResourceFixture): """Neutron Stateless Security Group Fixture. This SG will by default allow SSH and ICMP to the instance and also @@ -615,6 +615,12 @@ class StatelessSecurityGroupFixture(_fixture.ResourceFixture): self.description = description or self.description self.rules = rules or self.rules + @property + def neutron_required_quota_set(self) -> typing.Dict[str, int]: + requirements = super().neutron_required_quota_set + requirements['security_group'] += 1 + return requirements + @property def security_group_id(self): return self.resource_id diff --git a/tobiko/openstack/stacks/_nova.py b/tobiko/openstack/stacks/_nova.py index f252f2a88..02b76ccb1 100644 --- a/tobiko/openstack/stacks/_nova.py +++ b/tobiko/openstack/stacks/_nova.py @@ -28,6 +28,7 @@ from tobiko.openstack import glance from tobiko.openstack import heat from tobiko.openstack import neutron from tobiko.openstack import nova +from tobiko.openstack.base import _fixture as base_fixture from tobiko.openstack.stacks import _hot from tobiko.openstack.stacks import _neutron from tobiko.shell import curl @@ -271,7 +272,8 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC): self.validate_different_host_scheduler_hints( hypervisor=hypervisor) except nova.MigrateServerError as ex: - raise heat.InvalidStackError(name=self.stack_name) from ex + raise base_fixture.InvalidFixtureError( + name=self.stack_name) from ex def validate_same_host_scheduler_hints(self, hypervisor): if self.same_host: diff --git a/tobiko/tests/scenario/nova/test_server.py b/tobiko/tests/scenario/nova/test_server.py index 0791e9711..02624f5a8 100644 --- a/tobiko/tests/scenario/nova/test_server.py +++ b/tobiko/tests/scenario/nova/test_server.py @@ -22,10 +22,10 @@ import pytest import testtools import tobiko -from tobiko.openstack import heat from tobiko.openstack import keystone from tobiko.openstack import nova from tobiko.openstack import stacks +from tobiko.openstack.base import _fixture as base_fixture from tobiko.shell import ping @@ -177,7 +177,7 @@ class CirrosServerStackFixture(stacks.CirrosServerStackFixture): try: nova.activate_server(server) except nova.WaitForServerStatusTimeout as ex: - raise heat.InvalidStackError( + raise base_fixture.InvalidFixtureError( name=self.stack_name) from ex return stack