From 26b6909ceb836b58d67c3fb4d5803e0a561c07ac Mon Sep 17 00:00:00 2001 From: Slawek Kaplonski Date: Fri, 15 Mar 2024 11:51:40 +0100 Subject: [PATCH] Introduce new BaseResourceFixture class This new class is base for both: HeatStackFixture and ResourceFixture classes. It contains some common pieces of code like e.g. ensuring that quota is set as required by the fixture. This is done because fixture StatelessSecurityGroup, which don't use HeatStackFixture needs to set quota for security groups in Neutron but ensuring neutron (and nova) quotas are set correctly were done only in the HeatStackFixture class so far. Change-Id: Id3d3207f098853469bef87020fc017bec5aaba93 --- tobiko/openstack/base/__init__.py | 15 ++++ tobiko/openstack/{stacks => base}/_fixture.py | 86 ++++++++++++++++++- tobiko/openstack/heat/__init__.py | 1 - tobiko/openstack/heat/_stack.py | 61 ++----------- tobiko/openstack/stacks/_neutron.py | 12 ++- tobiko/openstack/stacks/_nova.py | 4 +- tobiko/tests/scenario/nova/test_server.py | 4 +- 7 files changed, 118 insertions(+), 65 deletions(-) create mode 100644 tobiko/openstack/base/__init__.py rename tobiko/openstack/{stacks => base}/_fixture.py (58%) diff --git a/tobiko/openstack/base/__init__.py b/tobiko/openstack/base/__init__.py new file mode 100644 index 000000000..0c45898c9 --- /dev/null +++ b/tobiko/openstack/base/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/tobiko/openstack/stacks/_fixture.py b/tobiko/openstack/base/_fixture.py similarity index 58% rename from tobiko/openstack/stacks/_fixture.py rename to tobiko/openstack/base/_fixture.py index 5a4c5d3da..7fca33cdb 100644 --- a/tobiko/openstack/stacks/_fixture.py +++ b/tobiko/openstack/base/_fixture.py @@ -14,18 +14,94 @@ from __future__ import absolute_import import abc +import collections import typing from oslo_log import log import tobiko from tobiko import config +from tobiko.openstack import keystone +from tobiko.openstack.neutron import _quota_set as neutron_quota +from tobiko.openstack.nova import _quota_set as nova_quota LOG = log.getLogger(__name__) -class ResourceFixture(tobiko.SharedFixture, abc.ABC): +class InvalidFixtureError(tobiko.TobikoException): + message = "invalid fixture {name!r}" + + +class BaseResourceFixture(tobiko.SharedFixture): + """Base class for fixtures both types: those which uses heat stacks and + those which are not. + """ + client: keystone.KeystoneClient = None + project: typing.Optional[str] = None + user: typing.Optional[str] = None + + def setup_fixture(self): + self.setup_client() + self.setup_project() + self.setup_user() + + def setup_project(self): + if self.project is None: + self.project = keystone.get_project_id(session=self.session) + + def setup_user(self): + if self.user is None: + self.user = keystone.get_user_id(session=self.session) + + @property + def session(self): + return self.setup_client().session + + def setup_client(self) -> keystone.KeystoneClient: + client = self.client + # NOTE(slaweq): it seems that due to bug + # https://github.com/python/mypy/issues/11673 + # in mypy this line is causing arg-type error so lets + # ignore it for now + if not isinstance( + client, keystone.KeystoneClient): # type: ignore[arg-type] + self.client = client = keystone.keystone_client(self.client) + return client + + def ensure_quota_limits(self): + """Ensures quota limits before creating a new stack + """ + try: + self.ensure_neutron_quota_limits() + self.ensure_nova_quota_limits() + except (nova_quota.EnsureNovaQuotaLimitsError, + neutron_quota.EnsureNeutronQuotaLimitsError) as ex: + raise InvalidFixtureError(name=self.fixture_name) from ex + + def ensure_neutron_quota_limits(self): + required_quota_set = self.neutron_required_quota_set + if required_quota_set: + neutron_quota.ensure_neutron_quota_limits(project=self.project, + **required_quota_set) + + def ensure_nova_quota_limits(self): + required_quota_set = self.nova_required_quota_set + if required_quota_set: + nova_quota.ensure_nova_quota_limits(project=self.project, + user=self.user, + **required_quota_set) + + @property + def neutron_required_quota_set(self) -> typing.Dict[str, int]: + return collections.defaultdict(int) + + @property + def nova_required_quota_set(self) -> typing.Dict[str, int]: + return collections.defaultdict(int) + + +class ResourceFixture(BaseResourceFixture, abc.ABC): """Base class for fixtures intended to manage Openstack resources not created using Heat, but with openstacksdk or other component clients (such as neutronclient, novaclient, manilaclient, etc). @@ -41,11 +117,11 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC): Child classes must define any other attributes required by the resource_create, resource_delete and resource_find methods. Examples: prefixes and default_prefixlen are needed for subnet_pools; description and - rules are needed for secutiry_groups; etc. + rules are needed for security_groups; etc. Child classes must define the resource_create, resource_delete and resource_find methods. In case of resource_create and resource_find, they - should return an object with the type defined for self._resouce. + should return an object with the type defined for self._resource. Child classes may optionally implement simple properties to access to resource_id and resource using a more representative name (these properties @@ -84,6 +160,7 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC): pass def setup_fixture(self): + super().setup_fixture() self.name = self.fixture_name if config.get_bool_env('TOBIKO_PREVENT_CREATE'): LOG.debug("%r should have been already created: %r", @@ -98,6 +175,9 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC): tobiko.addme_to_shared_resource(__name__, self.name) def try_create_resource(self): + # Ensure quota limits are OK just in time before start creating + # a new stack + self.ensure_quota_limits() if not self.resource: self._resource = self.resource_create() diff --git a/tobiko/openstack/heat/__init__.py b/tobiko/openstack/heat/__init__.py index ee19d0e81..4b9c90c1f 100644 --- a/tobiko/openstack/heat/__init__.py +++ b/tobiko/openstack/heat/__init__.py @@ -41,7 +41,6 @@ HeatStackNotFound = _stack.HeatStackNotFound heat_stack_parameters = _stack.heat_stack_parameters find_stack = _stack.find_stack list_stacks = _stack.list_stacks -InvalidStackError = _stack.InvalidStackError INIT_IN_PROGRESS = _stack.INIT_IN_PROGRESS INIT_COMPLETE = _stack.INIT_COMPLETE CREATE_IN_PROGRESS = _stack.CREATE_IN_PROGRESS diff --git a/tobiko/openstack/heat/_stack.py b/tobiko/openstack/heat/_stack.py index e7c47a507..ca38e88e9 100644 --- a/tobiko/openstack/heat/_stack.py +++ b/tobiko/openstack/heat/_stack.py @@ -13,7 +13,6 @@ # under the License. from __future__ import absolute_import -import collections from collections import abc import random import time @@ -28,8 +27,7 @@ from tobiko import config from tobiko.openstack.heat import _client from tobiko.openstack.heat import _template from tobiko.openstack import keystone -from tobiko.openstack import neutron -from tobiko.openstack import nova +from tobiko.openstack.base import _fixture as base_fixture LOG = log.getLogger(__name__) @@ -110,7 +108,7 @@ def find_stack(client: _client.HeatClientType = None, @keystone.skip_unless_has_keystone_credentials() -class HeatStackFixture(tobiko.SharedFixture): +class HeatStackFixture(base_fixture.BaseResourceFixture): """Manages Heat stacks.""" client: _client.HeatClientType = None @@ -124,8 +122,6 @@ class HeatStackFixture(tobiko.SharedFixture): stack: typing.Optional[StackType] = None stack_name: typing.Optional[str] = None parameters: typing.Optional['HeatStackParametersFixture'] = None - project: typing.Optional[str] = None - user: typing.Optional[str] = None output_needs_stack_complete: bool = True def __init__( @@ -150,12 +146,10 @@ class HeatStackFixture(tobiko.SharedFixture): self.wait_interval = wait_interval def setup_fixture(self): + super().setup_fixture() self.setup_stack_name() self.setup_template() self.setup_parameters() - self.setup_client() - self.setup_project() - self.setup_user() self.setup_stack() def setup_template(self): @@ -180,14 +174,6 @@ class HeatStackFixture(tobiko.SharedFixture): def session(self): return self.setup_client().http_client.session - def setup_project(self): - if self.project is None: - self.project = keystone.get_project_id(session=self.session) - - def setup_user(self): - if self.user is None: - self.user = keystone.get_user_id(session=self.session) - def setup_stack(self) -> stacks.Stack: stack = self.create_stack() tobiko.addme_to_shared_resource(__name__, stack.stack_name) @@ -207,7 +193,7 @@ class HeatStackFixture(tobiko.SharedFixture): try: stack = self.try_create_stack() break - except InvalidStackError: + except base_fixture.InvalidFixtureError: LOG.exception(f"Error creating stack '{self.stack_name}'", exc_info=1) if attempt.is_last: @@ -290,7 +276,7 @@ class HeatStackFixture(tobiko.SharedFixture): f"id='{stack_id}'.") try: stack = self.validate_created_stack() - except InvalidStackError as ex: + except base_fixture.InvalidFixtureError as ex: LOG.debug(f'Deleting invalid stack (name={self.stack_name}, "' f'"id={stack_id}): {ex}') # the stack shelf counter does not need to be decreased here, @@ -517,37 +503,6 @@ class HeatStackFixture(tobiko.SharedFixture): message = "Object {!r} has no attribute {!r}".format(self, name) raise AttributeError(message) - def ensure_quota_limits(self): - """Ensures quota limits before creating a new stack - """ - try: - self.ensure_neutron_quota_limits() - self.ensure_nova_quota_limits() - except (nova.EnsureNovaQuotaLimitsError, - neutron.EnsureNeutronQuotaLimitsError) as ex: - raise InvalidStackError(name=self.stack_name) from ex - - def ensure_neutron_quota_limits(self): - required_quota_set = self.neutron_required_quota_set - if required_quota_set: - neutron.ensure_neutron_quota_limits(project=self.project, - **required_quota_set) - - def ensure_nova_quota_limits(self): - required_quota_set = self.nova_required_quota_set - if required_quota_set: - nova.ensure_nova_quota_limits(project=self.project, - user=self.user, - **required_quota_set) - - @property - def neutron_required_quota_set(self) -> typing.Dict[str, int]: - return collections.defaultdict(int) - - @property - def nova_required_quota_set(self) -> typing.Dict[str, int]: - return collections.defaultdict(int) - class HeatStackKeyError(tobiko.TobikoException): message = "key {key!r} not found in stack {name!r}" @@ -717,11 +672,7 @@ class HeatStackNotFound(HeatStackError): message = "stack {name!r} not found" -class InvalidStackError(HeatStackError): - message = "invalid stack {name!r}" - - -class InvalidHeatStackStatus(InvalidStackError): +class InvalidHeatStackStatus(base_fixture.InvalidFixtureError): message = ("stack {name!r} status {observed!r} not in {expected!r}\n" "{status_reason!s}") diff --git a/tobiko/openstack/stacks/_neutron.py b/tobiko/openstack/stacks/_neutron.py index 4f762b775..a226803b0 100644 --- a/tobiko/openstack/stacks/_neutron.py +++ b/tobiko/openstack/stacks/_neutron.py @@ -26,8 +26,8 @@ import tobiko from tobiko import config from tobiko.openstack import heat from tobiko.openstack import neutron +from tobiko.openstack.base import _fixture as base_fixture from tobiko.openstack.stacks import _hot -from tobiko.openstack.stacks import _fixture from tobiko.shell import ip from tobiko.shell import sh from tobiko.shell import ssh @@ -284,7 +284,7 @@ class RouterNoSnatStackFixture(RouterStackFixture): @neutron.skip_if_missing_networking_extensions('subnet_allocation') -class SubnetPoolFixture(_fixture.ResourceFixture): +class SubnetPoolFixture(base_fixture.ResourceFixture): """Neutron Subnet Pool Fixture. A subnet pool is a dependency of network fixtures with either IPv4 or @@ -583,7 +583,7 @@ class SecurityGroupsFixture(heat.HeatStackFixture): @neutron.skip_if_missing_networking_extensions('stateful-security-group') -class StatelessSecurityGroupFixture(_fixture.ResourceFixture): +class StatelessSecurityGroupFixture(base_fixture.ResourceFixture): """Neutron Stateless Security Group Fixture. This SG will by default allow SSH and ICMP to the instance and also @@ -615,6 +615,12 @@ class StatelessSecurityGroupFixture(_fixture.ResourceFixture): self.description = description or self.description self.rules = rules or self.rules + @property + def neutron_required_quota_set(self) -> typing.Dict[str, int]: + requirements = super().neutron_required_quota_set + requirements['security_group'] += 1 + return requirements + @property def security_group_id(self): return self.resource_id diff --git a/tobiko/openstack/stacks/_nova.py b/tobiko/openstack/stacks/_nova.py index f252f2a88..02b76ccb1 100644 --- a/tobiko/openstack/stacks/_nova.py +++ b/tobiko/openstack/stacks/_nova.py @@ -28,6 +28,7 @@ from tobiko.openstack import glance from tobiko.openstack import heat from tobiko.openstack import neutron from tobiko.openstack import nova +from tobiko.openstack.base import _fixture as base_fixture from tobiko.openstack.stacks import _hot from tobiko.openstack.stacks import _neutron from tobiko.shell import curl @@ -271,7 +272,8 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC): self.validate_different_host_scheduler_hints( hypervisor=hypervisor) except nova.MigrateServerError as ex: - raise heat.InvalidStackError(name=self.stack_name) from ex + raise base_fixture.InvalidFixtureError( + name=self.stack_name) from ex def validate_same_host_scheduler_hints(self, hypervisor): if self.same_host: diff --git a/tobiko/tests/scenario/nova/test_server.py b/tobiko/tests/scenario/nova/test_server.py index 0791e9711..02624f5a8 100644 --- a/tobiko/tests/scenario/nova/test_server.py +++ b/tobiko/tests/scenario/nova/test_server.py @@ -22,10 +22,10 @@ import pytest import testtools import tobiko -from tobiko.openstack import heat from tobiko.openstack import keystone from tobiko.openstack import nova from tobiko.openstack import stacks +from tobiko.openstack.base import _fixture as base_fixture from tobiko.shell import ping @@ -177,7 +177,7 @@ class CirrosServerStackFixture(stacks.CirrosServerStackFixture): try: nova.activate_server(server) except nova.WaitForServerStatusTimeout as ex: - raise heat.InvalidStackError( + raise base_fixture.InvalidFixtureError( name=self.stack_name) from ex return stack