x/tobiko
Code Issues Proposed changes

Introduce new BaseResourceFixture class

Browse Source 
This new class is base for both: HeatStackFixture and ResourceFixture
classes. It contains some common pieces of code like e.g. ensuring
that quota is set as required by the fixture.

This is done because fixture StatelessSecurityGroup, which don't use
HeatStackFixture needs to set quota for security groups in Neutron but
ensuring neutron (and nova) quotas are set correctly were done only in
the HeatStackFixture class so far.

Change-Id: Id3d3207f098853469bef87020fc017bec5aaba93
This commit is contained in:
Slawek Kaplonski 2024-03-15 11:51:40 +01:00
parent 7733488f00
commit 26b6909ceb
7 changed files with 118 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
tobiko/openstack/base/__init__.py Normal file
View File

@ -0,0 +1,15 @@
# Copyright (c) 2024 Red Hat, Inc.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

86
tobiko/openstack/stacks/_fixture.py → tobiko/openstack/base/_fixture.py
View File

@ -14,18 +14,94 @@
from __future__ import absolute_import
import abc
import collections
import typing
from oslo_log import log
import tobiko
from tobiko import config
from tobiko.openstack import keystone
from tobiko.openstack.neutron import _quota_set as neutron_quota
from tobiko.openstack.nova import _quota_set as nova_quota
LOG = log.getLogger(__name__)
class ResourceFixture(tobiko.SharedFixture, abc.ABC):
class InvalidFixtureError(tobiko.TobikoException):
message = "invalid fixture {name!r}"
class BaseResourceFixture(tobiko.SharedFixture):
"""Base class for fixtures both types: those which uses heat stacks and
those which are not.
"""
client: keystone.KeystoneClient = None
project: typing.Optional[str] = None
user: typing.Optional[str] = None
def setup_fixture(self):
self.setup_client()
self.setup_project()
self.setup_user()
def setup_project(self):
if self.project is None:
self.project = keystone.get_project_id(session=self.session)
def setup_user(self):
if self.user is None:
self.user = keystone.get_user_id(session=self.session)
@property
def session(self):
return self.setup_client().session
def setup_client(self) -> keystone.KeystoneClient:
client = self.client
# NOTE(slaweq): it seems that due to bug
# https://github.com/python/mypy/issues/11673
# in mypy this line is causing arg-type error so lets
# ignore it for now
if not isinstance(
client, keystone.KeystoneClient): # type: ignore[arg-type]
self.client = client = keystone.keystone_client(self.client)
return client
def ensure_quota_limits(self):
"""Ensures quota limits before creating a new stack
"""
try:
self.ensure_neutron_quota_limits()
self.ensure_nova_quota_limits()
except (nova_quota.EnsureNovaQuotaLimitsError,
neutron_quota.EnsureNeutronQuotaLimitsError) as ex:
raise InvalidFixtureError(name=self.fixture_name) from ex
def ensure_neutron_quota_limits(self):
required_quota_set = self.neutron_required_quota_set
if required_quota_set:
neutron_quota.ensure_neutron_quota_limits(project=self.project,
**required_quota_set)
def ensure_nova_quota_limits(self):
required_quota_set = self.nova_required_quota_set
if required_quota_set:
nova_quota.ensure_nova_quota_limits(project=self.project,
user=self.user,
**required_quota_set)
@property
def neutron_required_quota_set(self) -> typing.Dict[str, int]:
return collections.defaultdict(int)
@property
def nova_required_quota_set(self) -> typing.Dict[str, int]:
return collections.defaultdict(int)
class ResourceFixture(BaseResourceFixture, abc.ABC):
"""Base class for fixtures intended to manage Openstack resources not
created using Heat, but with openstacksdk or other component clients (such
as neutronclient, novaclient, manilaclient, etc).
@ -41,11 +117,11 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC):
Child classes must define any other attributes required by the
resource_create, resource_delete and resource_find methods. Examples:
prefixes and default_prefixlen are needed for subnet_pools; description and
rules are needed for secutiry_groups; etc.
rules are needed for security_groups; etc.
Child classes must define the resource_create, resource_delete and
resource_find methods. In case of resource_create and resource_find, they
should return an object with the type defined for self._resouce.
should return an object with the type defined for self._resource.
Child classes may optionally implement simple properties to access to
resource_id and resource using a more representative name (these properties
@ -84,6 +160,7 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC):
pass
def setup_fixture(self):
super().setup_fixture()
self.name = self.fixture_name
if config.get_bool_env('TOBIKO_PREVENT_CREATE'):
LOG.debug("%r should have been already created: %r",
@ -98,6 +175,9 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC):
tobiko.addme_to_shared_resource(__name__, self.name)
def try_create_resource(self):
# Ensure quota limits are OK just in time before start creating
# a new stack
self.ensure_quota_limits()
if not self.resource:
self._resource = self.resource_create()

1
tobiko/openstack/heat/__init__.py
View File

@ -41,7 +41,6 @@ HeatStackNotFound = _stack.HeatStackNotFound
heat_stack_parameters = _stack.heat_stack_parameters
find_stack = _stack.find_stack
list_stacks = _stack.list_stacks
InvalidStackError = _stack.InvalidStackError
INIT_IN_PROGRESS = _stack.INIT_IN_PROGRESS
INIT_COMPLETE = _stack.INIT_COMPLETE
CREATE_IN_PROGRESS = _stack.CREATE_IN_PROGRESS

61
tobiko/openstack/heat/_stack.py
View File

@ -13,7 +13,6 @@
# under the License.
from __future__ import absolute_import
import collections
from collections import abc
import random
import time
@ -28,8 +27,7 @@ from tobiko import config
from tobiko.openstack.heat import _client
from tobiko.openstack.heat import _template
from tobiko.openstack import keystone
from tobiko.openstack import neutron
from tobiko.openstack import nova
from tobiko.openstack.base import _fixture as base_fixture
LOG = log.getLogger(__name__)
@ -110,7 +108,7 @@ def find_stack(client: _client.HeatClientType = None,
@keystone.skip_unless_has_keystone_credentials()
class HeatStackFixture(tobiko.SharedFixture):
class HeatStackFixture(base_fixture.BaseResourceFixture):
"""Manages Heat stacks."""
client: _client.HeatClientType = None
@ -124,8 +122,6 @@ class HeatStackFixture(tobiko.SharedFixture):
stack: typing.Optional[StackType] = None
stack_name: typing.Optional[str] = None
parameters: typing.Optional['HeatStackParametersFixture'] = None
project: typing.Optional[str] = None
user: typing.Optional[str] = None
output_needs_stack_complete: bool = True
def __init__(
@ -150,12 +146,10 @@ class HeatStackFixture(tobiko.SharedFixture):
self.wait_interval = wait_interval
def setup_fixture(self):
super().setup_fixture()
self.setup_stack_name()
self.setup_template()
self.setup_parameters()
self.setup_client()
self.setup_project()
self.setup_user()
self.setup_stack()
def setup_template(self):
@ -180,14 +174,6 @@ class HeatStackFixture(tobiko.SharedFixture):
def session(self):
return self.setup_client().http_client.session
def setup_project(self):
if self.project is None:
self.project = keystone.get_project_id(session=self.session)
def setup_user(self):
if self.user is None:
self.user = keystone.get_user_id(session=self.session)
def setup_stack(self) -> stacks.Stack:
stack = self.create_stack()
tobiko.addme_to_shared_resource(__name__, stack.stack_name)
@ -207,7 +193,7 @@ class HeatStackFixture(tobiko.SharedFixture):
try:
stack = self.try_create_stack()
break
except InvalidStackError:
except base_fixture.InvalidFixtureError:
LOG.exception(f"Error creating stack '{self.stack_name}'",
exc_info=1)
if attempt.is_last:
@ -290,7 +276,7 @@ class HeatStackFixture(tobiko.SharedFixture):
f"id='{stack_id}'.")
try:
stack = self.validate_created_stack()
except InvalidStackError as ex:
except base_fixture.InvalidFixtureError as ex:
LOG.debug(f'Deleting invalid stack (name={self.stack_name}, "'
f'"id={stack_id}): {ex}')
# the stack shelf counter does not need to be decreased here,
@ -517,37 +503,6 @@ class HeatStackFixture(tobiko.SharedFixture):
message = "Object {!r} has no attribute {!r}".format(self, name)
raise AttributeError(message)
def ensure_quota_limits(self):
"""Ensures quota limits before creating a new stack
"""
try:
self.ensure_neutron_quota_limits()
self.ensure_nova_quota_limits()
except (nova.EnsureNovaQuotaLimitsError,
neutron.EnsureNeutronQuotaLimitsError) as ex:
raise InvalidStackError(name=self.stack_name) from ex
def ensure_neutron_quota_limits(self):
required_quota_set = self.neutron_required_quota_set
if required_quota_set:
neutron.ensure_neutron_quota_limits(project=self.project,
**required_quota_set)
def ensure_nova_quota_limits(self):
required_quota_set = self.nova_required_quota_set
if required_quota_set:
nova.ensure_nova_quota_limits(project=self.project,
user=self.user,
**required_quota_set)
@property
def neutron_required_quota_set(self) -> typing.Dict[str, int]:
return collections.defaultdict(int)
@property
def nova_required_quota_set(self) -> typing.Dict[str, int]:
return collections.defaultdict(int)
class HeatStackKeyError(tobiko.TobikoException):
message = "key {key!r} not found in stack {name!r}"
@ -717,11 +672,7 @@ class HeatStackNotFound(HeatStackError):
message = "stack {name!r} not found"
class InvalidStackError(HeatStackError):
message = "invalid stack {name!r}"
class InvalidHeatStackStatus(InvalidStackError):
class InvalidHeatStackStatus(base_fixture.InvalidFixtureError):
message = ("stack {name!r} status {observed!r} not in {expected!r}\n"
"{status_reason!s}")

12
tobiko/openstack/stacks/_neutron.py
View File

@ -26,8 +26,8 @@ import tobiko
from tobiko import config
from tobiko.openstack import heat
from tobiko.openstack import neutron
from tobiko.openstack.base import _fixture as base_fixture
from tobiko.openstack.stacks import _hot
from tobiko.openstack.stacks import _fixture
from tobiko.shell import ip
from tobiko.shell import sh
from tobiko.shell import ssh
@ -284,7 +284,7 @@ class RouterNoSnatStackFixture(RouterStackFixture):
@neutron.skip_if_missing_networking_extensions('subnet_allocation')
class SubnetPoolFixture(_fixture.ResourceFixture):
class SubnetPoolFixture(base_fixture.ResourceFixture):
"""Neutron Subnet Pool Fixture.
A subnet pool is a dependency of network fixtures with either IPv4 or
@ -583,7 +583,7 @@ class SecurityGroupsFixture(heat.HeatStackFixture):
@neutron.skip_if_missing_networking_extensions('stateful-security-group')
class StatelessSecurityGroupFixture(_fixture.ResourceFixture):
class StatelessSecurityGroupFixture(base_fixture.ResourceFixture):
"""Neutron Stateless Security Group Fixture.
This SG will by default allow SSH and ICMP to the instance and also
@ -615,6 +615,12 @@ class StatelessSecurityGroupFixture(_fixture.ResourceFixture):
self.description = description or self.description
self.rules = rules or self.rules
@property
def neutron_required_quota_set(self) -> typing.Dict[str, int]:
requirements = super().neutron_required_quota_set
requirements['security_group'] += 1
return requirements
@property
def security_group_id(self):
return self.resource_id

4
tobiko/openstack/stacks/_nova.py
View File

@ -28,6 +28,7 @@ from tobiko.openstack import glance
from tobiko.openstack import heat
from tobiko.openstack import neutron
from tobiko.openstack import nova
from tobiko.openstack.base import _fixture as base_fixture
from tobiko.openstack.stacks import _hot
from tobiko.openstack.stacks import _neutron
from tobiko.shell import curl
@ -271,7 +272,8 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
self.validate_different_host_scheduler_hints(
hypervisor=hypervisor)
except nova.MigrateServerError as ex:
raise heat.InvalidStackError(name=self.stack_name) from ex
raise base_fixture.InvalidFixtureError(
name=self.stack_name) from ex
def validate_same_host_scheduler_hints(self, hypervisor):
if self.same_host:

4
tobiko/tests/scenario/nova/test_server.py
View File

@ -22,10 +22,10 @@ import pytest
import testtools
import tobiko
from tobiko.openstack import heat
from tobiko.openstack import keystone
from tobiko.openstack import nova
from tobiko.openstack import stacks
from tobiko.openstack.base import _fixture as base_fixture
from tobiko.shell import ping
@ -177,7 +177,7 @@ class CirrosServerStackFixture(stacks.CirrosServerStackFixture):
try:
nova.activate_server(server)
except nova.WaitForServerStatusTimeout as ex:
raise heat.InvalidStackError(
raise base_fixture.InvalidFixtureError(
name=self.stack_name) from ex
return stack