diff --git a/tobiko/openstack/keystone/__init__.py b/tobiko/openstack/keystone/__init__.py index cb073e2f9..0ffe131ee 100644 --- a/tobiko/openstack/keystone/__init__.py +++ b/tobiko/openstack/keystone/__init__.py @@ -16,6 +16,7 @@ from __future__ import absolute_import from tobiko.openstack.keystone import _client from tobiko.openstack.keystone import _clouds_file from tobiko.openstack.keystone import _credentials +from tobiko.openstack.keystone import _resource from tobiko.openstack.keystone import _services from tobiko.openstack.keystone import _session @@ -47,6 +48,13 @@ InvalidKeystoneCredentials = _credentials.InvalidKeystoneCredentials DEFAULT_KEYSTONE_CREDENTIALS_FIXTURES = \ _credentials.DEFAULT_KEYSTONE_CREDENTIALS_FIXTURES +get_keystone_resource_id = _resource.get_keystone_resource_id +get_project_id = _resource.get_project_id +get_user_id = _resource.get_user_id +KeystoneResourceType = _resource.KeystoneResourceType +ProjectType = _resource.ProjectType +UserType = _resource.UserType + has_service = _services.has_service is_service_missing = _services.is_service_missing skip_if_missing_service = _services.skip_if_missing_service diff --git a/tobiko/openstack/keystone/_client.py b/tobiko/openstack/keystone/_client.py index fa3f9029c..77ccd6991 100644 --- a/tobiko/openstack/keystone/_client.py +++ b/tobiko/openstack/keystone/_client.py @@ -13,6 +13,8 @@ # under the License. from __future__ import absolute_import +import typing + from keystoneclient import base from keystoneclient import client as keystoneclient from keystoneclient.v2_0 import client as v2_client @@ -36,13 +38,16 @@ class KeystoneClientManager(_client.OpenstackClientManager): CLIENTS = KeystoneClientManager() - - CLIENT_CLASSES = (v2_client.Client, v3_client.Client) +KeystoneClient = typing.Union[v2_client.Client, v3_client.Client] +KeystoneClientType = typing.Union[KeystoneClient, + KeystoneClientFixture, + typing.Type[KeystoneClientFixture], + None] -def keystone_client(obj): - if not obj: +def keystone_client(obj: KeystoneClientType) -> KeystoneClient: + if obj is None: return get_keystone_client() if isinstance(obj, CLIENT_CLASSES): @@ -50,14 +55,13 @@ def keystone_client(obj): fixture = tobiko.setup_fixture(obj) if isinstance(fixture, KeystoneClientFixture): - return fixture.client + return tobiko.setup_fixture(obj).client - message = "Object {!r} is not a KeystoneClientFixture".format(obj) - raise TypeError(message) + raise TypeError(f"Object {obj} is not a KeystoneClientFixture") def get_keystone_client(session=None, shared=True, init_client=None, - manager=None): + manager=None) -> KeystoneClient: manager = manager or CLIENTS client = manager.get_client(session=session, shared=shared, init_client=init_client) diff --git a/tobiko/openstack/keystone/_resource.py b/tobiko/openstack/keystone/_resource.py new file mode 100644 index 000000000..cba23912c --- /dev/null +++ b/tobiko/openstack/keystone/_resource.py @@ -0,0 +1,60 @@ +# Copyright 2021 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import typing + +from keystoneclient import base +from keystoneclient.v2_0 import tenants as tenants_v2 +from keystoneclient.v2_0 import users as users_v2 +from keystoneclient.v3 import projects as projects_v3 +from keystoneclient.v3 import users as users_v3 + + +KeystoneResourceType = typing.Union[str, base.Resource] +ProjectType = typing.Union[str, tenants_v2.Tenant, projects_v3.Project] +UserType = typing.Union[str, users_v2.User, users_v3.User] + + +def get_project_id( + project: typing.Optional[ProjectType] = None, + session=None) -> str: + if project is not None: + return get_keystone_resource_id(project) + if session is not None: + return session.auth.auth_ref.project_id + raise ValueError("'project' and 'session' can't be None ata the same " + "time.") + + +def get_user_id( + user: typing.Optional[UserType] = None, + session=None) -> str: + if user is not None: + return get_keystone_resource_id(user) + if session is not None: + return session.auth.auth_ref.user_id + raise ValueError("'project' and 'session' can't be None ata the same " + "time.") + + +def get_keystone_resource_id(resource: KeystoneResourceType) -> str: + if isinstance(resource, str): + return resource + + if isinstance(resource, base.Resource): + return resource.id + + raise TypeError(f"Object {resource} is not a valid Keystone resource " + "type") diff --git a/tobiko/openstack/nova/__init__.py b/tobiko/openstack/nova/__init__.py index c46bf8a36..8b247eb7c 100644 --- a/tobiko/openstack/nova/__init__.py +++ b/tobiko/openstack/nova/__init__.py @@ -16,11 +16,13 @@ from __future__ import absolute_import from tobiko.openstack.nova import _client from tobiko.openstack.nova import _cloud_init from tobiko.openstack.nova import _hypervisor +from tobiko.openstack.nova import _quota_set from tobiko.openstack.nova import _server from tobiko.openstack.nova import _service - CLIENT_CLASSES = _client.CLIENT_CLASSES +NovaClient = _client.NovaClient +NovaClientType = _client.NovaClientType get_console_output = _client.get_console_output get_nova_client = _client.get_nova_client get_server = _client.get_server @@ -38,8 +40,10 @@ WaitForServerStatusError = _client.WaitForServerStatusError WaitForServerStatusTimeout = _client.WaitForServerStatusTimeout shutoff_server = _client.shutoff_server activate_server = _client.activate_server +ensure_server_status = _client.ensure_server_status migrate_server = _client.migrate_server confirm_resize = _client.confirm_resize +NovaServer = _client.NovaServer WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError cloud_config = _cloud_init.cloud_config @@ -54,6 +58,10 @@ get_different_host_hypervisors = _hypervisor.get_different_host_hypervisors get_server_hypervisor = _hypervisor.get_server_hypervisor get_servers_hypervisors = _hypervisor.get_servers_hypervisors +get_nova_quota_set = _quota_set.get_nova_quota_set +ensure_nova_quota_limits = _quota_set.ensure_nova_quota_limits +set_nova_quota_set = _quota_set.set_nova_quota_set + find_server_ip_address = _server.find_server_ip_address HasServerMixin = _server.HasServerMixin list_server_ip_addresses = _server.list_server_ip_addresses diff --git a/tobiko/openstack/nova/_client.py b/tobiko/openstack/nova/_client.py index 0783dc833..6ae967706 100644 --- a/tobiko/openstack/nova/_client.py +++ b/tobiko/openstack/nova/_client.py @@ -13,37 +13,45 @@ # under the License. from __future__ import absolute_import -import time +import typing -from novaclient import client as novaclient -from novaclient.v2 import client as client_v2 +import novaclient +import novaclient.v2.client from oslo_log import log import tobiko from tobiko.openstack import _client -CLIENT_CLASSES = (client_v2.Client,) LOG = log.getLogger(__name__) +CLIENT_CLASSES = (novaclient.v2.client.Client,) +NovaClient = typing.Union[novaclient.v2.client.Client] +NovaServer = typing.Union[novaclient.v2.servers.Server] + class NovaClientFixture(_client.OpenstackClientFixture): - def init_client(self, session): - return novaclient.Client('2', session=session) + def init_client(self, session) -> NovaClient: + return novaclient.client.Client('2', session=session) class NovaClientManager(_client.OpenstackClientManager): - def create_client(self, session): + def create_client(self, session) -> NovaClientFixture: return NovaClientFixture(session=session) CLIENTS = NovaClientManager() +NovaClientType = typing.Union[NovaClient, + NovaClientFixture, + typing.Type[NovaClientFixture], + None] -def nova_client(obj): - if not obj: + +def nova_client(obj: NovaClientType) -> NovaClient: + if obj is None: return get_nova_client() if isinstance(obj, CLIENT_CLASSES): @@ -51,14 +59,15 @@ def nova_client(obj): fixture = tobiko.setup_fixture(obj) if isinstance(fixture, NovaClientFixture): + assert fixture.client is not None return fixture.client - message = "Object {!r} is not a NovaClientFixture".format(obj) + message = f"Object '{obj}' is not a NovaClientFixture" raise TypeError(message) def get_nova_client(session=None, shared=True, init_client=None, - manager=None): + manager=None) -> NovaClient: manager = manager or CLIENTS client = manager.get_client(session=session, shared=shared, init_client=init_client) @@ -66,13 +75,13 @@ def get_nova_client(session=None, shared=True, init_client=None, return client.client -def list_hypervisors(client=None, detailed=True, **params): +def list_hypervisors(client: NovaClientType = None, detailed=True, **params): client = nova_client(client) hypervisors = client.hypervisors.list(detailed=detailed) return tobiko.select(hypervisors).with_attributes(**params) -def find_hypervisor(client=None, unique=False, **params): +def find_hypervisor(client: NovaClientType = None, unique=False, **params): hypervisors = list_hypervisors(client=client, **params) if unique: return hypervisors.unique @@ -80,13 +89,14 @@ def find_hypervisor(client=None, unique=False, **params): return hypervisors.first -def list_servers(client=None, **params): - client = nova_client(client) - servers = client.servers.list() +def list_servers(client: NovaClientType = None, **params) -> \ + tobiko.Selection[NovaServer]: + servers = nova_client(client).servers.list() return tobiko.select(servers).with_attributes(**params) -def find_server(client=None, unique=False, **params): +def find_server(client: NovaClientType = None, unique=False, **params) -> \ + NovaServer: servers = list_servers(client=client, **params) if unique: return servers.unique @@ -94,13 +104,13 @@ def find_server(client=None, unique=False, **params): return servers.first -def list_services(client=None, **params) -> tobiko.Selection: +def list_services(client: NovaClientType = None, **params) -> tobiko.Selection: client = nova_client(client) services = client.services.list() return tobiko.select(services).with_attributes(**params) -def find_service(client=None, unique=False, **params): +def find_service(client: NovaClientType = None, unique=False, **params): services = list_services(client=client, **params) if unique: return services.unique @@ -108,29 +118,42 @@ def find_service(client=None, unique=False, **params): return services.first -def get_server_id(server): - if isinstance(server, str): - return server - else: - return server.id +ServerType = typing.Union[str, NovaServer] -def get_server(server, client=None, **params): - server_id = get_server_id(server) +def get_server_id(server: typing.Optional[ServerType] = None, + server_id: typing.Optional[str] = None) -> str: + if server_id is None: + if isinstance(server, str): + server_id = server + else: + assert server is not None + server_id = server.id + return server_id + + +def get_server(server: typing.Optional[ServerType] = None, + server_id: typing.Optional[str] = None, + client: NovaClientType = None, **params) -> NovaServer: + server_id = get_server_id(server=server, server_id=server_id) return nova_client(client).servers.get(server_id, **params) -def migrate_server(server, client=None, **params): +def migrate_server(server: typing.Optional[ServerType] = None, + server_id: typing.Optional[str] = None, + client: NovaClientType = None, **params): # pylint: disable=protected-access - server_id = get_server_id(server) + server_id = get_server_id(server=server, server_id=server_id) LOG.debug(f"Start server migration (server_id='{server_id}', " f"info={params})") return nova_client(client).servers._action('migrate', server_id, info=params) -def confirm_resize(server, client=None, **params): - server_id = get_server_id(server) +def confirm_resize(server: typing.Optional[ServerType] = None, + server_id: typing.Optional[str] = None, + client: NovaClientType = None, **params): + server_id = get_server_id(server=server, server_id=server_id) LOG.debug(f"Confirm server resize (server_id='{server_id}', " f"info={params})") return nova_client(client).servers.confirm_resize(server_id, **params) @@ -139,46 +162,67 @@ def confirm_resize(server, client=None, **params): MAX_SERVER_CONSOLE_OUTPUT_LENGTH = 1024 * 256 -def get_console_output(server, timeout=None, interval=1., length=None, - client=None): - client = nova_client(client) - start_time = time.time() +def get_console_output(server: typing.Optional[ServerType] = None, + server_id: typing.Optional[str] = None, + timeout: tobiko.Seconds = None, + interval: tobiko.Seconds = None, + length: typing.Optional[int] = None, + client: NovaClientType = None) -> \ + typing.Optional[str]: if length is not None: length = min(length, MAX_SERVER_CONSOLE_OUTPUT_LENGTH) else: length = MAX_SERVER_CONSOLE_OUTPUT_LENGTH - while True: + + server_id = get_server_id(server=server, server_id=server_id) + + for attempt in tobiko.retry(timeout=timeout, + interval=interval, + default_timeout=60., + default_interval=5.): try: - output = client.servers.get_console_output(server=server, - length=length) - except TypeError: - # For some reason it could happen resulting body cannot be - # translated to json object and it is converted to None - # on such case get_console_output would raise a TypeError - return None + output = nova_client(client).servers.get_console_output( + server=server_id, length=length) + except (TypeError, novaclient.exceptions.NotFound): + # Only active servers have console output + server = get_server(server_id=server_id) + if server.status != 'ACTIVE': + LOG.debug(f"Server '{server_id}' has no console output " + f"(status = '{server.status}').") + break + else: + # For some reason it could happen resulting body cannot be + # translated to json object and it is converted to None + # on such case get_console_output would raise a TypeError + LOG.exception(f"Error getting server '{server_id}' console " + "output") + else: + if output: + LOG.debug(f"got server '{server_id}' console output " + f"(length = {len(output)}).") + return output - if timeout is None or output: + try: + attempt.check_limits() + except tobiko.RetryLimitError: + LOG.info(f"No console output produced by server '{server_id}') " + f" after {attempt.elapsed_time} seconds") break + else: + LOG.debug(f"Waiting for server '{server_id}' console output...") - if time.time() - start_time > timeout: - LOG.warning("No console output produced by server (%r) after " - "%r seconds", server, timeout) - break - - LOG.debug('Waiting for server (%r) console output...', server) - time.sleep(interval) - - return output + return None class HasNovaClientMixin(object): - nova_client = None + nova_client: NovaClientType = None - def get_server(self, server, **params): + def get_server(self, server: ServerType, **params) -> NovaServer: return get_server(server=server, client=self.nova_client, **params) - def get_server_console_output(self, server, **params): + def get_server_console_output(self, server: ServerType, **params) -> \ + typing.Optional[str]: return get_console_output(server=server, client=self.nova_client, **params) @@ -193,76 +237,113 @@ class WaitForServerStatusTimeout(WaitForServerStatusError): "{server_status} to {status} status after {timeout} seconds") -NOVA_SERVER_TRANSIENT_STATUS = { - 'ACTIVE': ('BUILD', 'SHUTOFF'), - 'SHUTOFF': ('ACTIVE'), - 'VERIFY_RESIZE': ('RESIZE'), +NOVA_SERVER_TRANSIENT_STATUS: typing.Dict[str, typing.List[str]] = { + 'ACTIVE': ['BUILD', 'SHUTOFF'], + 'SHUTOFF': ['ACTIVE'], + 'VERIFY_RESIZE': ['RESIZE'], } -def wait_for_server_status(server, status, client=None, timeout=None, - sleep_time=None, transient_status=None): - if timeout is None: - timeout = 300. - if sleep_time is None: - sleep_time = 5. - start_time = time.time() +def wait_for_server_status( + server: ServerType, + status: str, + client: NovaClientType = None, + timeout: tobiko.Seconds = None, + sleep_time: tobiko.Seconds = None, + transient_status: typing.Optional[typing.List[str]] = None) -> \ + NovaServer: if transient_status is None: - transient_status = NOVA_SERVER_TRANSIENT_STATUS.get(status) or tuple() - while True: - server = get_server(server=server, client=client) - if server.status == status: + transient_status = NOVA_SERVER_TRANSIENT_STATUS.get(status) or [] + server_id = get_server_id(server) + for attempt in tobiko.retry(timeout=timeout, + interval=sleep_time, + default_timeout=300., + default_interval=5.): + _server = get_server(server_id=server_id, client=client) + if _server.status == status: break - if server.status not in transient_status: - raise WaitForServerStatusError(server_id=server.id, - server_status=server.status, + if _server.status not in transient_status: + raise WaitForServerStatusError(server_id=server_id, + server_status=_server.status, status=status) - - if time.time() - start_time >= timeout: - raise WaitForServerStatusTimeout(server_id=server.id, - server_status=server.status, + try: + attempt.check_time_left() + except tobiko.RetryTimeLimitError as ex: + raise WaitForServerStatusTimeout(server_id=server_id, + server_status=_server.status, status=status, - timeout=timeout) + timeout=timeout) from ex progress = getattr(server, 'progress', None) - LOG.debug(f"Waiting for server {server.id} status to get from " - f"{server.status} to {status} " + LOG.debug(f"Waiting for server {server_id} status to get from " + f"{_server.status} to {status} " f"(progress={progress}%)") - time.sleep(sleep_time) - return server + + return _server -def shutoff_server(server, client=None, timeout=None, sleep_time=None): +def shutoff_server(server: ServerType = None, + client: NovaClientType = None, + timeout: tobiko.Seconds = None, + sleep_time: tobiko.Seconds = None) -> NovaServer: client = nova_client(client) server = get_server(server=server, client=client) if server.status == 'SHUTOFF': return server + LOG.info(f"stop server '{server.id}' (status='{server.status}').") client.servers.stop(server.id) - return wait_for_server_status(server=server.id, status='SHUTOFF', - client=client, timeout=timeout, + return wait_for_server_status(server=server.id, + status='SHUTOFF', + client=client, + timeout=timeout, sleep_time=sleep_time) -def activate_server(server, client=None, timeout=None, sleep_time=None): +def activate_server(server: ServerType, + client: NovaClientType = None, + timeout: tobiko.Seconds = None, + sleep_time: tobiko.Seconds = None) -> NovaServer: client = nova_client(client) server = get_server(server=server, client=client) if server.status == 'ACTIVE': return server if server.status == 'SHUTOFF': + LOG.info(f"Start server '{server.id}' (status='{server.status}').") client.servers.start(server.id) elif server.status == 'RESIZE': - wait_for_server_status(server=server.id, status='VERIFY_RESIZE', - client=client, timeout=timeout, - sleep_time=sleep_time) + server = wait_for_server_status( + server=server.id, status='VERIFY_RESIZE', client=client, + timeout=timeout, sleep_time=sleep_time) + LOG.info(f"Confirm resize of server '{server.id}' " + f"(status='{server.status}').") client.servers.confirm_resize(server) elif server.status == 'VERIFY_RESIZE': + LOG.info(f"Confirm resize of server '{server.id}' " + f"(status='{server.status}').") client.servers.confirm_resize(server) else: + LOG.warning(f"Try activating server '{server.id}' by rebooting " + f"it (status='{server.status}').") client.servers.reboot(server.id, reboot_type='HARD') return wait_for_server_status(server=server.id, status='ACTIVE', client=client, timeout=timeout, sleep_time=sleep_time) + + +def ensure_server_status(server: ServerType, + status: str, + client: NovaClientType = None, + timeout: tobiko.Seconds = None, + sleep_time: tobiko.Seconds = None) -> NovaServer: + if status == 'ACTIVE': + return activate_server(server=server, client=client, timeout=timeout, + sleep_time=sleep_time) + elif status == 'SHUTOFF': + return shutoff_server(server=server, client=client, timeout=timeout, + sleep_time=sleep_time) + else: + raise ValueError(f"Unsupported server status: '{status}'") diff --git a/tobiko/openstack/nova/_quota_set.py b/tobiko/openstack/nova/_quota_set.py new file mode 100644 index 000000000..063e6ca3c --- /dev/null +++ b/tobiko/openstack/nova/_quota_set.py @@ -0,0 +1,87 @@ +# Copyright 2021 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +from oslo_log import log + +from tobiko.openstack import keystone +from tobiko.openstack.nova import _client + + +LOG = log.getLogger(__name__) + + +def get_nova_quota_set(project: keystone.ProjectType = None, + user: keystone.UserType = None, + client: _client.NovaClientType = None, + **params): + client = _client.nova_client(client) + project_id = keystone.get_project_id(project=project, + session=client.client.session) + user_id = user and keystone.get_user_id(user=user) or None + return client.quotas.get(project_id, user_id=user_id, **params) + + +def set_nova_quota_set(project: keystone.ProjectType = None, + user: keystone.UserType = None, + client: _client.NovaClientType = None, + **params): + client = _client.nova_client(client) + project_id = keystone.get_project_id(project=project, + session=client.client.session) + user_id = user and keystone.get_user_id(user=user) or None + return client.quotas.update(project_id, user_id=user_id, **params) + + +def ensure_nova_quota_limits(project: keystone.ProjectType = None, + user: keystone.UserType = None, + client: _client.NovaClientType = None, + **required): + client = _client.nova_client(client) + project = keystone.get_project_id(project=project, + session=client.client.session) + user = user and keystone.get_user_id(user=user) or None + if user: + # Must increase project limits before user ones + ensure_nova_quota_limits(project=project, client=client, + **required) + + quota_set = get_nova_quota_set(project=project, user=user, + client=client, detail=True) + actual_limits = {} + increment_limits = {} + for name, needed in required.items(): + quota = getattr(quota_set, name) + limit: int = quota['limit'] + if limit > 0: + in_use: int = max(0, quota['in_use']) + max(0, quota['reserved']) + if in_use + needed >= limit: + actual_limits[name] = limit + increment_limits[name] = max(10, limit * 2) + + if increment_limits: + LOG.info(f"Increment Nova quota limits (project={project}, " + f"user={user}): {actual_limits} -> {increment_limits}...") + set_nova_quota_set(project=project, user=user, client=client, + **increment_limits) + quota_set = get_nova_quota_set(project=project, user=user, + client=client, detail=True) + new_limits = { + name: getattr(quota_set, name)['limit'] + for name in increment_limits.keys()} + + LOG.info(f"Nova quota limit increased (project={project}, " + f"user={user}): {actual_limits} -> {new_limits}...") + + return quota_set diff --git a/tobiko/openstack/stacks/_nova.py b/tobiko/openstack/stacks/_nova.py index 7abee8a6a..f398608cb 100644 --- a/tobiko/openstack/stacks/_nova.py +++ b/tobiko/openstack/stacks/_nova.py @@ -27,6 +27,7 @@ import tobiko from tobiko import config from tobiko.openstack import glance from tobiko.openstack import heat +from tobiko.openstack import keystone from tobiko.openstack import neutron from tobiko.openstack import nova from tobiko.openstack.stacks import _hot @@ -96,6 +97,10 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC): #: stack with the internal where the server port is created network_stack = tobiko.required_setup_fixture(_neutron.NetworkStackFixture) + def create_stack(self, retry=None): + self.ensure_quota_limits() + super(ServerStackFixture, self).create_stack(retry=retry) + @property def image_fixture(self) -> glance.GlanceImageFixture: """Glance image used to create a Nova server instance""" @@ -320,22 +325,50 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC): 'maxsize': self.swap_maxsize}) return cloud_config - def ensure_server_status(self, status): - tobiko.setup_fixture(self) - try: - server = nova.wait_for_server_status(self.server_id, status) - except nova.WaitForServerStatusError: - server = nova.get_server(self.server_id) - LOG.debug(f"Server {server.id} status is {server.status} instead " - f"of {status}", exc_info=1) - if server.status == status: - return server - elif status == "ACTIVE": - tobiko.reset_fixture(self) - return nova.wait_for_server_status(self.server_id, 'ACTIVE') - else: - tobiko.skip_test(f"{type(self).__name__}.ensure_server_status " - "method not implemented") + def ensure_server_status( + self, status: str, + retry_count: typing.Optional[int] = None, + retry_timeout: tobiko.Seconds = None, + retry_interval: tobiko.Seconds = None): + self.ssh_client.close() + for attempt in tobiko.retry(count=retry_count, + timeout=retry_timeout, + interval=retry_interval, + default_count=3, + default_timeout=900., + default_interval=5.): + tobiko.setup_fixture(self) + server_id = self.server_id + try: + server = nova.ensure_server_status( + server=server_id, + status=status, + timeout=attempt.time_left) + except nova.WaitForServerStatusError: + attempt.check_limits() + LOG.warning( + f"Unable to change server '{server_id}' status to " + f"'{status}'", + exc_info=1) + tobiko.cleanup_fixture(self) + else: + assert server.status == status + break + + return server + + def ensure_quota_limits(self): + """Ensures Nova quota limits before creating a new server + """ + project = keystone.get_project_id( + session=self.client.http_client.session) + user = keystone.get_user_id( + session=self.client.http_client.session) + nova.ensure_nova_quota_limits( + project=project, + user=user, + instances=1, + cores=self.flavor_stack.vcpus or 1) class ExternalServerStackFixture(ServerStackFixture, abc.ABC): diff --git a/tobiko/tests/functional/openstack/test_nova.py b/tobiko/tests/functional/openstack/test_nova.py index a2f1dcea0..6f7baa489 100644 --- a/tobiko/tests/functional/openstack/test_nova.py +++ b/tobiko/tests/functional/openstack/test_nova.py @@ -103,21 +103,32 @@ class ClientTest(testtools.TestCase): self.assertEqual(server_id, server.id) self.assertEqual('ACTIVE', server.status) - def test_shutof_and_activate_server(self): - server_id = self.useFixture(stacks.CirrosServerStackFixture( - stack_name=self.id())).server_id - server = nova.wait_for_server_status(server=server_id, status='ACTIVE') - self.assertEqual(server_id, server.id) +class ServerActionsStack(stacks.CirrosServerStackFixture): + pass + + +class ServerActionsTest(testtools.TestCase): + + stack = tobiko.required_setup_fixture(ServerActionsStack) + + def test_activate_server(self, initial_status='SHUTOFF'): + self.stack.ensure_server_status(initial_status) + server = nova.activate_server(self.stack.server_id) self.assertEqual('ACTIVE', server.status) + ping.assert_reachable_hosts([self.stack.ip_address]) - server = nova.shutoff_server(server=server_id) - self.assertEqual(server_id, server.id) + def test_activate_server_when_shutoff(self): + self.test_activate_server(initial_status='SHUTOFF') + + def test_shutoff_server(self, initial_status='ACTIVE'): + self.stack.ensure_server_status(initial_status) + server = nova.shutoff_server(self.stack.server_id) self.assertEqual('SHUTOFF', server.status) + ping.assert_unreachable_hosts([self.stack.ip_address]) - server = nova.activate_server(server=server_id) - self.assertEqual(server_id, server.id) - self.assertEqual('ACTIVE', server.status) + def test_shutoff_server_when_shutoff(self): + self.test_shutoff_server(initial_status='SHUTOFF') @keystone.skip_unless_has_keystone_credentials() @@ -194,8 +205,8 @@ class MigrateServerTest(testtools.TestCase): target_hypervisor = hypervisor.hypervisor_hostname break else: - self.skip("Cannot find a valid hypervisor host to migrate server " - "to") + self.skipTest("Cannot find a valid hypervisor host to migrate " + "server to") server = self.migrate_server(server=server, host=target_hypervisor)